| 48 | |
| 49 | |
| 50 | class M4Summary: |
| 51 | def __init__(self, file_path, root_path): |
| 52 | self.file_path = file_path |
| 53 | self.training_set = M4Dataset.load(training=True, dataset_file=root_path) |
| 54 | self.test_set = M4Dataset.load(training=False, dataset_file=root_path) |
| 55 | self.naive_path = os.path.join(root_path, 'submission-Naive2.csv') |
| 56 | |
| 57 | def evaluate(self): |
| 58 | """ |
| 59 | Evaluate forecasts using M4 test dataset. |
| 60 | |
| 61 | :param forecast: Forecasts. Shape: timeseries, time. |
| 62 | :return: sMAPE and OWA grouped by seasonal patterns. |
| 63 | """ |
| 64 | grouped_owa = OrderedDict() |
| 65 | |
| 66 | naive2_forecasts = pd.read_csv(self.naive_path).values[:, 1:].astype(np.float32) |
| 67 | naive2_forecasts = np.array([v[~np.isnan(v)] for v in naive2_forecasts]) |
| 68 | |
| 69 | model_mases = {} |
| 70 | naive2_smapes = {} |
| 71 | naive2_mases = {} |
| 72 | grouped_smapes = {} |
| 73 | grouped_mapes = {} |
| 74 | for group_name in M4Meta.seasonal_patterns: |
| 75 | file_name = self.file_path + group_name + "_forecast.csv" |
| 76 | if os.path.exists(file_name): |
| 77 | model_forecast = pd.read_csv(file_name).values |
| 78 | |
| 79 | naive2_forecast = group_values(naive2_forecasts, self.test_set.groups, group_name) |
| 80 | target = group_values(self.test_set.values, self.test_set.groups, group_name) |
| 81 | # all timeseries within group have same frequency |
| 82 | frequency = self.training_set.frequencies[self.test_set.groups == group_name][0] |
| 83 | insample = group_values(self.training_set.values, self.test_set.groups, group_name) |
| 84 | |
| 85 | model_mases[group_name] = np.mean([mase(forecast=model_forecast[i], |
| 86 | insample=insample[i], |
| 87 | outsample=target[i], |
| 88 | frequency=frequency) for i in range(len(model_forecast))]) |
| 89 | naive2_mases[group_name] = np.mean([mase(forecast=naive2_forecast[i], |
| 90 | insample=insample[i], |
| 91 | outsample=target[i], |
| 92 | frequency=frequency) for i in range(len(model_forecast))]) |
| 93 | |
| 94 | naive2_smapes[group_name] = np.mean(smape_2(naive2_forecast, target)) |
| 95 | grouped_smapes[group_name] = np.mean(smape_2(forecast=model_forecast, target=target)) |
| 96 | grouped_mapes[group_name] = np.mean(mape(forecast=model_forecast, target=target)) |
| 97 | |
| 98 | grouped_smapes = self.summarize_groups(grouped_smapes) |
| 99 | grouped_mapes = self.summarize_groups(grouped_mapes) |
| 100 | grouped_model_mases = self.summarize_groups(model_mases) |
| 101 | grouped_naive2_smapes = self.summarize_groups(naive2_smapes) |
| 102 | grouped_naive2_mases = self.summarize_groups(naive2_mases) |
| 103 | for k in grouped_model_mases.keys(): |
| 104 | grouped_owa[k] = (grouped_model_mases[k] / grouped_naive2_mases[k] + |
| 105 | grouped_smapes[k] / grouped_naive2_smapes[k]) / 2 |
| 106 | |
| 107 | def round_all(d): |