-
Notifications
You must be signed in to change notification settings - Fork 23
/
interpret_results.py
executable file
·345 lines (300 loc) · 15.1 KB
/
interpret_results.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
import warnings
import numpy as np
import sklearn.metrics as metrics
from mosaiks.solve import data_parser as parse
from mosaiks.solve import solve_functions
def find_best_hp_idx(
kfold_metrics_outcome, hp_warnings_outcome, crit, minimize=False, val=None
):
"""Find the indices of the best hyperparameter combination,
as scored by 'crit'.
Args:
kfold_metrics_outcome (n_folds X n_outcomes x ... ndarray of dict (hps are last)):
Model performance array produced by kfold_solve for a
single outcome -- so n_outcomes must be 1
hp_warnings_outcome (nfolds x n_hps) bool array of whether an hp warning occured
crit (str): Key of the dicts in kfold_metrics that you want
to use to score the model performance
minimize (bool): If true, find minimum of crit. If false (default)
find maximum
val (str or int) : If not None, will tell you what outcome is being evaluated
if a warning is raised (e.g. hyperparameters hitting search grid bounds)
Returns:
tuple of int: Indices of optimal hyperparameters.
Length of tuple will be equal to number of
hyperparameters, i.e. len(kfold_metrics_test.shape) - 2
"""
if minimize:
finder = np.min
else:
finder = np.max
# allowable idxs is wherever a None or a False appears in hp_warnings_outcome
no_hp_warnings = (hp_warnings_outcome != True).all(axis=0)
didnt_record_hp_warnings = (hp_warnings_outcome == None).any(axis=0)
allowable_hp_idxs = np.where(no_hp_warnings == True)[0]
assert (
len(allowable_hp_idxs) > 0
), "all of your hp indices resulted in warnings so none is optimal"
# only work with the ones that you can actually use
kfold_metrics_outcome = kfold_metrics_outcome[:, allowable_hp_idxs]
# get extract value for selected criteria
# from kfold_metrics array for this particular outcome
def extract_crit(x):
return x[crit]
f = np.vectorize(extract_crit)
vals = f(kfold_metrics_outcome)
# average across folds
means = vals.mean(axis=0)
# get indices of optimal hyperparam(s) - shape: num_hps x num_optimal_hp_settings
idx_extreme = np.array(np.where(means == finder(means)))
# warn if hp hit the bounds of your grid search for any hp (there may be >1 hps)
for ix, this_hp in enumerate(idx_extreme):
n_hp_vals = means.shape[ix]
# if there's just one hp parameter, just throw one warning.
if n_hp_vals == 1:
warnings.warn(
"Only one value for hyperparameter number {0} supplied.".format(ix)
)
else:
# otherwise check if the optimal hp value is on the boundary of the search
if 0 in this_hp:
if didnt_record_hp_warnings[allowable_hp_idxs[ix]]:
warnings.warn(
"The optimal hyperparameter is the lowest of those supplied "
+ "(it was not checked for precision warnings). "
+ "hyperparameters supplied. "
+ "It is index {0} of the orignal hyperparamters passed in. ".format(
allowable_hp_idxs[ix]
)
)
else:
warnings.warn(
"The optimal hyperparameter is the lowest of the acceptable (i.e. no precision warnings) "
+ "hyperparameters supplied. "
+ "It is index {0} of the orignal hyperparamters passed in. ".format(
allowable_hp_idxs[ix]
)
+ "For reference, {0} of {1} ".format(
len(allowable_hp_idxs), len(no_hp_warnings)
)
+ "hyperparamters are considered acceptable; "
+ "their indices are {0}.".format(allowable_hp_idxs)
)
if (n_hp_vals - 1) in this_hp:
if didnt_record_hp_warnings[allowable_hp_idxs[ix]]:
warnings.warn(
"The optimal hyperparameter is the highest of those supplied "
+ "(it was not checked for precision warnings). "
+ "hyperparameters supplied. "
+ "It is index {0} of the orignal hyperparamters passed in. ".format(
allowable_hp_idxs[ix]
)
)
else:
warnings.warn(
"The optimal hyperparameter is the highest of the acceptable (i.e. no precision warnings) "
+ "hyperparameters supplied. "
+ "It is index {0} of the orignal hyperparamters passed in. ".format(
allowable_hp_idxs[ix]
)
+ "For reference, {0} of {1} ".format(
len(allowable_hp_idxs), len(no_hp_warnings)
)
+ "hyperparamters are considered acceptable; "
+ "their indices are {0}.".format(allowable_hp_idxs)
)
# warn if multiple optimal sets of hp
if idx_extreme.shape[1] > 1:
warnings.warn(
"Multiple optimal hyperparameters found for outcome {0}. Indices: {1}".format(
val, idx_extreme
)
)
# select first optimal set
return tuple(allowable_hp_idxs[idx_extreme[:, 0]])
def get_fold_results_by_hp_idx(kfold_metrics, idxs):
"""Slice model performance metrics array by
hyperparameter indices.
Args:
kfold_metrics (n_folds X n_outcomes X ... ndarray of dict):
Model performance array produced by kfold_solve
idxs (list of tuple): The indices of the hyperparameter values
swept over in cross-validation. The dimension of the list
indexes n_outcomes and the dimension of the tuples index ...
Returns:
n_folds X n_outcomes: Model performance for each fold using the
set of hyperparameters defined in idxs
"""
# initialize array of size n_folds X n_outcomes
res = np.empty(kfold_metrics.shape[:2], dtype=kfold_metrics.dtype)
for outcome_ix, i in enumerate(idxs):
# slice this outcome plus the optimal hp's for this outcome
# (first column is across folds, don't slice)
slicer = [slice(None), outcome_ix] + list(i)
res[:, outcome_ix] = kfold_metrics[tuple(slicer)]
return res
def _get_best_hps(hps, best_idxs):
best_hps = []
hp_names = [h[0] for h in hps]
n_outcomes = 1
for ox in range(n_outcomes):
this_best_hps = []
for hpx, hp in enumerate(best_idxs[ox]):
this_best_hps.append(hps[hpx][1][hp])
best_hps.append(this_best_hps)
hp_names = np.array(hp_names)
best_hps = np.array(best_hps)
return best_hps, hp_names
def interpret_kfold_results(
kfold_results, crits, minimize=False, save_weight_path=None, hps=None
):
"""Return the parsed results of the best performing model from
kfold_solve.
Args:
kfold_results (dict): As returned by kfold_solve()
crits (str or list of str): Metric criteria to base contractions
off of for each dimension. Must be str or list of length n_outcomes
minimize (bool or list of bool) : Whether to find minimal (True) or maximal
(False) value of each crit. (e.g. should be False for r2 and True for MSE)
save_weight_path (optional, str): Path where weights of model should be saved
(if not None). Should end in '.npz'. This file will have 3 arrays. 'weights'
will be n_folds X n_outcomes X n_features. 'hps' will be n_outcomes X n_hyperparams.
'hp_names' will be n_hyperparams and will have the hyperparemeter names in the same
order as the values appearing in 'hps'.
hps (list of 2-tuples): List of hyperparameters tested. Order of the tuples is
the same as the order they appear in kfold_results. e.g. [('lambda',[0,1,10])].
Required if save_weight_path is not None so that the optimal HP can be saved with
the weights.
Returns:
list of tuples: The indices of the best hyperparameters for each outcome. The dimension of
the list indexes outcomes, the dimension of the tuple indexes hyperparameters.
If more than one hyperparameter was swept over, this inner dimension will be >1.
In that case, the order is the same order that was used in the dimensions of
kfold_metrics arrays output by the solve function used to generate these results.
n_folds X n_outcomes 2darray of dict: Model performance array for optimal set of
hyperparameters for each outcome across folds
n_folds X n_outcomes 2darray of 1darray: Model predictions array for optimal set of
hyperparameters for each outcome across folds
"""
kfold_metrics = kfold_results["metrics_test"]
kfold_preds = kfold_results["y_pred_test"]
kfold_hp_warnings = kfold_results["hp_warning"]
if save_weight_path is not None:
kfold_models = kfold_results["models"]
kfold_shp = kfold_metrics.shape
num_folds = kfold_shp[0]
num_outputs = kfold_shp[1]
if isinstance(minimize, bool):
minimize = [minimize for i in range(num_outputs)]
if isinstance(crits, str):
crits = [crits for i in range(num_outputs)]
best_idxs = []
for j in range(num_outputs):
this_output_results_by_fold = kfold_metrics.take(indices=j, axis=1)
this_hp_warnings_by_fold = kfold_hp_warnings.take(indices=j, axis=1)
best_idxs_for_this_output = find_best_hp_idx(
this_output_results_by_fold,
this_hp_warnings_by_fold,
crits[j],
minimize=minimize[j],
val=j,
)
best_idxs.append(best_idxs_for_this_output)
# using the indices of the best hp values, return the model performance across all folds
# using those hp values
metrics_best_idx = get_fold_results_by_hp_idx(kfold_metrics, best_idxs)
# using the indices of the best hp values, return the model predictions across all folds
# using those hp values
y_pred_best_idx = get_fold_results_by_hp_idx(kfold_preds, best_idxs)
# using the indices of the best hp values, return the model weights across all folds
# using those hp values
if save_weight_path is not None:
best_hps, hp_names = _get_best_hps(hps, best_idxs)
models_best_idx = get_fold_results_by_hp_idx(kfold_models, best_idxs)
np.savez(
save_weight_path, weights=models_best_idx, hps=best_hps, hp_names=hp_names
)
# return the best idx along with the metrics and preds for all the folds corresponding to that index.
return best_idxs, metrics_best_idx, y_pred_best_idx
def interpret_single_results(
kfold_results, crits, minimize=False, save_weight_path=None, hps=None
):
"""Return the parsed results of the best performing model from
kfold_solve.
Args:
kfold_results (dict): As returned by kfold_solve()
crits (str or list of str): Metric criteria to base contractions
off of for each dimension. Must be str or list of length n_outcomes
minimize (bool or list of bool) : Whether to find minimal (True) or maximal
(False) value of each crit. (e.g. should be False for r2 and True for MSE)
save_weight_path (optional, str): Path where weights of model should be saved
(if not None). Should end in '.npz'. This file will have 3 arrays. 'weights'
will be n_features. 'hps' will be n_hyperparams.
'hp_names' will be n_hyperparams and will have the hyperparemeter names in the same
order as the values appearing in 'hps'.
hps (list of 2-tuples): List of hyperparameters tested. Order of the tuples is
the same as the order they appear in kfold_results. e.g. [('lambda',[0,1,10])].
Required if save_weight_path is not None so that the optimal HP can be saved with
the weights.
Returns:
list of tuples: The indices of the best hyperparameters for each outcome. The dimension of
the list indexes outcomes, the dimension of the tuple indexes hyperparameters.
If more than one hyperparameter was swept over, this inner dimension will be >1.
In that case, the order is the same order that was used in the dimensions of
kfold_metrics arrays output by the solve function used to generate these results.
n_folds X n_outcomes 2darray of dict: Model performance array for optimal set of
hyperparameters for each outcome across folds
n_folds X n_outcomes 2darray of 1darray: Model predictions array for optimal set of
hyperparameters for each outcome across folds
"""
kfold_metrics = kfold_results["metrics_test"]
kfold_preds = kfold_results["y_pred_test"]
kfold_hp_warnings = kfold_results["hp_warning"]
if save_weight_path is not None:
kfold_models = kfold_results["models"]
kfold_shp = kfold_metrics.shape
num_folds = kfold_shp[0]
num_outputs = kfold_shp[1]
assert num_folds == 1
if isinstance(minimize, bool):
minimize = [minimize for i in range(num_outputs)]
if isinstance(crits, str):
crits = [crits for i in range(num_outputs)]
# for each output (column of y), find the best hyperparameter
# values over all folds
best_idxs = []
for j in range(num_outputs):
this_output_results_by_fold = kfold_metrics.take(indices=j, axis=1)
this_hp_warnings_by_fold = kfold_hp_warnings.take(indices=j, axis=1)
best_idxs_for_this_output = find_best_hp_idx(
this_output_results_by_fold,
this_hp_warnings_by_fold,
crits[j],
minimize=minimize[j],
val=j,
)
best_idxs.append(best_idxs_for_this_output)
# using the indices of the best hp values, return the model performance across all folds
# using those hp values
metrics_best_idx = get_fold_results_by_hp_idx(kfold_metrics, best_idxs)
# using the indices of the best hp values, return the model predictions across all folds
# using those hp values
y_pred_best_idx = get_fold_results_by_hp_idx(kfold_preds, best_idxs)
# using the indices of the best hp values, return the model weights across all folds
# using those hp values
if save_weight_path is not None:
best_hps, hp_names = _get_best_hps(hps, best_idxs)
models_best_idx = get_fold_results_by_hp_idx(kfold_models, best_idxs)
np.savez(
save_weight_path,
weights=models_best_idx[0][0],
hps=best_hps[0],
hp_names=hp_names,
)
# return the best idx along with the metrics and preds for all the folds corresponding to that index.
# fold_idx = 0
# column_idx = 0
if num_outputs == 1:
return best_idxs[0], metrics_best_idx[0][0], y_pred_best_idx[0][0]
else:
return best_idxs, metrics_best_idx[0], y_pred_best_idx[0]