Skip to content

Commit

Permalink
fix test
Browse files Browse the repository at this point in the history
  • Loading branch information
@chenyangkang
chenyangkang committed Jan 23, 2024
1 parent 7c26050 commit 455ff16
Show file tree
Hide file tree
Showing 4 changed files with 26 additions and 39 deletions.
24 changes: 4 additions & 20 deletions stemflow/model/AdaSTEM.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -953,7 +953,10 @@ def assign_feature_importances_by_points(

# assign input spatio-temporal points to stixels

if not njobs > 1:
if njobs > 1:
raise NotImplementedError("Multi-threading is not implemented yet")

else:
# Single processing
round_res_list = []
iter_func_ = (
Expand All @@ -973,25 +976,6 @@ def assign_feature_importances_by_points(
)
round_res_list.append(res_list)

else:
# multi-processing
with Pool(njobs) as p:
plain_args_iterator = zip(
repeat(self.ensemble_df),
list(self.ensemble_df.ensemble_index.unique()),
repeat(Sample_ST_df),
repeat(self.Temporal1),
repeat(self.Spatio1),
repeat(self.Spatio2),
repeat(self.feature_importances_),
)
if verbosity > 0:
args_iterator = tqdm(plain_args_iterator, total=len(list(self.ensemble_df.ensemble_index.unique())))
else:
args_iterator = plain_args_iterator

round_res_list = p.starmap(assign_points_to_one_ensemble, args_iterator)

round_res_df = pd.concat(round_res_list, axis=0)
ensemble_available_count = round_res_df.groupby("sample_index").count().iloc[:, 0]

Expand Down
3 changes: 2 additions & 1 deletion stemflow/model/static_func_AdaSTEM.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,8 @@ def get_model_by_name(model_dict: dict, ensemble: str, grid_index: str) -> Union
model = model_dict[f"{ensemble}_{grid_index}_model"]
return model
except Exception as e:
warnings.warn(f"Cannot find model: {e}")
if not isinstance(e, KeyError):
warnings.warn(f"Cannot find model: {e}")
return None


Expand Down
16 changes: 9 additions & 7 deletions tests/make_models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
min_req = 1


def make_STEMClassifier(fold_, min_req, ensemble_models_disk_saver, ensemble_models_disk_saving_dir):
def make_STEMClassifier(fold_=2, min_req=1, ensemble_models_disk_saver=False, ensemble_models_disk_saving_dir=""):
model = STEMClassifier(
base_model=XGBClassifier(tree_method="hist", random_state=42, verbosity=0, n_jobs=1),
save_gridding_plot=True,
Expand All @@ -49,7 +49,7 @@ def make_STEMClassifier(fold_, min_req, ensemble_models_disk_saver, ensemble_mod
return model


def make_STEMRegressor(fold_, min_req, ensemble_models_disk_saver, ensemble_models_disk_saving_dir):
def make_STEMRegressor(fold_=2, min_req=1, ensemble_models_disk_saver=False, ensemble_models_disk_saving_dir=""):
model = STEMRegressor(
base_model=Hurdle(
classifier=XGBClassifier(tree_method="hist", random_state=42, verbosity=0, n_jobs=1),
Expand Down Expand Up @@ -78,8 +78,8 @@ def make_STEMRegressor(fold_, min_req, ensemble_models_disk_saver, ensemble_mode
return model


def make_AdaSTEMClassifier(fold_, min_req, ensemble_models_disk_saver, ensemble_models_disk_saving_dir):
model = make_AdaSTEMClassifier(
def make_AdaSTEMClassifier(fold_=2, min_req=1, ensemble_models_disk_saver=False, ensemble_models_disk_saving_dir=""):
model = AdaSTEMClassifier(
base_model=XGBClassifier(tree_method="hist", random_state=42, verbosity=0, n_jobs=1),
save_gridding_plot=True,
ensemble_fold=fold_,
Expand All @@ -104,7 +104,7 @@ def make_AdaSTEMClassifier(fold_, min_req, ensemble_models_disk_saver, ensemble_
return model


def make_AdaSTEMRegressor(fold_, min_req, ensemble_models_disk_saver, ensemble_models_disk_saving_dir):
def make_AdaSTEMRegressor(fold_=2, min_req=1, ensemble_models_disk_saver=False, ensemble_models_disk_saving_dir=""):
model = AdaSTEMRegressor(
base_model=Hurdle(
classifier=XGBClassifier(tree_method="hist", random_state=42, verbosity=0, n_jobs=1),
Expand Down Expand Up @@ -133,7 +133,9 @@ def make_AdaSTEMRegressor(fold_, min_req, ensemble_models_disk_saver, ensemble_m
return model


def make_SphereAdaSTEMRegressor(fold_, min_req, ensemble_models_disk_saver, ensemble_models_disk_saving_dir):
def make_SphereAdaSTEMRegressor(
fold_=2, min_req=1, ensemble_models_disk_saver=False, ensemble_models_disk_saving_dir=""
):
model = SphereAdaSTEMRegressor(
base_model=Hurdle(
classifier=XGBClassifier(tree_method="hist", random_state=42, verbosity=0, n_jobs=1),
Expand Down Expand Up @@ -161,7 +163,7 @@ def make_SphereAdaSTEMRegressor(fold_, min_req, ensemble_models_disk_saver, ense
return model


def make_SphereAdaClassifier(fold_, min_req, ensemble_models_disk_saver, ensemble_models_disk_saving_dir):
def make_SphereAdaClassifier(fold_=2, min_req=1, ensemble_models_disk_saver=False, ensemble_models_disk_saving_dir=""):
model = SphereAdaSTEMClassifier(
base_model=XGBClassifier(tree_method="hist", random_state=42, verbosity=0, n_jobs=1),
save_gridding_plot=True,
Expand Down
22 changes: 11 additions & 11 deletions tests/test_model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def test_STEMClassifier():

pred = model.predict(X_test)
assert len(pred) == len(X_test)
assert np.isnan(pred) / len(pred) <= 0.3
assert np.sum(np.isnan(pred)) / len(pred) <= 0.3

pred_df = pd.DataFrame(
{"y_true": y_test.flatten(), "y_pred": np.where(pred.flatten() < 0, 0, pred.flatten())}
Expand All @@ -47,7 +47,7 @@ def test_STEMClassifier():

importances_by_points = model.assign_feature_importances_by_points(verbosity=0, njobs=1)
assert importances_by_points.shape[0] > 0
assert importances_by_points.shape[1] == len(x_names) + 1
assert importances_by_points.shape[1] == len(x_names) + 3


def test_STEMRegressor():
Expand All @@ -60,7 +60,7 @@ def test_STEMRegressor():

pred = model.predict(X_test)
assert len(pred) == len(X_test)
assert np.isnan(pred) / len(pred) <= 0.3
assert np.sum(np.isnan(pred)) / len(pred) <= 0.3

pred_df = pd.DataFrame(
{"y_true": y_test.flatten(), "y_pred": np.where(pred.flatten() < 0, 0, pred.flatten())}
Expand All @@ -77,7 +77,7 @@ def test_STEMRegressor():

importances_by_points = model.assign_feature_importances_by_points(verbosity=0, njobs=1)
assert importances_by_points.shape[0] > 0
assert importances_by_points.shape[1] == len(x_names) + 1
assert importances_by_points.shape[1] == len(x_names) + 3


def test_AdaSTEMClassifier():
Expand All @@ -90,7 +90,7 @@ def test_AdaSTEMClassifier():

pred = model.predict(X_test)
assert len(pred) == len(X_test)
assert np.isnan(pred) / len(pred) <= 0.3
assert np.sum(np.isnan(pred)) / len(pred) <= 0.3

pred_df = pd.DataFrame(
{"y_true": y_test.flatten(), "y_pred": np.where(pred.flatten() < 0, 0, pred.flatten())}
Expand All @@ -107,7 +107,7 @@ def test_AdaSTEMClassifier():

importances_by_points = model.assign_feature_importances_by_points(verbosity=0, njobs=1)
assert importances_by_points.shape[0] > 0
assert importances_by_points.shape[1] == len(x_names) + 1
assert importances_by_points.shape[1] == len(x_names) + 3


def test_AdaSTEMRegressor():
Expand Down Expand Up @@ -137,7 +137,7 @@ def test_AdaSTEMRegressor():

importances_by_points = model.assign_feature_importances_by_points(verbosity=0, njobs=1)
assert importances_by_points.shape[0] > 0
assert importances_by_points.shape[1] == len(x_names) + 1
assert importances_by_points.shape[1] == len(x_names) + 3


def test_SphereAdaClassifier():
Expand All @@ -150,7 +150,7 @@ def test_SphereAdaClassifier():

pred = model.predict(X_test)
assert len(pred) == len(X_test)
assert np.isnan(pred) / len(pred) <= 0.3
assert np.sum(np.isnan(pred)) / len(pred) <= 0.3

pred_df = pd.DataFrame(
{"y_true": y_test.flatten(), "y_pred": np.where(pred.flatten() < 0, 0, pred.flatten())}
Expand All @@ -167,7 +167,7 @@ def test_SphereAdaClassifier():

importances_by_points = model.assign_feature_importances_by_points(verbosity=0, njobs=1)
assert importances_by_points.shape[0] > 0
assert importances_by_points.shape[1] == len(x_names) + 1
assert importances_by_points.shape[1] == len(x_names) + 3


def test_SphereAdaSTEMRegressor():
Expand All @@ -180,7 +180,7 @@ def test_SphereAdaSTEMRegressor():

pred = model.predict(X_test)
assert len(pred) == len(X_test)
assert np.isnan(pred) / len(pred) <= 0.3
assert np.sum(np.isnan(pred)) / len(pred) <= 0.3

pred_df = pd.DataFrame(
{"y_true": y_test.flatten(), "y_pred": np.where(pred.flatten() < 0, 0, pred.flatten())}
Expand All @@ -197,4 +197,4 @@ def test_SphereAdaSTEMRegressor():

importances_by_points = model.assign_feature_importances_by_points(verbosity=0, njobs=1)
assert importances_by_points.shape[0] > 0
assert importances_by_points.shape[1] == len(x_names) + 1
assert importances_by_points.shape[1] == len(x_names) + 3

0 comments on commit 455ff16

Please sign in to comment.