Skip to content

Commit

Permalink
fixed the emb_sim_scorer tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@chandralegend
chandralegend committed Mar 24, 2024
1 parent 0e9fd62 commit a527d2a
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 76 deletions.
2 changes: 1 addition & 1 deletion src/components/emb_sim_scorer.impl.jac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ import:py from nltk, ngrams;
with open(os.path.abspath(".human_eval_config/responses.json"),"r") as fp{responses_dict = json.load(fp);}
with open(os.path.abspath(".human_eval_config/models_responses.json"),"r") as fp{models_responses_dict = json.load(fp);}
config_name = f"{st.session_state['anchor_model']}_{st.session_state['embedder']}_{st.session_state['scorer']}";
path_string = os.path.join("sim_results", config_name);
path_string = os.path.join("results", config_name);
basedir = Path(os.path.abspath(path_string));
heatmap_placeholder = st.empty();
if not basedir.exists() or not any(basedir.glob('*.json')) {
Expand Down
87 changes: 12 additions & 75 deletions src/tests/test_emb_sim_eval.jac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,10 @@ import:py shutil;
import:jac from utils, get_item_by_label;
import:py from pathlib, Path;

glob app = AppTest.from_file("app.py").run(timeout=20);


test app_running {
:g: app;
app = AppTest.from_file("app.py").run(timeout=20);
assert not app.exception;
human_eval = Path(os.path.abspath(".human_eval_config"));
if human_eval.exists(){shutil.rmtree(human_eval);}
}


Expand All @@ -23,93 +19,34 @@ test test_model_selection {
shutil.unpack_archive(
os.path.join(os.path.dirname(__file__), "fixtures", "data.zip"), "."
);
app = AppTest.from_file("app.py").run(timeout=20);
app.session_state.admin_privileges = True;
app.run();

sim_scorer_tab = get_item_by_label(app, "tab", "Similarity Scorer");
selectbox = get_item_by_label(app, "selectbox", "Select Anchor Model");

selectbox.set_value(selectbox.options[0]).run();
assert selectbox.value == "llama2:7b-chat";

selectbox.set_value(selectbox.options[1]).run();
assert selectbox.value == "mistral:7b-instruct";

human_eval = Path(os.path.abspath(".human_eval_config"));
if human_eval.exists(){shutil.rmtree(human_eval);}
shutil.rmtree("data");

}


test test_embedder_type_selection {
shutil.unpack_archive(
os.path.join(os.path.dirname(__file__), "fixtures", "config.zip"), "."
);
shutil.unpack_archive(
os.path.join(os.path.dirname(__file__), "fixtures", "data.zip"), "."
);
app = AppTest.from_file("app.py").run(timeout=20);
app.session_state.admin_privileges = True;
app.run();
sim_scorer_tab = get_item_by_label(app, "tab", "Similarity Scorer");
selectbox = get_item_by_label(app, "selectbox", "Select Type of Embedder");
selectbox.set_value(selectbox.options[0]).run();
assert selectbox.value == "USE";
selectbox.set_value(selectbox.options[1]).run();
assert selectbox.value == "USE_QA";
selectbox.set_value(selectbox.options[2]).run();
assert selectbox.value == "SBERT";
anchor_selector = get_item_by_label(sim_scorer_tab, "selectbox", "Select Anchor Model");
embedder_selector = get_item_by_label(sim_scorer_tab, "selectbox", "Select Type of Embedder");
scorer_selector = get_item_by_label(sim_scorer_tab, "selectbox", "Select Scorer");

human_eval = Path(os.path.abspath(".human_eval_config"));
if human_eval.exists(){shutil.rmtree(human_eval);}
shutil.rmtree("data");
assert anchor_selector.options == ['llama2:7b-chat', 'mistral:7b-instruct'];
assert embedder_selector.options == ['USE', 'USE_QA', 'SBERT'];
assert scorer_selector.options == ['cos_sim', 'sem_bleu'];
}

test test_scorer_selection {
test test_embedding_score_calculation {
shutil.unpack_archive(
os.path.join(os.path.dirname(__file__), "fixtures", "config.zip"), "."
);
shutil.unpack_archive(
os.path.join(os.path.dirname(__file__), "fixtures", "data.zip"), "."
);
app = AppTest.from_file("app.py").run(timeout=20);
app.session_state.admin_privileges = True;
app.run();

sim_scorer_tab = get_item_by_label(app, "tab", "Similarity Scorer");
selectbox = get_item_by_label(app, "selectbox", "Select Scorer");

selectbox.set_value(selectbox.options[0]).run();
assert selectbox.value == "cos_sim";
get_item_by_label(sim_scorer_tab, "button", "Calculate Embedding Scores").set_value(True).run(timeout=200);

selectbox.set_value(selectbox.options[1]).run();
assert selectbox.value == "sem_bleu";
assert len(os.listdir(os.path.join("results", "llama2:7b-chat_SBERT_cos_sim"))) > 0;

human_eval = Path(os.path.abspath(".human_eval_config"));
if human_eval.exists(){shutil.rmtree(human_eval);}
shutil.rmtree("results");
shutil.rmtree(".human_eval_config");
shutil.rmtree("data");
}

test test_embedding_score_calculation {
shutil.unpack_archive(
os.path.join(os.path.dirname(__file__), "fixtures", "config.zip"), "."
);
shutil.unpack_archive(
os.path.join(os.path.dirname(__file__), "fixtures", "data.zip"), "."
);
app = AppTest.from_file("app.py").run(timeout=20);
app.session_state.admin_privileges = True;
app.run();

sim_scorer_tab = get_item_by_label(app, "tab", "Similarity Scorer");
calculate_embedding_score_button = get_item_by_label(app, "button", "Calculate Embedding Scores");
calculate_embedding_score_button.set_value(True).run();
spec = json.loads(app.get("plotly_chart")[0].proto.figure.spec);
assert len(spec["data"]) > 0;

human_eval = Path(os.path.abspath(".human_eval_config"));
if human_eval.exists(){shutil.rmtree(human_eval);}
# shutil.rmtree("data");
}

0 comments on commit a527d2a

Please sign in to comment.