Merge pull request #14 from iamgroot42/michael/gpt_em

Analysis Update for GPT-paraphrased Edited members

data/gpt_generated_paraphrases/analyze_paraphrase.py

@@ -40,7 +40,7 @@ def write(outputs, path):
         for i, p in enumerate(pm['paraphrases']):
             em_version['gpt'][str(i)].append(p)
 
-    assert len(em_version['gpt']['0']) == 50
+    assert len(em_version['gpt']['0']) == 1000
 
     with open(os.path.join(output_dir, f"em_version_{os.path.basename(paraphrase_path)}"), 'w') as out:
         json.dump(em_version, out)

notebooks/new_mi_experiment.py

@@ -326,7 +326,7 @@ def edit(x, n: int):
     if config.load_from_cache and not config.dump_cache:
         # For NE neighbors, 30% is masked
         with open(
-            f"/mmfs1/gscratch/h2lab/micdun/mimir/data/gpt_generated_paraphrases/out/em_version_{config.specific_source}_paraphrases_1000_samples_5_trials.jsonl", #f"edit_distance_members/ne/{config.specific_source}.json",
+            f"/mmfs1/gscratch/h2lab/micdun/mimir/data/gpt_generated_paraphrases/out/analysis/gpt4_{config.specific_source}/em_version_{config.specific_source}_paraphrases_1000_samples_5_trials.jsonl", #f"edit_distance_members/ne/{config.specific_source}.json",
             "r",
         ) as f:
             other_members_data = json.load(f)
@@ -372,5 +372,5 @@ def edit(x, n: int):
                 score_dict[attack][n][i] = other_blackbox_predictions[attack]
 
     pbar.close()
-    with open(f"edit_distance_members/scores/gpt_paraphrase_results_{config.specific_source}.json", "w") as f:
+    with open(f"edit_distance_members/scores/gpt4/gpt_paraphrase_results_{config.specific_source}.json", "w") as f:
         json.dump(score_dict, f, indent=4)

scripts/run_mia_new_mi.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
 version=new_mi
 ngram=13
-for subset in "wikipedia_(en)" #"github"
+for subset in "arxiv" "hackernews" #"wikipedia_(en)" #"github"
     do
-        python new_mi_experiment.py \
+        python notebooks/new_mi_experiment.py \
             --experiment_name $version \
             --config configs/new_mi.json \
-            --base_model "EleutherAI/pythia-1.4b-deduped" \
+            --base_model "EleutherAI/pythia-12b-deduped" \
             --revision step99000 \
             --specific_source ${subset} #_ngram_${ngram}_\<0.8_truncated
     done