-
Notifications
You must be signed in to change notification settings - Fork 17
/
evaluation.py
105 lines (98 loc) · 2.78 KB
/
evaluation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
import argparse
import os
def main():
args = ArgumentParser()
eval(args)
def eval(args):
# load all dev ids
ids = load_ids_dev(args.ids_file)
# load all anns for dev
anns = load_anns_dev(args.anns_file,ids)
# load predictions
preds = load_preds(args.dev_file,ids)
# evaluate and print errors
tps = 0
fps = 0
fns = 0
out_file = open(os.path.join(args.out_file), "w")
for id in ids:
# print(id,preds[id])
anns_id = anns[id]
if id in preds:
preds_id = preds[id]
else:
preds_id = []
# print(id,anns_id,preds_id)
for pred in preds_id:
if pred in anns_id:
tps += 1
out_file.write('TP\t'+id+'\t'+pred+'\n')
else:
fps += 1
out_file.write('FP\t'+id+'\t'+pred+'\n')
for ann in anns_id:
if ann not in preds_id:
fns += 1
out_file.write('FN\t'+id+'\t'+ann+'\n')
out_file.write('TPs='+str(tps)+'\n')
print('TPs='+str(tps))
out_file.write('FPs='+str(fps)+'\n')
print('FPs='+str(fps))
out_file.write('FNs='+str(fns)+'\n')
print('FNs='+str(fns))
precision = tps/(tps+fps)
out_file.write('precision='+str(precision)+'\n')
print('precision='+str(precision))
recall = tps/(tps+fns)
out_file.write('recall='+str(recall)+'\n')
print('recall='+str(recall))
fscore = 2*precision*recall/(precision+recall)
out_file.write('fscore='+str(fscore)+'\n')
print('fscore='+str(fscore))
def load_preds(dev_file,ids):
preds = {}
f = open(dev_file, "r")
for line in f:
# print(line,'**')
if '\t' in line.strip():
id,str_preds = line.strip().split('\t')
if id in ids:
if str_preds is not None:
preds[id] = str_preds.split('|')
else:
preds[id] = []
else:
id = line.strip()
preds[id] = []
print('Number of documents with predictions:',len(preds))
return preds
def load_anns_dev(anns_file,ids):
anns = {}
f = open(anns_file, "r")
for line in f:
id,str_anns = line.strip().split('\t')
if id in ids:
if str_anns is not None:
anns[id] = str_anns.split('|')
else:
anns[id] = []
# add dev doc without annotations
for id in ids:
if id not in anns.keys():
anns[id] = []
# print(len(anns))
return anns
def load_ids_dev(ids_file):
f = open(ids_file, "r")
ids = [line.strip() for line in f ]
print('Number of dev documents:',len(ids))
return ids
def ArgumentParser():
parser = argparse.ArgumentParser()
parser.add_argument('--dev_file', type=str, default='dev.txt', help='predictions for the documents of the development set')
parser.add_argument('--anns_file', type=str, default='anns_train_dev.txt', help='annotations from documents of the development set')
parser.add_argument('--ids_file', type=str, default='ids_development.txt', help='list of ids of the development set')
parser.add_argument('--out_file', type=str, default='output.txt', help='output file')
return parser.parse_args()
if __name__ == '__main__':
main()