-
Notifications
You must be signed in to change notification settings - Fork 17
/
evaluation.py
executable file
·98 lines (70 loc) · 2.55 KB
/
evaluation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
import numpy as np
def _compute_apk(targets, predictions, k):
if len(predictions) > k:
predictions = predictions[:k]
score = 0.0
num_hits = 0.0
for i, p in enumerate(predictions):
if p in targets and p not in predictions[:i]:
num_hits += 1.0
score += num_hits / (i + 1.0)
if not list(targets):
return 0.0
return score / min(len(targets), k)
def _compute_precision_recall(targets, predictions, k):
pred = predictions[:k]
num_hit = len(set(pred).intersection(set(targets)))
precision = float(num_hit) / len(pred)
recall = float(num_hit) / len(targets)
return precision, recall
def evaluate_ranking(model, test, train=None, k=10):
"""
Compute Precision@k, Recall@k scores and average precision (AP).
One score is given for every user with interactions in the test
set, representing the AP, Precision@k and Recall@k of all their
test items.
Parameters
----------
model: fitted instance of a recommender model
The model to evaluate.
test: :class:`spotlight.interactions.Interactions`
Test interactions.
train: :class:`spotlight.interactions.Interactions`, optional
Train interactions. If supplied, rated items in
interactions will be excluded.
k: int or array of int,
The maximum number of predicted items
"""
test = test.tocsr()
if train is not None:
train = train.tocsr()
if not isinstance(k, list):
ks = [k]
else:
ks = k
precisions = [list() for _ in range(len(ks))]
recalls = [list() for _ in range(len(ks))]
apks = list()
for user_id, row in enumerate(test):
if not len(row.indices):
continue
predictions = -model.predict(user_id)
predictions = predictions.argsort()
if train is not None:
rated = set(train[user_id].indices)
else:
rated = []
predictions = [p for p in predictions if p not in rated]
targets = row.indices
for i, _k in enumerate(ks):
precision, recall = _compute_precision_recall(targets, predictions, _k)
precisions[i].append(precision)
recalls[i].append(recall)
apks.append(_compute_apk(targets, predictions, k=np.inf))
precisions = [np.array(i) for i in precisions]
recalls = [np.array(i) for i in recalls]
if not isinstance(k, list):
precisions = precisions[0]
recalls = recalls[0]
mean_aps = np.mean(apks)
return precisions, recalls, mean_aps