-
Notifications
You must be signed in to change notification settings - Fork 0
/
eval_2_aprnn.py
161 lines (127 loc) · 4.86 KB
/
eval_2_aprnn.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
import warnings; warnings.filterwarnings("ignore")
from experiments import imagenet
from experiments.base import *
import sytorch as st
import argparse
from timeit import default_timer as timer
""" Extend beyond this experiment.
==============================
1. Different buggy network.
- Change `net` to load a different buggy network.
Specifically, the following command loads a torch DNN `torch_dnn_object`:
```
network = st.nn.from_torch(torch_dnn_object)
```
And the following command loads an ONNX DNN `onnx_dnn_path` from file:
```
network = st.nn.from_file(onnx_dnn_path)
```
2. Different dataset.
- Change the `images` and `labels` tensors to load expected buggy inputs
and the correct labels, in the same way as training a PyTorch DNN.
3. Different repair parameters:
- Change the `k` and `rows` parameters.
- Change `lb` and `ub`.
"""
parser = argparse.ArgumentParser(description='')
parser.add_argument('--net', type=str, dest='net', action='store', required=True,
help='resnet152, vgg19')
parser.add_argument('--device', type=str, dest='device', action='store', default='cpu',
help='device to use, e.g., cuda, cuda:0, cpu. (default=cpu).')
parser.add_argument('--use_artifact', dest='use_artifact', action='store_true',
help='use authors\' repaired DNN.')
args = parser.parse_args()
device = get_device(args.device)
dtype = st.float64
st.set_all_seed(0)
if args.net == 'resnet152':
npoints = 50
k = st.as_slice[-4,-1,0,0,-2]
rows = st.as_slice[0:200]
net = imagenet.models.resnet152(pretrained=True)\
.to(dtype=dtype, device=device)\
.eval()
elif args.net == 'vgg19':
npoints = 50
k = st.as_slice[-1, 3]
rows = st.as_slice[400:800]
net = imagenet.models.vgg19(pretrained=True)\
.to(dtype=dtype, device=device)\
.eval()
if not args.use_artifact:
repair_dataset = imagenet.datasets.ImageNet_A()\
.to(dtype=dtype, device=device)\
.misclassified(net, num=npoints, seed=None)
images, labels = repair_dataset.load(npoints)
with st.no_grad():
reference_output = net(images).cpu()
""" Repair Phase. """
start_time = timer()
""" Create a new solver. """
solver = st.LightningSolver()\
.verbose_()
""" Attach `net` to the solver and turn the repair (symbolic) mode on. """
net = net.to(solver)\
.repair()\
.requires_symbolic_(False)
net[k].requires_symbolic_(
lb = -10.,
ub = 10.,
rows= rows,
cols= None,
bias= False,
seed= 0
)
""" Compute the symbolic outputs of shape `(npoints, 10)`. Note that activation
constraints are added implicitly to the attached solver.
"""
symbolic_outputs = net(images)
""" Add the classification constraints. """
solver.add_constraints(symbolic_outputs.argmax(axis=-1) == labels)
""" Collect the (symbolic) deltas of all symbolic parameters, concatenated as an
1d-array.
"""
param_deltas = net.parameter_deltas(concat=True)
solver2 = solver.gurobi()
solver2.solver.Params.Method = 2
solver.solver = solver2
param_deltas = param_deltas.to(solver2)
print(param_deltas.shape)
output_deltas = (symbolic_outputs.to(solver2) - reference_output).alias()
print(output_deltas.shape)
assert solver2.verbose_().solve(
minimize = (
param_deltas.norm_ub(order="linf") +
param_deltas.norm_ub(order="l1_normalized") +
output_deltas.reshape(-1).norm_ub(order='linf') +
output_deltas.reshape(-1).norm_ub(order='l1_normalized')
),
)
time = timer() - start_time
net = net.update_().repair(False)
result_path = (get_results_root() / 'eval_2' / f'aprnn_{args.net}').as_posix()
else:
net[k].load(get_artifact_root() / 'eval_2' / f'aprnn_{args.net}_diff.pth')
time = None
result_path = (get_results_root() / 'eval_2' / f'artifact_aprnn_{args.net}').as_posix()
# See: https://pytorch.org/vision/stable/models.html#table-of-all-available-classification-weights
og_acc1, og_acc5 = {
'resnet152': (0.78312, 0.94046),
'vgg19' : (0.72376, 0.90876)
}[args.net]
with st.no_symbolic():
net = net.to(dtype=st.float32)
testset = imagenet.datasets.ImageNet(split='val').to(dtype=st.float32, device=device)
acc1, acc5 = testset.accuracy(net, topk=(1, 5))
result = {
'APRNN': {
(args.net, 'D@top-1'): float(og_acc1 - acc1),
(args.net, 'D@top-5'): float(og_acc5 - acc5),
(args.net, 'T'): "N/A" if time is None else f'{int(time)}s',
}
}
np.save(result_path+".npy", result, allow_pickle=True)
print_msg_box(
f"Experiment 2 using APRNN for {args.net} SUCCEED.\n"
f"Saved result to {result_path}.npy"
)