From 3e6e03b29683454e30f0d8b7d4ee83db5b713447 Mon Sep 17 00:00:00 2001 From: Farouk Mokhtar Date: Thu, 12 Jan 2023 10:14:52 +0100 Subject: [PATCH] ssl-based mlpf first iteration (#158) * add ssl notebook * add ssl notebook * Squashed commit of the following: commit a4d5077e9b2bcb7c5d312d79a5c3c961a4fdf016 Author: Farouk Date: Thu Nov 3 15:26:54 2022 +0100 up commit 9012e78b991625248b81a4ceb43af8d6c781db42 Author: Farouk Date: Thu Nov 3 15:04:17 2022 +0100 up commit 29151308370dace58d26930685c34ce154871019 Author: Farouk Date: Thu Nov 3 14:29:01 2022 +0100 add ssl-VICreg notebook * up * VICReg apples to apples * major cleanup and reorganization * ude full dataset for VICReg * update readme pyg * add first iteration of ssl * before merge * cleanup and ssl organization * add flake8 * revert changes * up --- .gitignore | 3 + mlpf/data_clic/postprocessing.py | 8 + mlpf/data_cms/postprocessing2.py | 7 +- mlpf/lrp/__init__.py | 3 - mlpf/lrp/lrp_mlpf.py | 280 -- mlpf/lrp/make_rmaps.py | 190 - mlpf/lrp/model.py | 222 - mlpf/lrp_mlpf_pipeline.py | 129 - mlpf/pyg/PFGraphDataset.py | 69 +- mlpf/pyg/README.md | 16 +- mlpf/pyg/__init__.py | 35 - mlpf/pyg/args.py | 153 +- mlpf/pyg/{ => cms}/cms_plots.py | 221 +- mlpf/pyg/{ => cms}/cms_utils.py | 29 +- mlpf/pyg/{ => cms}/get_data_cms.sh | 18 +- mlpf/pyg/{ => delphes}/delphes_plots.py | 494 +- mlpf/pyg/delphes/delphes_utils.py | 53 + mlpf/pyg/{ => delphes}/get_data_delphes.sh | 22 +- mlpf/pyg/delphes_utils.py | 657 --- mlpf/pyg/evaluate.py | 193 +- mlpf/pyg/model.py | 123 +- mlpf/pyg/training.py | 11 +- mlpf/pyg/utils.py | 82 +- mlpf/pyg_pipeline.py | 68 +- mlpf/pyg_ssl/PFGraphDataset.py | 164 + mlpf/pyg_ssl/README.md | 28 + mlpf/pyg_ssl/VICReg.py | 88 + mlpf/pyg_ssl/__init__.py | 0 mlpf/pyg_ssl/args.py | 51 + mlpf/pyg_ssl/clic/get_data_clic.sh | 30 + mlpf/pyg_ssl/environment.yml | 20 + mlpf/pyg_ssl/evaluate.py | 92 + mlpf/pyg_ssl/mlpf.py | 55 + mlpf/pyg_ssl/training_VICReg.py | 256 ++ mlpf/pyg_ssl/training_mlpf.py | 224 + mlpf/pyg_ssl/utils.py | 164 + mlpf/ssl_pipeline.py | 223 + notebooks/ssl-VICreg.ipynb | 4800 ++++++++++++++++++++ 38 files changed, 6963 insertions(+), 2318 deletions(-) delete mode 100644 mlpf/lrp/__init__.py delete mode 100644 mlpf/lrp/lrp_mlpf.py delete mode 100644 mlpf/lrp/make_rmaps.py delete mode 100644 mlpf/lrp/model.py delete mode 100644 mlpf/lrp_mlpf_pipeline.py rename mlpf/pyg/{ => cms}/cms_plots.py (78%) rename mlpf/pyg/{ => cms}/cms_utils.py (88%) rename mlpf/pyg/{ => cms}/get_data_cms.sh (71%) rename mlpf/pyg/{ => delphes}/delphes_plots.py (50%) create mode 100644 mlpf/pyg/delphes/delphes_utils.py rename mlpf/pyg/{ => delphes}/get_data_delphes.sh (63%) delete mode 100644 mlpf/pyg/delphes_utils.py create mode 100644 mlpf/pyg_ssl/PFGraphDataset.py create mode 100644 mlpf/pyg_ssl/README.md create mode 100644 mlpf/pyg_ssl/VICReg.py create mode 100644 mlpf/pyg_ssl/__init__.py create mode 100644 mlpf/pyg_ssl/args.py create mode 100755 mlpf/pyg_ssl/clic/get_data_clic.sh create mode 100644 mlpf/pyg_ssl/environment.yml create mode 100644 mlpf/pyg_ssl/evaluate.py create mode 100644 mlpf/pyg_ssl/mlpf.py create mode 100644 mlpf/pyg_ssl/training_VICReg.py create mode 100644 mlpf/pyg_ssl/training_mlpf.py create mode 100644 mlpf/pyg_ssl/utils.py create mode 100644 mlpf/ssl_pipeline.py create mode 100644 notebooks/ssl-VICreg.ipynb diff --git a/.gitignore b/.gitignore index 7fa7529da..5c1cfd5b2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,8 @@ nohup.out *.pkl *.pkl.bz2 +*.parquet slurm-*.out + +.vscode diff --git a/mlpf/data_clic/postprocessing.py b/mlpf/data_clic/postprocessing.py index a37403017..63cc74c88 100644 --- a/mlpf/data_clic/postprocessing.py +++ b/mlpf/data_clic/postprocessing.py @@ -189,6 +189,13 @@ def flatten_event(df_tr, df_cl, df_gen, df_pfs, pairs): def prepare_data_clic(fn): + """ + Processing function that takes as input a raw parquet file and processes it. + + Returns + a list of events, each containing three arrays [Xs, ygen, ycand]. + + """ data = awkward.from_parquet(fn) @@ -308,6 +315,7 @@ def prepare_data_clic(fn): # print(df_gen.loc[gp]) Xs, ys_gen, ys_cand = flatten_event(df_tr, df_cl, df_gen, df_pfs, pairs) + ret.append([Xs, ys_gen, ys_cand]) return ret diff --git a/mlpf/data_cms/postprocessing2.py b/mlpf/data_cms/postprocessing2.py index faf4a312d..430ed7c97 100644 --- a/mlpf/data_cms/postprocessing2.py +++ b/mlpf/data_cms/postprocessing2.py @@ -810,12 +810,7 @@ def parse_args(): import argparse parser = argparse.ArgumentParser() - parser.add_argument( - "--input", - type=str, - help="Input file from PFAnalysis", - required=True, - ) + parser.add_argument("--input", type=str, help="Input file from PFAnalysis", required=True) parser.add_argument("--outpath", type=str, default="raw", help="output path") parser.add_argument( "--save-full-graph", diff --git a/mlpf/lrp/__init__.py b/mlpf/lrp/__init__.py deleted file mode 100644 index 1b451cc4e..000000000 --- a/mlpf/lrp/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from lrp.lrp_mlpf import LRP_MLPF # noqa: F401 -from lrp.make_rmaps import make_Rmaps # noqa: F401 -from lrp.model import MLPF # noqa: F401 diff --git a/mlpf/lrp/lrp_mlpf.py b/mlpf/lrp/lrp_mlpf.py deleted file mode 100644 index b2ba8a0d7..000000000 --- a/mlpf/lrp/lrp_mlpf.py +++ /dev/null @@ -1,280 +0,0 @@ -import torch - - -class LRP_MLPF: - - """ - A class that introduces useful functionality to perform layerwise-relevance propagation (LRP) on MLPF. - This class is meant to work for any model with any number of GravNetConv layers. - The main trick is to realize that the ".lin_s" layers in GravNetConv are irrelevant for explanations so shall be skipped. - The hack, however, is to substitute them precisely with the message_passing step. - - Differences from standard LRP - a. Rscores become tensors/graphs of input features per output neuron instead of vectors - b. accomodates message passing steps by using the adjacency matrix as the weight matrix in standard LRP, - and redistributing Rscores over the other dimension (over nodes instead of features) - """ - - def __init__(self, device, model, epsilon): - - self.device = device - self.model = model.to(device) - self.epsilon = epsilon # for stability reasons in the lrp-epsilon rule (by default: a very small number) - - # check if the model has any skip connections to accomodate them - self.skip_connections = self.find_skip_connections() - self.msg_passing_layers = self.find_msg_passing_layers() - - """ - explanation functions - """ - - def explain(self, input, neuron_to_explain): - """ - Primary function to call on an LRP instance to start explaining predictions. - First, it registers hooks and runs a forward pass on the input. - Then, it attempts to explain the whole model by looping over the layers in the - model and invoking the explain_single_layer function. - - Args: - input: tensor containing the input sample you wish to explain - neuron_to_explain: the index for a particular neuron in the output layer you wish to explain - - Returns: - R_tensor: a tensor/graph containing the relevance scores of the input graph for - a particular output neuron - preds: the model predictions of the input (for further plotting/processing purposes only) - input: the input that was explained (for further plotting/processing purposes only) - """ - - # register forward hooks to retrieve intermediate activations - # in simple words, when the forward pass is called, - # the following dict() will be filled with (key, value) = ("layer_name", activations) - activations = {} - - def get_activation(name): - def hook(model, input, output): - activations[name] = input[0] - - return hook - - for name, module in self.model.named_modules(): - # unfold any containers so as to register hooks only for their child - # modules (equivalently we are demanding type(module) != nn.Sequential)) - if ( - ("Linear" in str(type(module))) - or ("activation" in str(type(module))) - or ("BatchNorm1d" in str(type(module))) - ): - module.register_forward_hook(get_activation(name)) - - # run a forward pass - self.model.eval() - preds, self.A, self.msg_activations = self.model(input.to(self.device)) - - # get the activations - self.activations = activations - self.num_layers = len(activations.keys()) - self.in_features_dim = self.name2layer(list(activations.keys())[0]).in_features - - print(f"Total number of layers: {self.num_layers}") - - # initialize Rscores for skip connections (in case there are any) - if len(self.skip_connections) != 0: - self.skip_connections_relevance = 0 - - # initialize the Rscores tensor using the output predictions - Rscores = preds[:, neuron_to_explain].reshape(-1, 1).detach() - - # build the Rtensor which is going to be a whole graph of Rscores per node - R_tensor = torch.zeros([Rscores.shape[0], Rscores.shape[0], Rscores.shape[1]]).to(self.device) - for node in range(R_tensor.shape[0]): - R_tensor[node][node] = Rscores[node] - - # loop over layers in the model to propagate Rscores backward - for layer_index in range(self.num_layers, 0, -1): - R_tensor = self.explain_single_layer(R_tensor, layer_index, neuron_to_explain) - - print("Finished explaining all layers.") - - if len(self.skip_connections) != 0: - return R_tensor + self.skip_connections_relevance, preds, input - - return R_tensor, preds, input - - def explain_single_layer(self, R_tensor_old, layer_index, neuron_to_explain): - """ - Attempts to explain a single layer in the model by propagating Rscores backwards using the lrp-epsilon rule. - - Args: - R_tensor_old: a tensor/graph containing the Rscores, of the current layer, to be propagated backwards - layer_index: index that corresponds to the position of the layer in the model (see helper functions) - neuron_to_explain: the index for a particular neuron in the output layer to explain - - Returns: - R_tensor_new: a tensor/graph containing the computed Rscores of the previous layer - """ - - # get layer information - layer_name = self.index2name(layer_index) - layer = self.name2layer(layer_name) - - # get layer activations (depends wether it's a message passing step) - if layer_name in self.msg_passing_layers.keys(): - print(f"Explaining layer {self.num_layers+1-layer_index}/{self.num_layers}: MessagePassing layer") - input = self.msg_activations[layer_name[:-6]].to(self.device).detach() - msg_passing_layer = True - else: - print(f"Explaining layer {self.num_layers+1-layer_index}/{self.num_layers}: {layer}") - input = self.activations[layer_name].to(self.device).detach() - msg_passing_layer = False - - # run lrp - if "Linear" in str(layer): - R_tensor_new = self.eps_rule( - self, - layer, - layer_name, - input, - R_tensor_old, - neuron_to_explain, - msg_passing_layer, - ) - print("- Finished computing Rscores") - return R_tensor_new - else: - if "activation" in str(layer): - print("- skipping layer because it's an activation layer") - elif "BatchNorm1d" in str(layer): - print("- skipping layer because it's a BatchNorm layer") - print("- Rscores do not need to be computed") - return R_tensor_old - - """ - lrp-epsilon rule - """ - - @staticmethod - def eps_rule( - self, - layer, - layer_name, - x, - R_tensor_old, - neuron_to_explain, - msg_passing_layer, - ): - """ - Implements the lrp-epsilon rule presented in the following reference: - https://doi.org/10.1007/978-3-030-28954-6_10. - - Can accomodate message_passing layers if the adjacency matrix and the activations - before the message_passing are provided. - The trick (or as we like to call it, the message_passing hack) is in - a. using the adjacency matrix as the weight matrix in the standard lrp rule - b. transposing the activations to distribute the Rscores over the other dimension - (over nodes instead of features) - - Args: - layer: a torch.nn module with a corresponding weight matrix W - x: vector containing the activations of the previous layer - R_tensor_old: a tensor/graph containing the Rscores, of the current layer, - to be propagated backwards - neuron_to_explain: the index for a particular neuron in the output layer to explain - - Returns: - R_tensor_new: a tensor/graph containing the computed Rscores of the previous layer - """ - - torch.cuda.empty_cache() - - if msg_passing_layer: # message_passing hack - x = torch.transpose(x, 0, 1) # transpose the activations to distribute the Rscores over the other dimension - # (over nodes instead of features) - W = self.A[layer_name[:-6]].detach().to(self.device) # use the adjacency matrix as the weight matrix - else: - W = layer.weight.detach() # get weight matrix - W = torch.transpose(W, 0, 1) # sanity check of forward pass: - # (torch.matmul(x, W) + layer.bias) == layer(x) - - # for the output layer, pick the part of the weight matrix connecting only - # to the neuron you're attempting to explain - if layer == list(self.model.modules())[-1]: - W = W[:, neuron_to_explain].reshape(-1, 1) - - # (1) compute the denominator - denominator = torch.matmul(x, W) + self.epsilon - # (2) scale the Rscores - if msg_passing_layer: # message_passing hack - R_tensor_old = torch.transpose(R_tensor_old, 1, 2) - scaledR = R_tensor_old / denominator - # (3) compute the new Rscores - R_tensor_new = torch.matmul(scaledR, torch.transpose(W, 0, 1)) * x - - # checking conservation of Rscores for a given random node (# 17) - rtol = [1e-5, 1e-4, 1e-3, 1e-2, 1e-1] - for tol in rtol: - if torch.allclose(R_tensor_new[17].sum(), R_tensor_old[17].sum(), rtol=tol): - print(f"- Rscores are conserved up to relative tolerance {str(tol)}") - break - - if layer in self.skip_connections: - # set aside the relevance of the input_features in the skip connection - # recall: it is assumed that the skip connections are defined in the following order: - # torch.cat[(input_features, ...)] ) - self.skip_connections_relevance = self.skip_connections_relevance + R_tensor_new[:, :, : self.in_features_dim] - return R_tensor_new[:, :, self.in_features_dim :] - - if msg_passing_layer: # message_passing hack - return torch.transpose(R_tensor_new, 1, 2) - - return R_tensor_new - - """ - helper functions - """ - - def index2name(self, layer_index): - """ - Given the index of a layer (e.g. 3) returns the name of the layer (e.g. .nn1.3) - """ - layer_name = list(self.activations.keys())[layer_index - 1] - return layer_name - - def name2layer(self, layer_name): - """ - Given the name of a layer (e.g. .nn1.3) returns the corresponding torch module (e.g. Linear(...)) - """ - for name, module in self.model.named_modules(): - if layer_name == name: - return module - - def find_skip_connections(self): - """ - Given a torch model, retuns a list of layers with skip connections... - the elements are torch modules (e.g. Linear(...)) - """ - explainable_layers = [] - for name, module in self.model.named_modules(): - if "lin_s" in name: # for models that are based on Gravnet, skip the lin_s layers - continue - if "Linear" in str(type(module)): - explainable_layers.append(module) - - skip_connections = [] - for layer_index in range(len(explainable_layers) - 1): - if explainable_layers[layer_index].out_features != explainable_layers[layer_index + 1].in_features: - skip_connections.append(explainable_layers[layer_index + 1]) - - return skip_connections - - def find_msg_passing_layers(self): - """ - Returns a list of ".lin_s" layers from model.named_modules() that shall be substituted with message passing - """ - msg_passing_layers = {} - for name, module in self.model.named_modules(): - if "lin_s" in name: # for models that are based on Gravnet, replace the .lin_s layers with message_passing - msg_passing_layers[name] = {} - - return msg_passing_layers diff --git a/mlpf/lrp/make_rmaps.py b/mlpf/lrp/make_rmaps.py deleted file mode 100644 index 28d6e7be0..000000000 --- a/mlpf/lrp/make_rmaps.py +++ /dev/null @@ -1,190 +0,0 @@ -import os - -import matplotlib -import matplotlib.pyplot as plt -import numpy as np -import torch - -# this script makes Rmaps from a processed list of R_tensors - -label_to_class = { - 0: "null", - 1: "chhadron", - 2: "nhadron", - 3: "photon", - 4: "electron", - 5: "muon", -} - -label_to_p4 = { - 6: "charge", - 7: "pt", - 8: "eta", - 9: "sin phi", - 10: "cos phi", - 11: "energy", -} - - -def indexing_by_relevance(num, pid): - labels = [] - labels.append(pid.capitalize()) - for i in range(num - 1): - if i == 0: - labels.append("Most relevant neighbor") - elif i == 1: - labels.append("2nd most relevant neighbor") - elif i == 2: - labels.append("3rd most relevant neighbor") - else: - labels.append(str(i + 1) + "th most relevant neighbor") - return labels - - -def process_Rtensor(node, Rtensor, neighbors): - """ - Given an Rtensor ~ (nodes, in_features) does some preprocessing on it - - Args - node: an index for the node we're prcoessing the Rmap for - Rtensor: the tensor/graph of Rscores for that node - neighbors: # of neighbors to keep when processing the Rmap - - Returns - an absolutized, normalized, and sorted Rtensor (sorted the - rows/neighbors by relevance aside from the first row which - is always the node itself) - """ - Rtensor = Rtensor.absolute() - Rtensor = Rtensor / Rtensor.sum() - - # put node itself as the first one - tmp = Rtensor[0] - Rtensor[0] = Rtensor[node] - Rtensor[node] = tmp - - # rank all the others by relevance - rank_relevance_msk = ( - Rtensor[1:].sum(axis=1).sort(descending=True)[1] - ) # the index ":1" is to skip the node itself when sorting - Rtensor[1:] = Rtensor[1:][rank_relevance_msk] - - # Rtensor[Rtensor.sum(axis=1).bool()] # remove zero rows - return Rtensor[: neighbors + 1] - - -def make_Rmaps( - outpath, - Rtensors, - inputs, - preds, - pid="chhadron", - neighbors=2, - out_neuron=0, -): # noqa C901 - """ - Recall each event has a corresponding Rmap per node in the event. - This function process the Rmaps for a given pid. - - Args - Rtensors: a list of len()=events processed. Each element is an Rtensor ~ (nodes, nodes, in_features) - pid: class label to process (choices are ['null', 'chhadron', 'nhadron', photon', electron', muon']) - neighbors: how many neighbors to show in the Rmap - """ - in_features = Rtensors[0].shape[-1] - - Rtensor_correct, Rtensor_incorrect = torch.zeros(neighbors + 1, in_features), torch.zeros(neighbors + 1, in_features) - num_Rtensors_correct, num_Rtensors_incorrect = 0, 0 - - for event, event_Rscores in enumerate(Rtensors): - for node, node_Rtensor in enumerate(event_Rscores): - true_class = torch.argmax(inputs[event]["ygen_id"][node]).item() - pred_class = torch.argmax(preds[event][node][:6]).item() - - # plot for a particular pid - if label_to_class[true_class] == pid: - # check if the node was correctly classified - if pred_class == true_class: - Rtensor_correct = Rtensor_correct + process_Rtensor(node, node_Rtensor, neighbors) - num_Rtensors_correct = num_Rtensors_correct + 1 - else: - Rtensor_incorrect = Rtensor_incorrect + process_Rtensor(node, node_Rtensor, neighbors) - num_Rtensors_incorrect = num_Rtensors_incorrect + 1 - - Rtensor_correct = Rtensor_correct / num_Rtensors_correct - Rtensor_incorrect = Rtensor_incorrect / num_Rtensors_incorrect - tot_num = num_Rtensors_correct + num_Rtensors_incorrect - - features = [ - "Track|cluster", - "$p_{T}|E_{T}$", - r"$\eta$", - r"$\phi$", - "P|E", - r"$\eta_\mathrm{out}|E_{em}$", - r"$\phi_\mathrm{out}|E_{had}$", - "charge", - "is_gen_mu", - "is_gen_el", - ] - - node_types = indexing_by_relevance(neighbors + 1, pid) # only plot 6 rows/neighbors in Rmap - - for status, var in { - "correct": Rtensor_correct, - "incorrect": Rtensor_incorrect, - }.items(): - print(f"Making Rmaps for {status}ly classified {pid}") - if status == "correct": - num = num_Rtensors_correct - else: - num = num_Rtensors_incorrect - - print(f"fraction is: {num}/{tot_num}") - - if num == 0: - continue - - _, ax = plt.subplots(figsize=(20, 10)) - if out_neuron < 6: - ax.set_title( - f"Average relevance score matrix for {pid}s's classification score " - + f"of {num}/{tot_num} {status}ly classified elements", - fontsize=26, - ) - else: - ax.set_title( - f"Average relevance score matrix for {pid}'s {label_to_p4[out_neuron]} " - + f"of {num}/{tot_num} {status}ly classified elements", - fontsize=26, - ) - - ax.set_xticks(np.arange(len(features))) - ax.set_yticks(np.arange(len(node_types))) - ax.set_xticklabels(features, fontsize=22) - ax.set_yticklabels(node_types, fontsize=20) - for col in range(len(features)): - for row in range(len(node_types)): - ax.text( - col, - row, - round(var[row, col].item(), 5), - ha="center", - va="center", - color="w", - fontsize=14, - ) - - plt.imshow( - (var[: neighbors + 1] + 1e-12).numpy(), - cmap="copper", - aspect="auto", - norm=matplotlib.colors.LogNorm(vmin=1e-3), - ) - - # create directory to hold Rmaps - rmap_dir = outpath + "/rmaps/" - if not os.path.exists(rmap_dir): - os.makedirs(rmap_dir) - - plt.savefig(f"{rmap_dir}/Rmap_{pid}_{status}_neuron_{out_neuron}.pdf") diff --git a/mlpf/lrp/model.py b/mlpf/lrp/model.py deleted file mode 100644 index e9eddac1e..000000000 --- a/mlpf/lrp/model.py +++ /dev/null @@ -1,222 +0,0 @@ -from typing import Optional, Union - -import torch -import torch.nn as nn -from torch import Tensor -from torch.nn import Linear -from torch_geometric.nn.conv import MessagePassing -from torch_geometric.typing import OptTensor, PairOptTensor, PairTensor -from torch_geometric.utils import to_dense_adj -from torch_scatter import scatter - -try: - from torch_cluster import knn -except ImportError: - knn = None -# from torch_cluster import knn_graph as knn - - -class MLPF(nn.Module): - """ - GNN model based on Gravnet... - - Forward pass returns - preds: tensor of predictions containing a concatenated representation of the pids and p4 - A: dict() object containing adjacency matrices for each message passing - msg_activations: dict() object containing activations before each message passing - """ - - def __init__( - self, - input_dim=12, - output_dim_id=6, - output_dim_p4=6, - embedding_dim=64, - hidden_dim1=64, - hidden_dim2=60, - num_convs=2, - space_dim=4, - propagate_dim=30, - k=8, - ): - super(MLPF, self).__init__() - - # self.act = nn.ReLU - self.act = nn.ELU - - # (1) embedding - self.nn1 = nn.Sequential( - nn.Linear(input_dim, hidden_dim1), - self.act(), - nn.Linear(hidden_dim1, hidden_dim1), - self.act(), - nn.Linear(hidden_dim1, hidden_dim1), - self.act(), - nn.Linear(hidden_dim1, embedding_dim), - ) - - self.conv = nn.ModuleList() - for i in range(num_convs): - self.conv.append( - GravNetConv_LRP( - embedding_dim, - embedding_dim, - space_dim, - propagate_dim, - k, - ) - ) - - # (3) DNN layer: classifiying pid - self.nn2 = nn.Sequential( - nn.Linear(input_dim + embedding_dim, hidden_dim2), - self.act(), - nn.Linear(hidden_dim2, hidden_dim2), - self.act(), - nn.Linear(hidden_dim2, hidden_dim2), - self.act(), - nn.Linear(hidden_dim2, output_dim_id), - ) - - # (4) DNN layer: regressing p4 - self.nn3 = nn.Sequential( - nn.Linear(input_dim + output_dim_id, hidden_dim2), - self.act(), - nn.Linear(hidden_dim2, hidden_dim2), - self.act(), - nn.Linear(hidden_dim2, hidden_dim2), - self.act(), - nn.Linear(hidden_dim2, output_dim_p4), - ) - - def forward(self, batch): - - x0 = batch.x - - # embed the inputs - embedding = self.nn1(x0) - - # preform a series of graph convolutions - A = {} - msg_activations = {} - for num, conv in enumerate(self.conv): - ( - embedding, - A[f"conv.{num}"], - msg_activations[f"conv.{num}"], - ) = conv(embedding) - - # predict the pid's - preds_id = self.nn2(torch.cat([x0, embedding], axis=-1)) - - # predict the p4's - preds_p4 = self.nn3(torch.cat([x0, preds_id], axis=-1)) - - return torch.cat([preds_id, preds_p4], axis=-1), A, msg_activations - - -class GravNetConv_LRP(MessagePassing): - """ - Copied from pytorch_geometric source code, with the following edits - a. used reduce='sum' instead of reduce='mean' in the message passing - b. removed skip connection - c. retrieved adjacency matrix and the activations before the message passing, - both are useful only for LRP purposes - d. switched the execution of self.lin_s & self.lin_p so that the message passing - step can substitute out of the box self.lin_s for lrp purposes - """ - - def __init__( - self, - in_channels: int, - out_channels: int, - space_dimensions: int, - propagate_dimensions: int, - k: int, - num_workers: int = 1, - **kwargs, - ): - super().__init__(flow="source_to_target", **kwargs) - - if knn is None: - raise ImportError("`GravNetConv` requires `torch-cluster`.") - - self.in_channels = in_channels - self.out_channels = out_channels - self.k = k - self.num_workers = num_workers - - self.lin_p = Linear(in_channels, propagate_dimensions) - self.lin_s = Linear(in_channels, space_dimensions) - self.lin_out = Linear(propagate_dimensions, out_channels) - - self.reset_parameters() - - def reset_parameters(self): - self.lin_s.reset_parameters() - self.lin_p.reset_parameters() - self.lin_out.reset_parameters() - - def forward( - self, - x: Union[Tensor, PairTensor], - batch: Union[OptTensor, Optional[PairTensor]] = None, - ) -> Tensor: - """""" - - is_bipartite: bool = True - if isinstance(x, Tensor): - x: PairTensor = (x, x) - is_bipartite = False - - if x[0].dim() != 2: - raise ValueError("Static graphs not supported in 'GravNetConv'") - - b: PairOptTensor = (None, None) - if isinstance(batch, Tensor): - b = (batch, batch) - elif isinstance(batch, tuple): - assert batch is not None - b = (batch[0], batch[1]) - - # embed the inputs before message passing - msg_activations = self.lin_p(x[0]) - - # transform to the space dimension to build the graph - s_l: Tensor = self.lin_s(x[0]) - s_r: Tensor = self.lin_s(x[1]) if is_bipartite else s_l - - edge_index = knn(s_l, s_r, self.k, b[0], b[1]).flip([0]) - # edge_index = knn_graph(s_l, self.k, b[0], b[1]).flip([0]) - - edge_weight = (s_l[edge_index[0]] - s_r[edge_index[1]]).pow(2).sum(-1) - edge_weight = torch.exp(-10.0 * edge_weight) # 10 gives a better spread - - # return the adjacency matrix of the graph for lrp purposes - A = to_dense_adj(edge_index.to("cpu"), edge_attr=edge_weight.to("cpu"))[0] # adjacency matrix - - # message passing - out = self.propagate( - edge_index, - x=(msg_activations, None), - edge_weight=edge_weight, - size=(s_l.size(0), s_r.size(0)), - ) - - return self.lin_out(out), A, msg_activations - - def message(self, x_j: Tensor, edge_weight: Tensor) -> Tensor: - return x_j * edge_weight.unsqueeze(1) - - def aggregate(self, inputs: Tensor, index: Tensor, dim_size: Optional[int] = None) -> Tensor: - out_mean = scatter( - inputs, - index, - dim=self.node_dim, - dim_size=dim_size, - reduce="sum", - ) - return out_mean - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.in_channels}, " f"{self.out_channels}, k={self.k})" diff --git a/mlpf/lrp_mlpf_pipeline.py b/mlpf/lrp_mlpf_pipeline.py deleted file mode 100644 index e6eba7811..000000000 --- a/mlpf/lrp_mlpf_pipeline.py +++ /dev/null @@ -1,129 +0,0 @@ -import argparse -import pickle as pkl - -import torch -from lrp import LRP_MLPF, MLPF, make_Rmaps -from pyg import PFGraphDataset, dataloader_qcd, load_model - -# this script runs lrp on a trained MLPF model - -parser = argparse.ArgumentParser() - -# for saving the model -parser.add_argument( - "--dataset_qcd", - type=str, - default="../data/delphes/pythia8_qcd", - help="testing dataset path", -) -parser.add_argument( - "--outpath", - type=str, - default="../experiments/", - help="path to the trained model directory", -) -parser.add_argument("--load_model", type=str, default="", help="Which model to load") -parser.add_argument( - "--load_epoch", - type=int, - default=0, - help="Which epoch of the model to load", -) -parser.add_argument( - "--out_neuron", - type=int, - default=0, - help="the output neuron you wish to explain", -) -parser.add_argument("--pid", type=str, default="chhadron", help="Which model to load") -parser.add_argument( - "--n_test", - type=int, - default=50, - help="number of data files to use for testing.. each file contains 100 events", -) -parser.add_argument("--run_lrp", dest="run_lrp", action="store_true", help="runs lrp") -parser.add_argument( - "--make_rmaps", - dest="make_rmaps", - action="store_true", - help="makes rmaps", -) - -args = parser.parse_args() - - -if __name__ == "__main__": - - if args.run_lrp: - # Check if the GPU configuration and define the global base device - if torch.cuda.device_count() > 0: - print(f"Will use {torch.cuda.device_count()} gpu(s)") - print("GPU model:", torch.cuda.get_device_name(0)) - device = torch.device("cuda:0") - else: - print("Will use cpu") - device = torch.device("cpu") - - # get sample dataset - print("Fetching the data..") - full_dataset_qcd = PFGraphDataset(args.dataset_qcd) - loader = dataloader_qcd( - full_dataset_qcd, - multi_gpu=False, - n_test=args.n_test, - batch_size=1, - ) - - # load a pretrained model and update the outpath - outpath = args.outpath + args.load_model - state_dict, model_kwargs, outpath = load_model(device, outpath, args.load_model, args.load_epoch) - model = MLPF(**model_kwargs) - model.load_state_dict(state_dict) - model.to(device) - model.eval() - - # initialize placeholders for Rscores, the event inputs, and the event predictions - Rtensors_list, preds_list, inputs_list = [], [], [] - - # define the lrp instance - lrp_instance = LRP_MLPF(device, model, epsilon=1e-9) - - # loop over events to explain them - for i, event in enumerate(loader): - print(f"Explaining event # {i}") - - # run lrp on the event - Rtensor, pred, input = lrp_instance.explain(event, neuron_to_explain=args.out_neuron) - - # store the Rscores, the event inputs, and the event predictions - Rtensors_list.append(Rtensor.detach().to("cpu")) - preds_list.append(pred.detach().to("cpu")) - inputs_list.append(input.detach().to("cpu").to_dict()) - - with open(f"{outpath}/Rtensors_list.pkl", "wb") as f: - pkl.dump(Rtensors_list, f) - with open(f"{outpath}/inputs_list.pkl", "wb") as f: - pkl.dump(inputs_list, f) - with open(f"{outpath}/preds_list.pkl", "wb") as f: - pkl.dump(preds_list, f) - - if args.make_rmaps: - outpath = args.outpath + args.load_model - with open(f"{outpath}/Rtensors_list.pkl", "rb") as f: - Rtensors_list = pkl.load(f) - with open(f"{outpath}/inputs_list.pkl", "rb") as f: - inputs_list = pkl.load(f) - with open(f"{outpath}/preds_list.pkl", "rb") as f: - preds_list = pkl.load(f) - - print("Making Rmaps..") - make_Rmaps( - args.outpath, - Rtensors_list, - inputs_list, - preds_list, - pid=args.pid, - neighbors=3, - out_neuron=args.out_neuron, - ) diff --git a/mlpf/pyg/PFGraphDataset.py b/mlpf/pyg/PFGraphDataset.py index 7f0aede6d..f9339a726 100644 --- a/mlpf/pyg/PFGraphDataset.py +++ b/mlpf/pyg/PFGraphDataset.py @@ -1,15 +1,9 @@ -try: - from pyg.cms_utils import prepare_data_cms -except ImportError: - from cms_utils import prepare_data_cms - import multiprocessing import os.path as osp -import pickle from glob import glob import torch -from torch_geometric.data import Data, Dataset +from torch_geometric.data import Dataset def process_func(args): @@ -37,7 +31,7 @@ def __init__(self, root, data, transform=None, pre_transform=None): @property def raw_file_names(self): - raw_list = glob(osp.join(self.raw_dir, "*.pkl")) + raw_list = glob(osp.join(self.raw_dir, "*")) print("PFGraphDataset nfiles={}".format(len(raw_list))) return sorted([raw_path.replace(self.raw_dir, ".") for raw_path in raw_list]) @@ -65,50 +59,38 @@ def download(self): def process_single_file(self, raw_file_name): """ - Loads a list of 100 events from a pkl file and generates pytorch geometric Data() objects - and stores them in .pt format. - For cms data, each element is assumed to be a dict('Xelem', 'ygen', ycand') - of numpy rec_arrays with the first element in ygen/ycand is the pid - For delphes data, each element is assumed to be a dict('X', 'ygen', ycand') - of numpy standard arrays with the first element in ygen/ycand is the pid + Loads raw datafile information and generates PyG Data() objects and stores them in .pt format. Args - raw_file_name: a pkl file + raw_file_name: raw data file name. Returns batched_data: a list of Data() objects of the form - cms ~ Data(x=[#elem, 41], ygen=[#elem, 6], ygen_id=[#elem, 9], ycand=[#elem, 6], ycand_id=[#elem, 9]) - delphes ~ Data(x=[#elem, 12], ygen=[#elem, 6], ygen_id=[#elem, 6], ycand=[#elem, 6], ycand_id=[#elem, 6]) + cms ~ Data(x=[#, 41], ygen=[#, 6], ygen_id=[#, 9], ycand=[#, 6], ycand_id=[#, 9]) + delphes ~ Data(x=[#, 12], ygen=[#elem, 6], ygen_id=[#, 6], ycand=[#, 6], ycand_id=[#, 6]) """ if self.data == "cms": + from cms.cms_utils import prepare_data_cms + return prepare_data_cms(osp.join(self.raw_dir, raw_file_name)) elif self.data == "delphes": - # load the data pkl file - with open(osp.join(self.raw_dir, raw_file_name), "rb") as fi: - data = pickle.load(fi, encoding="iso-8859-1") - - batched_data = [] - for i in range(len(data["X"])): - # remove from ygen & ycand the first element (PID) so that they only contain the regression variables - d = Data( - x=torch.tensor(data["X"][i], dtype=torch.float), - ygen=torch.tensor(data["ygen"][i], dtype=torch.float)[:, 1:], - ygen_id=torch.tensor(data["ygen"][i], dtype=torch.float)[:, 0].long(), - ycand=torch.tensor(data["ycand"][i], dtype=torch.float)[:, 1:], - ycand_id=torch.tensor(data["ycand"][i], dtype=torch.float)[:, 0].long(), - ) + from delphes.delphes_utils import prepare_data_delphes - batched_data.append(d) - - return batched_data + return prepare_data_delphes(osp.join(self.raw_dir, raw_file_name)) def process_multiple_files(self, filenames, idx_file): - datas = [self.process_single_file(fn) for fn in filenames] + datas = [] + for fn in filenames: + x = self.process_single_file(fn) + if x is None: + continue + datas.append(x) + datas = sum(datas, []) p = osp.join(self.processed_dir, "data_{}.pt".format(idx_file)) - print(p) torch.save(datas, p) + print(f"saved file {p}") def process(self, num_files_to_batch): idx_file = 0 @@ -140,19 +122,8 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--data", type=str, required=True, help="'cms' or 'delphes'?") parser.add_argument("--dataset", type=str, required=True, help="Input data path") - parser.add_argument( - "--processed_dir", - type=str, - help="processed", - required=False, - default=None, - ) - parser.add_argument( - "--num-files-merge", - type=int, - default=10, - help="number of files to merge", - ) + parser.add_argument("--processed_dir", type=str, help="processed", required=False, default=None) + parser.add_argument("--num-files-merge", type=int, default=10, help="number of files to merge") parser.add_argument("--num-proc", type=int, default=24, help="number of processes") args = parser.parse_args() return args diff --git a/mlpf/pyg/README.md b/mlpf/pyg/README.md index 3f064c376..c12d8c70f 100644 --- a/mlpf/pyg/README.md +++ b/mlpf/pyg/README.md @@ -6,13 +6,14 @@ conda env create -f environment.yml conda activate mlpf ``` -# Training +# Supervised training ### DELPHES training The dataset is available from zenodo: https://doi.org/10.5281/zenodo.4452283. To download and process the full DELPHES dataset: ```bash +cd delphes/ ./get_data_delphes.sh ``` @@ -21,7 +22,7 @@ This script will download and process the data under a directory called `data/de To perform a quick training on the dataset: ```bash cd ../ -python -u pyg_pipeline.py --data delphes --dataset= --dataset_qcd= +python -u pyg_pipeline.py --data delphes --dataset=../data/delphes/ --dataset_test=../data/delphes/pythia8_ttbar ``` To load a pretrained model which is stored in a directory under `particleflow/experiments` for evaluation: @@ -34,6 +35,7 @@ python -u pyg_pipeline.py --data delphes --load --load_model= - To download and process the full CMS dataset: ```bash +cd cms/ ./get_data_cms.sh ``` This script will download and process the data under a directory called `data/cms` under `particleflow`. @@ -41,7 +43,7 @@ This script will download and process the data under a directory called `data/cm To perform a quick training on the dataset: ```bash cd ../ -python -u pyg_pipeline.py --data cms --dataset= --dataset_qcd= +python -u pyg_pipeline.py --data cms --dataset=../data/cms/TTbar_14TeV_TuneCUETP8M1_cfi/ --dataset_test=../data/cms/QCDForPF_14TeV_TuneCUETP8M1_cfi/ ``` To load a pretrained model which is stored in a directory under `particleflow/experiments` for evaluation: @@ -49,11 +51,3 @@ To load a pretrained model which is stored in a directory under `particleflow/ex cd ../ python -u pyg_pipeline.py --data cms --load --load_model= --load_epoch= --dataset= --dataset_qcd= ``` - -### XAI and LRP studies on MLPF - -You must have a pre-trained model under `particleflow/experiments`: -```bash -cd ../ -python -u lrp_mlpf_pipeline.py --run_lrp --make_rmaps --load_model= --load_epoch= -``` diff --git a/mlpf/pyg/__init__.py b/mlpf/pyg/__init__.py index 67d2a6a21..e69de29bb 100644 --- a/mlpf/pyg/__init__.py +++ b/mlpf/pyg/__init__.py @@ -1,35 +0,0 @@ -from pyg.args import parse_args # noqa F401 -from pyg.cms_plots import distribution_icls -from pyg.cms_plots import plot_cm -from pyg.cms_plots import plot_dist -from pyg.cms_plots import plot_eff_and_fake_rate -from pyg.cms_plots import plot_energy_res -from pyg.cms_plots import plot_eta_res -from pyg.cms_plots import plot_met -from pyg.cms_plots import plot_multiplicity -from pyg.cms_plots import plot_numPFelements -from pyg.cms_plots import plot_sum_energy -from pyg.cms_plots import plot_sum_pt -from pyg.cms_utils import CLASS_NAMES_CMS -from pyg.cms_utils import CLASS_NAMES_CMS_LATEX -from pyg.cms_utils import prepare_data_cms -from pyg.delphes_plots import name_to_pid_cms -from pyg.delphes_plots import name_to_pid_delphes -from pyg.delphes_plots import pid_to_name_cms -from pyg.delphes_plots import pid_to_name_delphes -from pyg.evaluate import make_plots_cms -from pyg.evaluate import make_predictions -from pyg.evaluate import postprocess_predictions -from pyg.model import MLPF # noqa F401 -from pyg.PFGraphDataset import PFGraphDataset # noqa F401 -from pyg.training import training_loop # noqa F401 -from pyg.utils import dataloader_qcd -from pyg.utils import dataloader_ttbar -from pyg.utils import features_cms -from pyg.utils import features_delphes -from pyg.utils import load_model -from pyg.utils import make_file_loaders -from pyg.utils import make_plot_from_lists -from pyg.utils import one_hot_embedding -from pyg.utils import save_model -from pyg.utils import target_p4 diff --git a/mlpf/pyg/args.py b/mlpf/pyg/args.py index bf8c19af2..088aed741 100644 --- a/mlpf/pyg/args.py +++ b/mlpf/pyg/args.py @@ -5,108 +5,42 @@ def parse_args(): parser = argparse.ArgumentParser() # for data loading - parser.add_argument( - "--num_workers", - type=int, - default=2, - help="number of subprocesses used for data loading", - ) - parser.add_argument( - "--prefetch_factor", - type=int, - default=4, - help="number of samples loaded in advance by each worker", - ) + parser.add_argument("--num_workers", type=int, default=2, help="number of subprocesses used for data loading") + parser.add_argument("--prefetch_factor", type=int, default=4, help="number of samples loaded in advance by each worker") # for saving the model - parser.add_argument( - "--outpath", - type=str, - default="../experiments/", - help="output folder", - ) - parser.add_argument( - "--model_prefix", - type=str, - default="MLPF_model", - help="directory to hold the model and all plots under args.outpath/args.model_prefix", - ) + parser.add_argument("--outpath", type=str, default="../experiments/", help="output folder") + parser.add_argument("--model_prefix", type=str, default="MLPF_model", help="directory to hold the model and all plots") # for loading the data parser.add_argument("--data", type=str, required=True, help="cms or delphes?") - parser.add_argument( - "--dataset", - type=str, - default="../data/delphes/pythia8_ttbar", - help="training dataset path", - ) - parser.add_argument( - "--dataset_test", - type=str, - default="../data/delphes/pythia8_qcd", - help="testing dataset path", - ) + parser.add_argument("--dataset", type=str, default="../data/delphes/pythia8_ttbar", help="training dataset path") + parser.add_argument("--dataset_test", type=str, default="../data/delphes/pythia8_qcd", help="testing dataset path") parser.add_argument("--sample", type=str, default="QCD", help="sample to test on") parser.add_argument( - "--n_train", - type=int, - default=1, - help="number of data files to use for training.. each file contains 100 events", + "--n_train", type=int, default=1, help="number of data files to use for training.. each file contains 100 events" ) parser.add_argument( - "--n_valid", - type=int, - default=1, - help="number of data files to use for validation.. each file contains 100 events", + "--n_valid", type=int, default=1, help="number of data files to use for validation.. each file contains 100 events" ) parser.add_argument( - "--n_test", - type=int, - default=1, - help="number of data files to use for testing.. each file contains 100 events", + "--n_test", type=int, default=1, help="number of data files to use for testing.. each file contains 100 events" ) - parser.add_argument( - "--overwrite", - dest="overwrite", - action="store_true", - help="Overwrites the model if True", - ) + parser.add_argument("--overwrite", dest="overwrite", action="store_true", help="Overwrites the model if True") # for loading a pre-trained model - parser.add_argument( - "--load", - dest="load", - action="store_true", - help="Load the model (no training)", - ) - parser.add_argument( - "--load_epoch", - type=int, - default=-1, - help="Which epoch of the model to load for evaluation", - ) + parser.add_argument("--load", dest="load", action="store_true", help="Load the model (no training)") + parser.add_argument("--load_epoch", type=int, default=-1, help="Which epoch of the model to load for evaluation") # for training hyperparameters parser.add_argument("--n_epochs", type=int, default=3, help="number of training epochs") parser.add_argument( - "--batch_size", - type=int, - default=1, - help="number of events to run inference on before updating the loss", + "--batch_size", type=int, default=1, help="number of events to run inference on before updating the loss" ) + parser.add_argument("--patience", type=int, default=30, help="patience before early stopping") parser.add_argument( - "--patience", - type=int, - default=30, - help="patience before early stopping", - ) - parser.add_argument( - "--target", - type=str, - default="gen", - choices=["cand", "gen"], - help="Regress to PFCandidates or GenParticles", + "--target", type=str, default="gen", choices=["cand", "gen"], help="Regress to PFCandidates or GenParticles" ) parser.add_argument("--lr", type=float, default=1e-4, help="learning rate") parser.add_argument( @@ -124,61 +58,22 @@ def parse_args(): # for model architecture parser.add_argument( - "--hidden_dim1", - type=int, - default=126, - help="hidden dimension of layers before the graph convolutions", - ) - parser.add_argument( - "--hidden_dim2", - type=int, - default=256, - help="hidden dimension of layers after the graph convolutions", - ) - parser.add_argument( - "--embedding_dim", - type=int, - default=32, - help="encoded element dimension", - ) - parser.add_argument( - "--num_convs", - type=int, - default=3, - help="number of graph convolutions", - ) - parser.add_argument( - "--space_dim", - type=int, - default=4, - help="Spatial dimension for clustering in gravnet layer", - ) - parser.add_argument( - "--propagate_dim", - type=int, - default=8, - help="The number of features to be propagated between the vertices", + "--hidden_dim1", type=int, default=126, help="hidden dimension of layers before the graph convolutions" ) parser.add_argument( - "--nearest", - type=int, - default=4, - help="k nearest neighbors in gravnet layer", + "--hidden_dim2", type=int, default=256, help="hidden dimension of layers after the graph convolutions" ) + parser.add_argument("--embedding_dim", type=int, default=32, help="encoded element dimension") + parser.add_argument("--num_convs", type=int, default=3, help="number of graph convolutions") + parser.add_argument("--space_dim", type=int, default=4, help="Gravnet hyperparameter") + parser.add_argument("--propagate_dim", type=int, default=8, help="Gravnet hyperparameter") + parser.add_argument("--nearest", type=int, default=4, help="k nearest neighbors in gravnet layer") # for testing the model parser.add_argument( - "--make_predictions", - dest="make_predictions", - action="store_true", - help="run inference on the test data", - ) - parser.add_argument( - "--make_plots", - dest="make_plots", - action="store_true", - help="makes plots of the test predictions", + "--make_predictions", dest="make_predictions", action="store_true", help="run inference on the test data" ) + parser.add_argument("--make_plots", dest="make_plots", action="store_true", help="makes plots of the test predictions") args = parser.parse_args() diff --git a/mlpf/pyg/cms_plots.py b/mlpf/pyg/cms/cms_plots.py similarity index 78% rename from mlpf/pyg/cms_plots.py rename to mlpf/pyg/cms/cms_plots.py index f9b511d4c..28a16173f 100644 --- a/mlpf/pyg/cms_plots.py +++ b/mlpf/pyg/cms/cms_plots.py @@ -1,19 +1,21 @@ import itertools +import time import boost_histogram as bh import matplotlib.pyplot as plt import mplhep import numpy as np import sklearn.metrics -from pyg.cms_utils import ( - CLASS_LABELS_CMS, - CLASS_NAMES_CMS, - CLASS_NAMES_CMS_LATEX, -) +import torch + +from .cms_utils import CLASS_LABELS_CMS, CLASS_NAMES_CMS, CLASS_NAMES_CMS_LATEX mplhep.style.use(mplhep.styles.CMS) +class_names = {k: v for k, v in zip(CLASS_LABELS_CMS, CLASS_NAMES_CMS)} + + def cms_label(ax, x0=0.01, x1=0.15, x2=0.98, y=0.94): plt.figtext( x0, @@ -43,12 +45,6 @@ def cms_label(ax, x0=0.01, x1=0.15, x2=0.98, y=0.94): ) -# def cms_label_sample_label(x0=0.12, x1=0.23, x2=0.67, y=0.90): -# plt.figtext(x0, y,'CMS',fontweight='bold', wrap=True, horizontalalignment='left') -# plt.figtext(x1, y,'Simulation Preliminary', style='italic', wrap=True, horizontalalignment='left') -# plt.figtext(x2, y,'Run 3 (14 TeV), $\mathrm{t}\overline{\mathrm{t}}$ events', wrap=False, horizontalalignment='left') - - def sample_label(sample, ax, additional_text="", x=0.01, y=0.87): if sample == "QCD": plt.text( @@ -68,35 +64,6 @@ def sample_label(sample, ax, additional_text="", x=0.01, y=0.87): ) -def apply_thresholds_f(ypred_raw_f, thresholds): - msk = np.ones_like(ypred_raw_f) - for i in range(len(thresholds)): - msk[:, i + 1] = ypred_raw_f[:, i + 1] > thresholds[i] - ypred_id_f = np.argmax(ypred_raw_f * msk, axis=-1) - - # best_2 = np.partition(ypred_raw_f, -2, axis=-1)[..., -2:] - # diff = np.abs(best_2[:, -1] - best_2[:, -2]) - # ypred_id_f[diff<0.05] = 0 - - return ypred_id_f - - -def apply_thresholds(ypred_raw, thresholds): - msk = np.ones_like(ypred_raw) - for i in range(len(thresholds)): - msk[:, :, i + 1] = ypred_raw[:, :, i + 1] > thresholds[i] - ypred_id = np.argmax(ypred_raw * msk, axis=-1) - - # best_2 = np.partition(ypred_raw, -2, axis=-1)[..., -2:] - # diff = np.abs(best_2[:, :, -1] - best_2[:, :, -2]) - # ypred_id[diff<0.05] = 0 - - return ypred_id - - -class_names = {k: v for k, v in zip(CLASS_LABELS_CMS, CLASS_NAMES_CMS)} - - def plot_numPFelements(X, outpath, sample): plt.figure() ax = plt.axes() @@ -110,7 +77,9 @@ def plot_numPFelements(X, outpath, sample): plt.close() -def plot_met(X, yvals, outpath, sample): +def plot_met(yvals, outpath, sample): + print("plot_met...") + sum_px = np.sum(yvals["gen_px"], axis=1) sum_py = np.sum(yvals["gen_py"], axis=1) gen_met = np.sqrt(sum_px**2 + sum_py**2)[:, 0] @@ -153,7 +122,9 @@ def plot_met(X, yvals, outpath, sample): plt.close() -def plot_sum_energy(X, yvals, outpath, sample): +def plot_sum_energy(yvals, outpath, sample): + print("plot_sum_energy...") + plt.figure() ax = plt.axes() @@ -179,7 +150,8 @@ def plot_sum_energy(X, yvals, outpath, sample): plt.close() -def plot_sum_pt(X, yvals, outpath, sample): +def plot_sum_pt(yvals, outpath, sample): + print("plot_sum_pt...") plt.figure() ax = plt.axes() @@ -206,7 +178,7 @@ def plot_sum_pt(X, yvals, outpath, sample): plt.close() -def plot_energy_res(X, yvals_f, pid, b, ylim, outpath, sample): +def plot_energy_res(yvals_f, pid, b, ylim, outpath, sample): plt.figure() ax = plt.axes() @@ -239,14 +211,11 @@ def plot_energy_res(X, yvals_f, pid, b, ylim, outpath, sample): sample_label(sample, ax, f", {CLASS_NAMES_CMS_LATEX[pid]}") plt.legend(loc=(0.4, 0.7)) plt.ylim(1, ylim) - plt.savefig( - f"{outpath}/energy_res_{CLASS_NAMES_CMS[pid]}.pdf", - bbox_inches="tight", - ) + plt.savefig(f"{outpath}/energy_res_{CLASS_NAMES_CMS[pid]}.pdf", bbox_inches="tight") plt.close() -def plot_eta_res(X, yvals_f, pid, ylim, outpath, sample): +def plot_eta_res(yvals_f, pid, ylim, outpath, sample): plt.figure() ax = plt.axes() @@ -281,14 +250,11 @@ def plot_eta_res(X, yvals_f, pid, ylim, outpath, sample): sample_label(sample, ax, f", {CLASS_NAMES_CMS_LATEX[pid]}") plt.legend(loc=(0.0, 0.7)) plt.ylim(1, ylim) - plt.savefig( - f"{outpath}/eta_res_{CLASS_NAMES_CMS[pid]}.pdf", - bbox_inches="tight", - ) + plt.savefig(f"{outpath}/eta_res_{CLASS_NAMES_CMS[pid]}.pdf", bbox_inches="tight") plt.close() -def plot_multiplicity(X, yvals, outpath, sample): +def plot_multiplicity(yvals, outpath, sample): for icls in range(1, 8): # Plot the particle multiplicities npred = np.sum(yvals["pred_cls_id"] == icls, axis=1) @@ -487,14 +453,6 @@ def plot_eff_and_fake_rate( plt.savefig(f"{outpath}/fake_icls{icls}_ivar{ivar}.pdf", bbox_inches="tight") plt.close() - # mplhep.histplot(fake, bins=hist_gen[1], label="fake rate", color="red") - - -# plt.legend(frameon=False) -# plt.ylim(0,1.4) -# plt.xlabel(xlabel) -# plt.ylabel("Fraction of particles / bin") - def plot_cm(yvals_f, msk_X_f, label, outpath): @@ -529,11 +487,7 @@ def plot_cm(yvals_f, msk_X_f, label, outpath): cms_label(ax, y=1.01) # cms_label_sample_label(x1=0.18, x2=0.52, y=0.82) - plt.xticks( - range(len(CLASS_NAMES_CMS_LATEX)), - CLASS_NAMES_CMS_LATEX, - rotation=45, - ) + plt.xticks(range(len(CLASS_NAMES_CMS_LATEX)), CLASS_NAMES_CMS_LATEX, rotation=45) plt.yticks(range(len(CLASS_NAMES_CMS_LATEX)), CLASS_NAMES_CMS_LATEX) plt.xlabel(f"{label} candidate ID") @@ -602,3 +556,136 @@ def distribution_icls(yvals_f, outpath): plt.tight_layout() plt.savefig(f"{outpath}/distribution_icls{icls}.pdf", bbox_inches="tight") plt.close() + + +def make_plots_cms(pred_path, plot_path, sample): + + t0 = time.time() + + print("--> Loading the processed predictions") + X = torch.load(f"{pred_path}/post_processed_Xs.pt") + X_f = torch.load(f"{pred_path}/post_processed_X_f.pt") + msk_X_f = torch.load(f"{pred_path}/post_processed_msk_X_f.pt") + yvals = torch.load(f"{pred_path}/post_processed_yvals.pt") + yvals_f = torch.load(f"{pred_path}/post_processed_yvals_f.pt") + print(f"Time taken to load the processed predictions is: {round(((time.time() - t0) / 60), 2)} min") + + print(f"--> Making plots using {len(X)} events...") + + # plot distributions + print("plot_dist...") + plot_dist(yvals_f, "pt", np.linspace(0, 200, 61), r"$p_T$", plot_path, sample) + plot_dist(yvals_f, "energy", np.linspace(0, 2000, 61), r"$E$", plot_path, sample) + plot_dist(yvals_f, "eta", np.linspace(-6, 6, 61), r"$\eta$", plot_path, sample) + + # plot cm + print("plot_cm...") + plot_cm(yvals_f, msk_X_f, "MLPF", plot_path) + plot_cm(yvals_f, msk_X_f, "PF", plot_path) + + # plot eff_and_fake_rate + print("plot_eff_and_fake_rate...") + plot_eff_and_fake_rate( + X_f, + yvals_f, + plot_path, + sample, + icls=1, + ivar=4, + ielem=1, + bins=np.logspace(-1, 3, 41), + log=True, + ) + plot_eff_and_fake_rate( + X_f, + yvals_f, + plot_path, + sample, + icls=1, + ivar=3, + ielem=1, + bins=np.linspace(-4, 4, 41), + log=False, + xlabel=r"PFElement $\eta$", + ) + plot_eff_and_fake_rate( + X_f, + yvals_f, + plot_path, + sample, + icls=2, + ivar=4, + ielem=5, + bins=np.logspace(-1, 3, 41), + log=True, + ) + plot_eff_and_fake_rate( + X_f, + yvals_f, + plot_path, + sample, + icls=2, + ivar=3, + ielem=5, + bins=np.linspace(-5, 5, 41), + log=False, + xlabel=r"PFElement $\eta$", + ) + plot_eff_and_fake_rate( + X_f, + yvals_f, + plot_path, + sample, + icls=5, + ivar=4, + ielem=4, + bins=np.logspace(-1, 2, 41), + log=True, + ) + plot_eff_and_fake_rate( + X_f, + yvals_f, + plot_path, + sample, + icls=5, + ivar=3, + ielem=4, + bins=np.linspace(-5, 5, 41), + log=False, + xlabel=r"PFElement $\eta$", + ) + + # distribution_icls + print("distribution_icls...") + distribution_icls(yvals_f, plot_path) + + print("plot_numPFelements...") + plot_numPFelements(X, plot_path, sample) + + plot_met(yvals, plot_path, sample) + plot_sum_energy(yvals, plot_path, sample) + plot_sum_pt(X, yvals, plot_path, sample) + print("plot_multiplicity...") + plot_multiplicity(yvals, plot_path, sample) + + # for energy resolution plotting purposes, initialize pid -> (ylim, bins) dictionary + print("plot_energy_res...") + dic = { + 1: (1e9, np.linspace(-2, 15, 100)), + 2: (1e7, np.linspace(-2, 15, 100)), + 3: (1e7, np.linspace(-2, 40, 100)), + 4: (1e7, np.linspace(-2, 30, 100)), + 5: (1e7, np.linspace(-2, 10, 100)), + 6: (1e4, np.linspace(-1, 1, 100)), + 7: (1e4, np.linspace(-0.1, 0.1, 100)), + } + for pid, tuple in dic.items(): + plot_energy_res(yvals_f, pid, tuple[1], tuple[0], plot_path, sample) + + # for eta resolution plotting purposes, initialize pid -> (ylim) dictionary + print("plot_eta_res...") + dic = {1: 1e10, 2: 1e8} + for pid, ylim in dic.items(): + plot_eta_res(yvals_f, pid, ylim, plot_path, sample) + + print(f"Time taken to make plots is: {round(((time.time() - t0) / 60), 2)} min") diff --git a/mlpf/pyg/cms_utils.py b/mlpf/pyg/cms/cms_utils.py similarity index 88% rename from mlpf/pyg/cms_utils.py rename to mlpf/pyg/cms/cms_utils.py index 3fdd263b3..f0d27d3cd 100644 --- a/mlpf/pyg/cms_utils.py +++ b/mlpf/pyg/cms/cms_utils.py @@ -28,11 +28,11 @@ # https://github.com/cms-sw/cmssw/blob/master/DataFormats/ParticleFlowCandidate/src/PFCandidate.cc#L254 CLASS_LABELS_CMS = [0, 211, 130, 1, 2, 22, 11, 13, 15] CLASS_NAMES_CMS_LATEX = [ - r"none", - r"chhad", - r"nhad", - r"HFEM", - r"HFHAD", + "none", + "chhad", + "nhad", + "HFEM", + "HFHAD", r"$\gamma$", r"$e^\pm$", r"$\mu^\pm$", @@ -71,7 +71,7 @@ "muon", ] -X_FEATURES = [ +X_FEATURES_CMS = [ "typ_idx", "pt", "eta", @@ -116,7 +116,6 @@ ] Y_FEATURES = [ - "typ_idx", "charge", "pt", "eta", @@ -127,13 +126,19 @@ def prepare_data_cms(fn): + """ + Takes as input a bz2 file that contains the cms raw information, and returns a list of PyG Data() objects. + Each element of the list looks like this ~ Data(x=[#, 41], ygen=[#, 6], ygen_id=[#, 9], ycand=[#, 6], ycand_id=[#, 9]) + + Args + raw_file_name: raw parquet data file. + Returns + list of Data() objects. + """ batched_data = [] - if fn.endswith(".pkl"): - data = pickle.load(open(fn, "rb"), encoding="iso-8859-1") - elif fn.endswith(".pkl.bz2"): - data = pickle.load(bz2.BZ2File(fn, "rb")) + data = pickle.load(bz2.BZ2File(fn, "rb")) for event in data: Xelem = event["Xelem"] @@ -173,7 +178,7 @@ def prepare_data_cms(fn): ) Xelem_flat = np.stack( - [Xelem[k].view(np.float32).data for k in X_FEATURES], + [Xelem[k].view(np.float32).data for k in X_FEATURES_CMS], axis=-1, ) ygen_flat = np.stack( diff --git a/mlpf/pyg/get_data_cms.sh b/mlpf/pyg/cms/get_data_cms.sh similarity index 71% rename from mlpf/pyg/get_data_cms.sh rename to mlpf/pyg/cms/get_data_cms.sh index cd057293b..97188a4af 100755 --- a/mlpf/pyg/get_data_cms.sh +++ b/mlpf/pyg/cms/get_data_cms.sh @@ -2,9 +2,6 @@ set -e -# make data/ directory to hold the cms/ directory of datafiles under particleflow/ -mkdir -p ../../data - # get the cms data rsync -r --progress lxplus.cern.ch:/eos/user/j/jpata/mlpf/cms . @@ -17,12 +14,19 @@ for sample in cms/* ; do cd ../../../ done -# process the cms data -for sample in cms/* ; do +# make data/ directory to hold the cms/ directory of datafiles under particleflow/ +mkdir -p ../../../data + +# move the cms/ directory of datafiles there +mv cms ../../../data/ +cd .. + +# process the raw datafiles +echo ----------------------- +for sample in ../../data/cms/* ; do echo $sample #generate pytorch data files from pkl files python3 PFGraphDataset.py --data cms --dataset $sample \ --processed_dir $sample/processed --num-files-merge 1 --num-proc 1 done - -mv cms ../../data/ +echo ----------------------- diff --git a/mlpf/pyg/delphes_plots.py b/mlpf/pyg/delphes/delphes_plots.py similarity index 50% rename from mlpf/pyg/delphes_plots.py rename to mlpf/pyg/delphes/delphes_plots.py index c3ac52398..144ee83d6 100644 --- a/mlpf/pyg/delphes_plots.py +++ b/mlpf/pyg/delphes/delphes_plots.py @@ -1,8 +1,11 @@ import itertools +import time import matplotlib.pyplot as plt import mplhep as hep import numpy as np +import sklearn +import torch plt.style.use(hep.style.ROOT) @@ -16,18 +19,6 @@ 5: "Muons", } -pid_to_name_cms = { - 0: "null", - 1: "chhadron", - 2: "nhadron", - 3: "HFHAD", - 4: "HFEM", - 5: "photon", - 6: "ele", - 7: "mu", - 8: "tau", -} - name_to_pid_delphes = { "null": 0, "chhadron": 1, @@ -37,18 +28,6 @@ "mu": 5, } -name_to_pid_cms = { - "null": 0, - "chhadron": 1, - "nhadron": 2, - "HFHAD": 3, - "HFEM": 4, - "photon": 5, - "ele": 6, - "mu": 7, - "tau": 8, -} - var_names = { "pt": r"$p_\mathrm{T}$ [GeV]", "eta": r"$\eta$", @@ -105,53 +84,24 @@ def divide_zero(a, b): return out -def plot_distribution( - data, - pid, - target, - mlpf, - var_name, - rng, - target_type, - fname, - legend_title="", -): +def plot_distribution(data, pid, target, mlpf, var_name, rng, target_type, fname, legend_title=""): """ plot distributions for the target and mlpf of a given feature for a given PID """ plt.style.use(hep.style.CMS) - if data == "delphes": - pid_to_name = pid_to_name_delphes - elif data == "cms": - pid_to_name = pid_to_name_cms - fig = plt.figure(figsize=(10, 10)) if target_type == "cand": - plt.hist( - target, - bins=rng, - density=True, - histtype="step", - lw=2, - label="cand", - ) + plt.hist(target, bins=rng, density=True, histtype="step", lw=2, label="cand") elif target_type == "gen": - plt.hist( - target, - bins=rng, - density=True, - histtype="step", - lw=2, - label="gen", - ) + plt.hist(target, bins=rng, density=True, histtype="step", lw=2, label="gen") plt.hist(mlpf, bins=rng, density=True, histtype="step", lw=2, label="MLPF") plt.xlabel(var_name) if pid != -1: - plt.legend(frameon=False, title=legend_title + pid_to_name[pid]) + plt.legend(frameon=False, title=legend_title + pid_to_name_delphes[pid]) else: plt.legend(frameon=False, title=legend_title) @@ -181,11 +131,6 @@ def plot_distributions_pid( """ plt.style.use("default") - if data == "delphes": - pid_to_name = pid_to_name_delphes - elif data == "cms": - pid_to_name = pid_to_name_cms - for i, bin_dict in enumerate(bins.items()): true = true_p4[true_id == pid, i].flatten().detach().cpu().numpy() pred = pred_p4[pred_id == pid, i].flatten().detach().cpu().numpy() @@ -197,7 +142,7 @@ def plot_distributions_pid( bin_dict[0], bin_dict[1], target, - fname=outpath + "/distribution_plots/" + pid_to_name[pid] + f"_{bin_dict[0]}_distribution", + fname=outpath + "/distribution_plots/" + pid_to_name_delphes[pid] + f"_{bin_dict[0]}_distribution", legend_title=legend_title, ) @@ -244,14 +189,7 @@ def plot_particle_multiplicity(data, list, key, ax=None, legend_title=""): """ plt.style.use(hep.style.ROOT) - if data == "delphes": - name_to_pid = name_to_pid_delphes - pid_to_name = pid_to_name_delphes - elif data == "cms": - name_to_pid = name_to_pid_cms - pid_to_name = pid_to_name_cms - - pid = name_to_pid[key] + pid = name_to_pid_delphes[key] if not ax: plt.figure(figsize=(4, 4)) ax = plt.axes() @@ -306,7 +244,7 @@ def plot_particle_multiplicity(data, list, key, ax=None, legend_title=""): ax.set_xlim(lims) ax.set_ylim(lims) plt.tight_layout() - ax.legend(frameon=False, title=legend_title + pid_to_name[pid]) + ax.legend(frameon=False, title=legend_title + pid_to_name_delphes[pid]) ax.set_xlabel("Truth particles / event") ax.set_ylabel("Reconstructed particles / event") plt.title("Particle multiplicity") @@ -324,10 +262,6 @@ def draw_efficiency_fakerate( both=True, legend_title="", ): - if data == "delphes": - pid_to_name = pid_to_name_delphes - elif data == "cms": - pid_to_name = pid_to_name_cms var_idx = var_indices[var] @@ -371,7 +305,7 @@ def draw_efficiency_fakerate( marker=".", markersize=10, ) - ax1.legend(frameon=False, loc=0, title=legend_title + pid_to_name[pid]) + ax1.legend(frameon=False, loc=0, title=legend_title + pid_to_name_delphes[pid]) ax1.set_ylim(0, 1.2) # if var=="energy": # ax1.set_xlim(0,30) @@ -411,7 +345,7 @@ def draw_efficiency_fakerate( marker=".", markersize=10, ) - ax2.legend(frameon=False, loc=0, title=legend_title + pid_to_name[pid]) + ax2.legend(frameon=False, loc=0, title=legend_title + pid_to_name_delphes[pid]) ax2.set_ylim(0, 1.0) # plt.yscale("log") ax2.set_xlabel(var_names[var]) @@ -426,11 +360,7 @@ def draw_efficiency_fakerate( def plot_reso(data, ygen, ypred, ycand, pfcand, var, outpath, legend_title=""): plt.style.use(hep.style.ROOT) - if data == "delphes": - name_to_pid = name_to_pid_delphes - elif data == "cms": - name_to_pid = name_to_pid_cms - pid = name_to_pid[pfcand] + pid = name_to_pid_delphes[pfcand] var_idx = var_indices[var] msk = (ygen[:, 0] == pid) & (ycand[:, 0] == pid) @@ -590,3 +520,401 @@ def plot_confusion_matrix( # torch.save(cm, outpath + save_as + '.pt') return fig, ax + + +def make_plots_delphes(model, test_loader, outpath, target, device, epoch, tag): + + print("Making plots...") + t0 = time.time() + + # load the necessary predictions to make the plots + gen_ids = torch.load(outpath + "/gen_ids.pt", map_location=device) + gen_p4 = torch.load(outpath + "/gen_p4.pt", map_location=device) + pred_ids = torch.load(outpath + "/pred_ids.pt", map_location=device) + pred_p4 = torch.load(outpath + "/pred_p4.pt", map_location=device) + cand_ids = torch.load(outpath + "/cand_ids.pt", map_location=device) + cand_p4 = torch.load(outpath + "/cand_p4.pt", map_location=device) + + list_for_multiplicities = torch.load(outpath + "/list_for_multiplicities.pt", map_location=device) + + predictions = torch.load(outpath + "/predictions.pt", map_location=device) + + # reformat a bit + ygen = predictions["ygen"].reshape(-1, 7) + ypred = predictions["ypred"].reshape(-1, 7) + ycand = predictions["ycand"].reshape(-1, 7) + + # make confusion matrix for MLPF + target_names = ["none", "ch.had", "n.had", "g", "el", "mu"] + conf_matrix_mlpf = sklearn.metrics.confusion_matrix(gen_ids.cpu(), pred_ids.cpu(), labels=range(6), normalize="true") + + plot_confusion_matrix( + conf_matrix_mlpf, + target_names, + epoch, + outpath + "/confusion_matrix_plots/", + f"cm_mlpf_epoch_{str(epoch)}", + ) + + # make confusion matrix for rule based PF + conf_matrix_cand = sklearn.metrics.confusion_matrix(gen_ids.cpu(), cand_ids.cpu(), labels=range(6), normalize="true") + + plot_confusion_matrix( + conf_matrix_cand, + target_names, + epoch, + outpath + "/confusion_matrix_plots/", + "cm_cand", + target="rule-based", + ) + + # making all the other plots + if "QCD" in tag: + sample = "QCD, 14 TeV, PU200" + else: + sample = "$t\\bar{t}$, 14 TeV, PU200" + + # make distribution plots + plot_distributions_pid( + 1, + gen_ids, + gen_p4, + pred_ids, + pred_p4, + cand_ids, + cand_p4, # distribution plots for chhadrons + target, + epoch, + outpath, + legend_title=sample + "\n", + ) + plot_distributions_pid( + 2, + gen_ids, + gen_p4, + pred_ids, + pred_p4, + cand_ids, + cand_p4, # distribution plots for nhadrons + target, + epoch, + outpath, + legend_title=sample + "\n", + ) + plot_distributions_pid( + 3, + gen_ids, + gen_p4, + pred_ids, + pred_p4, + cand_ids, + cand_p4, # distribution plots for photons + target, + epoch, + outpath, + legend_title=sample + "\n", + ) + plot_distributions_pid( + 4, + gen_ids, + gen_p4, + pred_ids, + pred_p4, + cand_ids, + cand_p4, # distribution plots for electrons + target, + epoch, + outpath, + legend_title=sample + "\n", + ) + plot_distributions_pid( + 5, + gen_ids, + gen_p4, + pred_ids, + pred_p4, + cand_ids, + cand_p4, # distribution plots for muons + target, + epoch, + outpath, + legend_title=sample + "\n", + ) + + plot_distributions_all( + gen_ids, + gen_p4, + pred_ids, + pred_p4, + cand_ids, + cand_p4, # distribution plots for all together + target, + epoch, + outpath, + legend_title=sample + "\n", + ) + + # plot particle multiplicity plots + fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) + plot_particle_multiplicity(list_for_multiplicities, "null", ax) + plt.savefig(outpath + "/multiplicity_plots/num_null.png", bbox_inches="tight") + plt.savefig(outpath + "/multiplicity_plots/num_null.pdf", bbox_inches="tight") + plt.close(fig) + + fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) + plot_particle_multiplicity(list_for_multiplicities, "chhadron", ax) + plt.savefig(outpath + "/multiplicity_plots/num_chhadron.png", bbox_inches="tight") + plt.savefig(outpath + "/multiplicity_plots/num_chhadron.pdf", bbox_inches="tight") + plt.close(fig) + + fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) + plot_particle_multiplicity(list_for_multiplicities, "nhadron", ax) + plt.savefig(outpath + "/multiplicity_plots/num_nhadron.png", bbox_inches="tight") + plt.savefig(outpath + "/multiplicity_plots/num_nhadron.pdf", bbox_inches="tight") + plt.close(fig) + + fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) + plot_particle_multiplicity(list_for_multiplicities, "photon", ax) + plt.savefig(outpath + "/multiplicity_plots/num_photon.png", bbox_inches="tight") + plt.savefig(outpath + "/multiplicity_plots/num_photon.pdf", bbox_inches="tight") + plt.close(fig) + + fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) + plot_particle_multiplicity(list_for_multiplicities, "electron", ax) + plt.savefig(outpath + "/multiplicity_plots/num_electron.png", bbox_inches="tight") + plt.savefig(outpath + "/multiplicity_plots/num_electron.pdf", bbox_inches="tight") + plt.close(fig) + + fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) + plot_particle_multiplicity(list_for_multiplicities, "muon", ax) + plt.savefig(outpath + "/multiplicity_plots/num_muon.png", bbox_inches="tight") + plt.savefig(outpath + "/multiplicity_plots/num_muon.pdf", bbox_inches="tight") + plt.close(fig) + + # make efficiency and fake rate plots for charged hadrons + ax, _ = draw_efficiency_fakerate( + ygen, + ypred, + ycand, + 1, + "pt", + np.linspace(0, 3, 61), + outpath + "/efficiency_plots/eff_fake_pid1_pt.png", + both=True, + legend_title=sample + "\n", + ) + ax, _ = draw_efficiency_fakerate( + ygen, + ypred, + ycand, + 1, + "eta", + np.linspace(-3, 3, 61), + outpath + "/efficiency_plots/eff_fake_pid1_eta.png", + both=True, + legend_title=sample + "\n", + ) + ax, _ = draw_efficiency_fakerate( + ygen, + ypred, + ycand, + 1, + "energy", + np.linspace(0, 50, 75), + outpath + "/efficiency_plots/eff_fake_pid1_energy.png", + both=True, + legend_title=sample + "\n", + ) + + # make efficiency and fake rate plots for neutral hadrons + ax, _ = draw_efficiency_fakerate( + ygen, + ypred, + ycand, + 2, + "pt", + np.linspace(0, 3, 61), + outpath + "/efficiency_plots/eff_fake_pid2_pt.png", + both=True, + legend_title=sample + "\n", + ) + ax, _ = draw_efficiency_fakerate( + ygen, + ypred, + ycand, + 2, + "eta", + np.linspace(-3, 3, 61), + outpath + "/efficiency_plots/eff_fake_pid2_eta.png", + both=True, + legend_title=sample + "\n", + ) + ax, _ = draw_efficiency_fakerate( + ygen, + ypred, + ycand, + 2, + "energy", + np.linspace(0, 50, 75), + outpath + "/efficiency_plots/eff_fake_pid2_energy.png", + both=True, + legend_title=sample + "\n", + ) + + # make resolution plots for chhadrons: pid=1 + fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 1, "pt", 2, ax=ax1, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid1_pt.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid1_pt.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 1, "eta", 0.2, ax=ax2, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid1_eta.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid1_eta.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso( + ygen, + ypred, + ycand, + 1, + "energy", + 0.2, + ax=ax3, + legend_title=sample + "\n", + ) + plt.savefig(outpath + "/resolution_plots/res_pid1_energy.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid1_energy.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + # make resolution plots for nhadrons: pid=2 + fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 2, "pt", 2, ax=ax1, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid2_pt.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid2_pt.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 2, "eta", 0.2, ax=ax2, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid2_eta.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid2_eta.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso( + ygen, + ypred, + ycand, + 2, + "energy", + 0.2, + ax=ax3, + legend_title=sample + "\n", + ) + plt.savefig(outpath + "/resolution_plots/res_pid2_energy.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid2_energy.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + # make resolution plots for photons: pid=3 + fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 3, "pt", 2, ax=ax1, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid3_pt.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid3_pt.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 3, "eta", 0.2, ax=ax2, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid3_eta.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid3_eta.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso( + ygen, + ypred, + ycand, + 3, + "energy", + 0.2, + ax=ax3, + legend_title=sample + "\n", + ) + plt.savefig(outpath + "/resolution_plots/res_pid3_energy.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid3_energy.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + # make resolution plots for electrons: pid=4 + fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 4, "pt", 2, ax=ax1, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid4_pt.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid4_pt.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 4, "eta", 0.2, ax=ax2, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid4_eta.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid4_eta.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso( + ygen, + ypred, + ycand, + 4, + "energy", + 0.2, + ax=ax3, + legend_title=sample + "\n", + ) + plt.savefig(outpath + "/resolution_plots/res_pid4_energy.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid4_energy.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + # make resolution plots for muons: pid=5 + fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 5, "pt", 2, ax=ax1, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid5_pt.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid5_pt.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso(ygen, ypred, ycand, 5, "eta", 0.2, ax=ax2, legend_title=sample + "\n") + plt.savefig(outpath + "/resolution_plots/res_pid5_eta.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid5_eta.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) + plot_reso( + ygen, + ypred, + ycand, + 5, + "energy", + 0.2, + ax=ax3, + legend_title=sample + "\n", + ) + plt.savefig(outpath + "/resolution_plots/res_pid5_energy.png", bbox_inches="tight") + plt.savefig(outpath + "/resolution_plots/res_pid5_energy.pdf", bbox_inches="tight") + plt.tight_layout() + plt.close(fig) + + t1 = time.time() + print("Time taken to make plots is:", round(((t1 - t0) / 60), 2), "min") diff --git a/mlpf/pyg/delphes/delphes_utils.py b/mlpf/pyg/delphes/delphes_utils.py new file mode 100644 index 000000000..ab6b9ba6c --- /dev/null +++ b/mlpf/pyg/delphes/delphes_utils.py @@ -0,0 +1,53 @@ +import pickle + +import matplotlib +import torch +from torch_geometric.data import Data + +matplotlib.use("Agg") + + +X_FEATURES_DELPHES = [ + "Track|cluster", + "$p_{T}|E_{T}$", + r"$\eta$", + r"$Sin(\phi)$", + r"$Cos(\phi)$", + "P|E", + r"$\eta_\mathrm{out}|E_{em}$", + r"$Sin(\(phi)_\mathrm{out}|E_{had}$", + r"$Cos(\phi)_\mathrm{out}|E_{had}$", + "charge", + "is_gen_mu", + "is_gen_el", +] + + +def prepare_data_delphes(fn): + + """ + Takes as input a pkl file that contains the delphes raw information, and returns a list of PyG Data() objects. + Each element of the list looks like this ~ Data(x=[#, 12], ygen=[#, 6], ygen_id=[#, 6], ycand=[#, 6], ycand_id=[#, 6]) + + Args + raw_file_name: raw parquet data file. + Returns + list of Data() objects. + """ + + with open(fn, "rb") as fi: + data = pickle.load(fi, encoding="iso-8859-1") + + batched_data = [] + for i in range(len(data["X"])): + # remove from ygen & ycand the first element (PID) so that they only contain the regression variables + d = Data( + x=torch.tensor(data["X"][i], dtype=torch.float), + ygen=torch.tensor(data["ygen"][i], dtype=torch.float)[:, 1:], + ygen_id=torch.tensor(data["ygen"][i], dtype=torch.float)[:, 0].long(), + ycand=torch.tensor(data["ycand"][i], dtype=torch.float)[:, 1:], + ycand_id=torch.tensor(data["ycand"][i], dtype=torch.float)[:, 0].long(), + ) + + batched_data.append(d) + return batched_data diff --git a/mlpf/pyg/get_data_delphes.sh b/mlpf/pyg/delphes/get_data_delphes.sh similarity index 63% rename from mlpf/pyg/get_data_delphes.sh rename to mlpf/pyg/delphes/get_data_delphes.sh index e8fb39f37..e851ddb73 100755 --- a/mlpf/pyg/get_data_delphes.sh +++ b/mlpf/pyg/delphes/get_data_delphes.sh @@ -2,9 +2,6 @@ set -e -# make data/ directory to hold the delphes/ directory of datafiles under particleflow/ -mkdir -p ../../data - # make delphes directories mkdir -p delphes/pythia8_ttbar/raw mkdir -p delphes/pythia8_ttbar/processed @@ -30,16 +27,19 @@ do wget --no-check-certificate -nc https://zenodo.org/record/4559324/files/tev14_pythia8_qcd_10_"$i".pkl.bz2 done bzip2 -d * +cd ../../../ -# get back in the pytorch directory -cd ../../../../ +# make data/ directory to hold the delphes/ directory of datafiles under particleflow/ +mkdir -p ../../../data -#generate pytorch data files from pkl files -python3 PFGraphDataset.py --data delphes --dataset delphes/pythia8_ttbar \ - --processed_dir delphes/pythia8_ttbar/processed --num-files-merge 1 --num-proc 1 +# move the delphes/ directory of datafiles there +mv delphes ../../../data/ +cd .. #generate pytorch data files from pkl files -python3 PFGraphDataset.py --data delphes --dataset delphes/pythia8_qcd \ - --processed_dir delphes/pythia8_qcd/processed --num-files-merge 1 --num-proc 1 +python3 PFGraphDataset.py --data delphes --dataset ../../data/delphes/pythia8_ttbar \ + --processed_dir ../../data/delphes/pythia8_ttbar/processed --num-files-merge 1 --num-proc 1 -mv delphes ../../data/ +#generate pytorch data files from pkl files +python3 PFGraphDataset.py --data delphes --dataset ../../data/delphes/pythia8_qcd \ + --processed_dir ../../data/delphes/pythia8_qcd/processed --num-files-merge 1 --num-proc 1 diff --git a/mlpf/pyg/delphes_utils.py b/mlpf/pyg/delphes_utils.py deleted file mode 100644 index a1fe2f499..000000000 --- a/mlpf/pyg/delphes_utils.py +++ /dev/null @@ -1,657 +0,0 @@ -import time - -import matplotlib -import matplotlib.pyplot as plt -import numpy as np -import sklearn -import torch -from pyg.delphes_plots import ( - draw_efficiency_fakerate, - plot_confusion_matrix, - plot_distributions_all, - plot_distributions_pid, - plot_particle_multiplicity, - plot_reso, -) -from pyg.utils import one_hot_embedding - -matplotlib.use("Agg") - - -def make_predictions_delphes(model, multi_gpu, test_loader, outpath, device, epoch, num_classes): - - print("Making predictions...") - t0 = time.time() - - gen_list = { - "null": [], - "chhadron": [], - "nhadron": [], - "photon": [], - "electron": [], - "muon": [], - } - pred_list = { - "null": [], - "chhadron": [], - "nhadron": [], - "photon": [], - "electron": [], - "muon": [], - } - cand_list = { - "null": [], - "chhadron": [], - "nhadron": [], - "photon": [], - "electron": [], - "muon": [], - } - - t = [] - - for i, batch in enumerate(test_loader): - if multi_gpu: - X = batch # a list (not torch) instance so can't be passed to device - else: - X = batch.to(device) - - ti = time.time() - - pred_ids_one_hot, pred_p4 = model(X) - - gen_p4 = X.ygen.detach().to("cpu") - cand_ids_one_hot = one_hot_embedding(X.ycand_id.detach().to("cpu"), num_classes) - gen_ids_one_hot = one_hot_embedding(X.ygen_id.detach().to("cpu"), num_classes) - cand_p4 = X.ycand.detach().to("cpu") - - tf = time.time() - if i != 0: - t.append(round((tf - ti), 2)) - - _, gen_ids = torch.max(gen_ids_one_hot.detach(), -1) - _, pred_ids = torch.max(pred_ids_one_hot.detach(), -1) - _, cand_ids = torch.max(cand_ids_one_hot.detach(), -1) - - # to make "num_gen vs num_pred" plots - gen_list["null"].append((gen_ids == 0).sum().item()) - gen_list["chhadron"].append((gen_ids == 1).sum().item()) - gen_list["nhadron"].append((gen_ids == 2).sum().item()) - gen_list["photon"].append((gen_ids == 3).sum().item()) - gen_list["electron"].append((gen_ids == 4).sum().item()) - gen_list["muon"].append((gen_ids == 5).sum().item()) - - pred_list["null"].append((pred_ids == 0).sum().item()) - pred_list["chhadron"].append((pred_ids == 1).sum().item()) - pred_list["nhadron"].append((pred_ids == 2).sum().item()) - pred_list["photon"].append((pred_ids == 3).sum().item()) - pred_list["electron"].append((pred_ids == 4).sum().item()) - pred_list["muon"].append((pred_ids == 5).sum().item()) - - cand_list["null"].append((cand_ids == 0).sum().item()) - cand_list["chhadron"].append((cand_ids == 1).sum().item()) - cand_list["nhadron"].append((cand_ids == 2).sum().item()) - cand_list["photon"].append((cand_ids == 3).sum().item()) - cand_list["electron"].append((cand_ids == 4).sum().item()) - cand_list["muon"].append((cand_ids == 5).sum().item()) - - gen_p4 = gen_p4.detach() - pred_p4 = pred_p4.detach() - cand_p4 = cand_p4.detach() - - if i == 0: - gen_ids_all = gen_ids - gen_p4_all = gen_p4 - - pred_ids_all = pred_ids - pred_p4_all = pred_p4 - - cand_ids_all = cand_ids - cand_p4_all = cand_p4 - else: - gen_ids_all = torch.cat([gen_ids_all, gen_ids]) - gen_p4_all = torch.cat([gen_p4_all, gen_p4]) - - pred_ids_all = torch.cat([pred_ids_all, pred_ids]) - pred_p4_all = torch.cat([pred_p4_all, pred_p4]) - - cand_ids_all = torch.cat([cand_ids_all, cand_ids]) - cand_p4_all = torch.cat([cand_p4_all, cand_p4]) - - if len(test_loader) < 5000: - print(f"event #: {i+1}/{len(test_loader)}") - else: - print(f"event #: {i+1}/{5000}") - - if i == 4999: - break - - print( - "Average Inference time per event is: ", - round((sum(t) / len(t)), 2), - "s", - ) - - t1 = time.time() - - print( - "Time taken to make predictions is:", - round(((t1 - t0) / 60), 2), - "min", - ) - - # store the 3 list dictionaries in a list (this is done only to compute the particle multiplicity plots) - list = [pred_list, gen_list, cand_list] - - torch.save(list, outpath + "/list_for_multiplicities.pt") - - torch.save(gen_ids_all, outpath + "/gen_ids.pt") - torch.save(gen_p4_all, outpath + "/gen_p4.pt") - torch.save(pred_ids_all, outpath + "/pred_ids.pt") - torch.save(pred_p4_all, outpath + "/pred_p4.pt") - torch.save(cand_ids_all, outpath + "/cand_ids.pt") - torch.save(cand_p4_all, outpath + "/cand_p4.pt") - - ygen = torch.cat([gen_ids_all.reshape(-1, 1).float(), gen_p4_all], axis=1) - ypred = torch.cat([pred_ids_all.reshape(-1, 1).float(), pred_p4_all], axis=1) - ycand = torch.cat([cand_ids_all.reshape(-1, 1).float(), cand_p4_all], axis=1) - - # store the actual predictions to make all the other plots - predictions = { - "ygen": ygen.reshape(1, -1, 7).detach().cpu().numpy(), - "ycand": ycand.reshape(1, -1, 7).detach().cpu().numpy(), - "ypred": ypred.detach().reshape(1, -1, 7).cpu().numpy(), - } - - torch.save(predictions, outpath + "/predictions.pt") - - -def make_plots_delphes(model, test_loader, outpath, target, device, epoch, tag): - - print("Making plots...") - t0 = time.time() - - # load the necessary predictions to make the plots - gen_ids = torch.load(outpath + "/gen_ids.pt", map_location=device) - gen_p4 = torch.load(outpath + "/gen_p4.pt", map_location=device) - pred_ids = torch.load(outpath + "/pred_ids.pt", map_location=device) - pred_p4 = torch.load(outpath + "/pred_p4.pt", map_location=device) - cand_ids = torch.load(outpath + "/cand_ids.pt", map_location=device) - cand_p4 = torch.load(outpath + "/cand_p4.pt", map_location=device) - - list_for_multiplicities = torch.load(outpath + "/list_for_multiplicities.pt", map_location=device) - - predictions = torch.load(outpath + "/predictions.pt", map_location=device) - - # reformat a bit - ygen = predictions["ygen"].reshape(-1, 7) - ypred = predictions["ypred"].reshape(-1, 7) - ycand = predictions["ycand"].reshape(-1, 7) - - # make confusion matrix for MLPF - target_names = ["none", "ch.had", "n.had", "g", "el", "mu"] - conf_matrix_mlpf = sklearn.metrics.confusion_matrix(gen_ids.cpu(), pred_ids.cpu(), labels=range(6), normalize="true") - - plot_confusion_matrix( - conf_matrix_mlpf, - target_names, - epoch, - outpath + "/confusion_matrix_plots/", - f"cm_mlpf_epoch_{str(epoch)}", - ) - - # make confusion matrix for rule based PF - conf_matrix_cand = sklearn.metrics.confusion_matrix(gen_ids.cpu(), cand_ids.cpu(), labels=range(6), normalize="true") - - plot_confusion_matrix( - conf_matrix_cand, - target_names, - epoch, - outpath + "/confusion_matrix_plots/", - "cm_cand", - target="rule-based", - ) - - # making all the other plots - if "QCD" in tag: - sample = "QCD, 14 TeV, PU200" - else: - sample = "$t\\bar{t}$, 14 TeV, PU200" - - # make distribution plots - plot_distributions_pid( - 1, - gen_ids, - gen_p4, - pred_ids, - pred_p4, - cand_ids, - cand_p4, # distribution plots for chhadrons - target, - epoch, - outpath, - legend_title=sample + "\n", - ) - plot_distributions_pid( - 2, - gen_ids, - gen_p4, - pred_ids, - pred_p4, - cand_ids, - cand_p4, # distribution plots for nhadrons - target, - epoch, - outpath, - legend_title=sample + "\n", - ) - plot_distributions_pid( - 3, - gen_ids, - gen_p4, - pred_ids, - pred_p4, - cand_ids, - cand_p4, # distribution plots for photons - target, - epoch, - outpath, - legend_title=sample + "\n", - ) - plot_distributions_pid( - 4, - gen_ids, - gen_p4, - pred_ids, - pred_p4, - cand_ids, - cand_p4, # distribution plots for electrons - target, - epoch, - outpath, - legend_title=sample + "\n", - ) - plot_distributions_pid( - 5, - gen_ids, - gen_p4, - pred_ids, - pred_p4, - cand_ids, - cand_p4, # distribution plots for muons - target, - epoch, - outpath, - legend_title=sample + "\n", - ) - - plot_distributions_all( - gen_ids, - gen_p4, - pred_ids, - pred_p4, - cand_ids, - cand_p4, # distribution plots for all together - target, - epoch, - outpath, - legend_title=sample + "\n", - ) - - # plot particle multiplicity plots - fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) - plot_particle_multiplicity(list_for_multiplicities, "null", ax) - plt.savefig(outpath + "/multiplicity_plots/num_null.png", bbox_inches="tight") - plt.savefig(outpath + "/multiplicity_plots/num_null.pdf", bbox_inches="tight") - plt.close(fig) - - fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) - plot_particle_multiplicity(list_for_multiplicities, "chhadron", ax) - plt.savefig( - outpath + "/multiplicity_plots/num_chhadron.png", - bbox_inches="tight", - ) - plt.savefig( - outpath + "/multiplicity_plots/num_chhadron.pdf", - bbox_inches="tight", - ) - plt.close(fig) - - fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) - plot_particle_multiplicity(list_for_multiplicities, "nhadron", ax) - plt.savefig( - outpath + "/multiplicity_plots/num_nhadron.png", - bbox_inches="tight", - ) - plt.savefig( - outpath + "/multiplicity_plots/num_nhadron.pdf", - bbox_inches="tight", - ) - plt.close(fig) - - fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) - plot_particle_multiplicity(list_for_multiplicities, "photon", ax) - plt.savefig(outpath + "/multiplicity_plots/num_photon.png", bbox_inches="tight") - plt.savefig(outpath + "/multiplicity_plots/num_photon.pdf", bbox_inches="tight") - plt.close(fig) - - fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) - plot_particle_multiplicity(list_for_multiplicities, "electron", ax) - plt.savefig( - outpath + "/multiplicity_plots/num_electron.png", - bbox_inches="tight", - ) - plt.savefig( - outpath + "/multiplicity_plots/num_electron.pdf", - bbox_inches="tight", - ) - plt.close(fig) - - fig, ax = plt.subplots(1, 1, figsize=(8, 2 * 8)) - plot_particle_multiplicity(list_for_multiplicities, "muon", ax) - plt.savefig(outpath + "/multiplicity_plots/num_muon.png", bbox_inches="tight") - plt.savefig(outpath + "/multiplicity_plots/num_muon.pdf", bbox_inches="tight") - plt.close(fig) - - # make efficiency and fake rate plots for charged hadrons - ax, _ = draw_efficiency_fakerate( - ygen, - ypred, - ycand, - 1, - "pt", - np.linspace(0, 3, 61), - outpath + "/efficiency_plots/eff_fake_pid1_pt.png", - both=True, - legend_title=sample + "\n", - ) - ax, _ = draw_efficiency_fakerate( - ygen, - ypred, - ycand, - 1, - "eta", - np.linspace(-3, 3, 61), - outpath + "/efficiency_plots/eff_fake_pid1_eta.png", - both=True, - legend_title=sample + "\n", - ) - ax, _ = draw_efficiency_fakerate( - ygen, - ypred, - ycand, - 1, - "energy", - np.linspace(0, 50, 75), - outpath + "/efficiency_plots/eff_fake_pid1_energy.png", - both=True, - legend_title=sample + "\n", - ) - - # make efficiency and fake rate plots for neutral hadrons - ax, _ = draw_efficiency_fakerate( - ygen, - ypred, - ycand, - 2, - "pt", - np.linspace(0, 3, 61), - outpath + "/efficiency_plots/eff_fake_pid2_pt.png", - both=True, - legend_title=sample + "\n", - ) - ax, _ = draw_efficiency_fakerate( - ygen, - ypred, - ycand, - 2, - "eta", - np.linspace(-3, 3, 61), - outpath + "/efficiency_plots/eff_fake_pid2_eta.png", - both=True, - legend_title=sample + "\n", - ) - ax, _ = draw_efficiency_fakerate( - ygen, - ypred, - ycand, - 2, - "energy", - np.linspace(0, 50, 75), - outpath + "/efficiency_plots/eff_fake_pid2_energy.png", - both=True, - legend_title=sample + "\n", - ) - - # make resolution plots for chhadrons: pid=1 - fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso(ygen, ypred, ycand, 1, "pt", 2, ax=ax1, legend_title=sample + "\n") - plt.savefig(outpath + "/resolution_plots/res_pid1_pt.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid1_pt.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 1, - "eta", - 0.2, - ax=ax2, - legend_title=sample + "\n", - ) - plt.savefig(outpath + "/resolution_plots/res_pid1_eta.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid1_eta.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 1, - "energy", - 0.2, - ax=ax3, - legend_title=sample + "\n", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid1_energy.png", - bbox_inches="tight", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid1_energy.pdf", - bbox_inches="tight", - ) - plt.tight_layout() - plt.close(fig) - - # make resolution plots for nhadrons: pid=2 - fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso(ygen, ypred, ycand, 2, "pt", 2, ax=ax1, legend_title=sample + "\n") - plt.savefig(outpath + "/resolution_plots/res_pid2_pt.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid2_pt.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 2, - "eta", - 0.2, - ax=ax2, - legend_title=sample + "\n", - ) - plt.savefig(outpath + "/resolution_plots/res_pid2_eta.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid2_eta.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 2, - "energy", - 0.2, - ax=ax3, - legend_title=sample + "\n", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid2_energy.png", - bbox_inches="tight", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid2_energy.pdf", - bbox_inches="tight", - ) - plt.tight_layout() - plt.close(fig) - - # make resolution plots for photons: pid=3 - fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso(ygen, ypred, ycand, 3, "pt", 2, ax=ax1, legend_title=sample + "\n") - plt.savefig(outpath + "/resolution_plots/res_pid3_pt.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid3_pt.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 3, - "eta", - 0.2, - ax=ax2, - legend_title=sample + "\n", - ) - plt.savefig(outpath + "/resolution_plots/res_pid3_eta.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid3_eta.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 3, - "energy", - 0.2, - ax=ax3, - legend_title=sample + "\n", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid3_energy.png", - bbox_inches="tight", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid3_energy.pdf", - bbox_inches="tight", - ) - plt.tight_layout() - plt.close(fig) - - # make resolution plots for electrons: pid=4 - fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso(ygen, ypred, ycand, 4, "pt", 2, ax=ax1, legend_title=sample + "\n") - plt.savefig(outpath + "/resolution_plots/res_pid4_pt.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid4_pt.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 4, - "eta", - 0.2, - ax=ax2, - legend_title=sample + "\n", - ) - plt.savefig(outpath + "/resolution_plots/res_pid4_eta.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid4_eta.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 4, - "energy", - 0.2, - ax=ax3, - legend_title=sample + "\n", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid4_energy.png", - bbox_inches="tight", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid4_energy.pdf", - bbox_inches="tight", - ) - plt.tight_layout() - plt.close(fig) - - # make resolution plots for muons: pid=5 - fig, (ax1) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso(ygen, ypred, ycand, 5, "pt", 2, ax=ax1, legend_title=sample + "\n") - plt.savefig(outpath + "/resolution_plots/res_pid5_pt.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid5_pt.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax2) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 5, - "eta", - 0.2, - ax=ax2, - legend_title=sample + "\n", - ) - plt.savefig(outpath + "/resolution_plots/res_pid5_eta.png", bbox_inches="tight") - plt.savefig(outpath + "/resolution_plots/res_pid5_eta.pdf", bbox_inches="tight") - plt.tight_layout() - plt.close(fig) - - fig, (ax3) = plt.subplots(1, 1, figsize=(8, 8)) - plot_reso( - ygen, - ypred, - ycand, - 5, - "energy", - 0.2, - ax=ax3, - legend_title=sample + "\n", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid5_energy.png", - bbox_inches="tight", - ) - plt.savefig( - outpath + "/resolution_plots/res_pid5_energy.pdf", - bbox_inches="tight", - ) - plt.tight_layout() - plt.close(fig) - - t1 = time.time() - print("Time taken to make plots is:", round(((t1 - t0) / 60), 2), "min") diff --git a/mlpf/pyg/evaluate.py b/mlpf/pyg/evaluate.py index 39de10fdb..2fdb35e02 100644 --- a/mlpf/pyg/evaluate.py +++ b/mlpf/pyg/evaluate.py @@ -5,24 +5,26 @@ import numpy as np import torch import torch_geometric -from pyg.cms_plots import ( - distribution_icls, - plot_cm, - plot_dist, - plot_eff_and_fake_rate, - plot_energy_res, - plot_eta_res, - plot_met, - plot_multiplicity, - plot_numPFelements, - plot_sum_energy, - plot_sum_pt, -) -from pyg.utils import one_hot_embedding, target_p4 + +from .utils import Y_FEATURES matplotlib.use("Agg") +def one_hot_embedding(labels, num_classes): + """ + Embedding labels to one-hot form. + + Args: + labels: (LongTensor) class labels, sized [N,]. + num_classes: (int) number of classes. + Returns: + (tensor) encoded labels, sized [N, #classes]. + """ + y = torch.eye(num_classes) + return y[labels] + + def make_predictions(rank, model, file_loader, batch_size, num_classes, PATH): """ Runs inference on the qcd test dataset to evaluate performance. @@ -176,7 +178,7 @@ def postprocess_predictions(pred_path): yvals["cand_cls"] = Y_pids[:, 1, :, :].numpy() yvals["pred_cls"] = Y_pids[:, 2, :, :].numpy() - for feat, key in enumerate(target_p4): + for feat, key in enumerate(Y_FEATURES): yvals[f"gen_{key}"] = Y_p4s[:, 0, :, feat].unsqueeze(-1).numpy() yvals[f"cand_{key}"] = Y_p4s[:, 1, :, feat].unsqueeze(-1).numpy() yvals[f"pred_{key}"] = Y_p4s[:, 2, :, feat].unsqueeze(-1).numpy() @@ -215,166 +217,9 @@ def flatten(arr): t0 = time.time() torch.save(Xs, f"{pred_path}/post_processed_Xs.pt", pickle_protocol=4) torch.save(X_f, f"{pred_path}/post_processed_X_f.pt", pickle_protocol=4) - torch.save( - msk_X_f, - f"{pred_path}/post_processed_msk_X_f.pt", - pickle_protocol=4, - ) + torch.save(msk_X_f, f"{pred_path}/post_processed_msk_X_f.pt", pickle_protocol=4) torch.save(yvals, f"{pred_path}/post_processed_yvals.pt", pickle_protocol=4) - torch.save( - yvals_f, - f"{pred_path}/post_processed_yvals_f.pt", - pickle_protocol=4, - ) + torch.save(yvals_f, f"{pred_path}/post_processed_yvals_f.pt", pickle_protocol=4) print(f"Time taken to save the predictions is: {round(((time.time() - t0) / 60), 2)} min") return Xs, X_f, msk_X_f, yvals, yvals_f - - -def make_plots_cms(pred_path, plot_path, sample): - - t0 = time.time() - - print("--> Loading the processed predictions") - X = torch.load(f"{pred_path}/post_processed_Xs.pt") - X_f = torch.load(f"{pred_path}/post_processed_X_f.pt") - msk_X_f = torch.load(f"{pred_path}/post_processed_msk_X_f.pt") - yvals = torch.load(f"{pred_path}/post_processed_yvals.pt") - yvals_f = torch.load(f"{pred_path}/post_processed_yvals_f.pt") - print(f"Time taken to load the processed predictions is: {round(((time.time() - t0) / 60), 2)} min") - - print(f"--> Making plots using {len(X)} events...") - - # plot distributions - print("plot_dist...") - plot_dist(yvals_f, "pt", np.linspace(0, 200, 61), r"$p_T$", plot_path, sample) - plot_dist( - yvals_f, - "energy", - np.linspace(0, 2000, 61), - r"$E$", - plot_path, - sample, - ) - plot_dist( - yvals_f, - "eta", - np.linspace(-6, 6, 61), - r"$\eta$", - plot_path, - sample, - ) - - # plot cm - print("plot_cm...") - plot_cm(yvals_f, msk_X_f, "MLPF", plot_path) - plot_cm(yvals_f, msk_X_f, "PF", plot_path) - - # plot eff_and_fake_rate - print("plot_eff_and_fake_rate...") - plot_eff_and_fake_rate( - X_f, - yvals_f, - plot_path, - sample, - icls=1, - ivar=4, - ielem=1, - bins=np.logspace(-1, 3, 41), - log=True, - ) - plot_eff_and_fake_rate( - X_f, - yvals_f, - plot_path, - sample, - icls=1, - ivar=3, - ielem=1, - bins=np.linspace(-4, 4, 41), - log=False, - xlabel=r"PFElement $\eta$", - ) - plot_eff_and_fake_rate( - X_f, - yvals_f, - plot_path, - sample, - icls=2, - ivar=4, - ielem=5, - bins=np.logspace(-1, 3, 41), - log=True, - ) - plot_eff_and_fake_rate( - X_f, - yvals_f, - plot_path, - sample, - icls=2, - ivar=3, - ielem=5, - bins=np.linspace(-5, 5, 41), - log=False, - xlabel=r"PFElement $\eta$", - ) - plot_eff_and_fake_rate( - X_f, - yvals_f, - plot_path, - sample, - icls=5, - ivar=4, - ielem=4, - bins=np.logspace(-1, 2, 41), - log=True, - ) - plot_eff_and_fake_rate( - X_f, - yvals_f, - plot_path, - sample, - icls=5, - ivar=3, - ielem=4, - bins=np.linspace(-5, 5, 41), - log=False, - xlabel=r"PFElement $\eta$", - ) - - # distribution_icls - print("distribution_icls...") - distribution_icls(yvals_f, plot_path) - - print("plot_numPFelements...") - plot_numPFelements(X, plot_path, sample) - print("plot_met...") - plot_met(X, yvals, plot_path, sample) - print("plot_sum_energy...") - plot_sum_energy(X, yvals, plot_path, sample) - print("plot_sum_pt...") - plot_sum_pt(X, yvals, plot_path, sample) - print("plot_multiplicity...") - plot_multiplicity(X, yvals, plot_path, sample) - - # for energy resolution plotting purposes, initialize pid -> (ylim, bins) dictionary - print("plot_energy_res...") - dic = { - 1: (1e9, np.linspace(-2, 15, 100)), - 2: (1e7, np.linspace(-2, 15, 100)), - 3: (1e7, np.linspace(-2, 40, 100)), - 4: (1e7, np.linspace(-2, 30, 100)), - 5: (1e7, np.linspace(-2, 10, 100)), - 6: (1e4, np.linspace(-1, 1, 100)), - 7: (1e4, np.linspace(-0.1, 0.1, 100)), - } - for pid, tuple in dic.items(): - plot_energy_res(X, yvals_f, pid, tuple[1], tuple[0], plot_path, sample) - - # for eta resolution plotting purposes, initialize pid -> (ylim) dictionary - print("plot_eta_res...") - dic = {1: 1e10, 2: 1e8} - for pid, ylim in dic.items(): - plot_eta_res(X, yvals_f, pid, ylim, plot_path, sample) - - print(f"Time taken to make plots is: {round(((time.time() - t0) / 60), 2)} min") diff --git a/mlpf/pyg/model.py b/mlpf/pyg/model.py index 2f0e1d91b..8bb6a147b 100644 --- a/mlpf/pyg/model.py +++ b/mlpf/pyg/model.py @@ -48,15 +48,7 @@ def __init__( self.conv = nn.ModuleList() for i in range(num_convs): - self.conv.append( - GravNetConv_MLPF( - embedding_dim, - embedding_dim, - space_dim, - propagate_dim, - k, - ) - ) + self.conv.append(GravNetConv_MLPF(embedding_dim, embedding_dim, space_dim, propagate_dim, k)) # self.conv.append(GravNetConv_cmspepr(embedding_dim, embedding_dim, space_dim, propagate_dim, k)) # self.conv.append(EdgeConvBlock(embedding_dim, embedding_dim, k)) @@ -200,119 +192,8 @@ def message(self, x_j: Tensor, edge_weight: Tensor) -> Tensor: return x_j * edge_weight.unsqueeze(1) def aggregate(self, inputs: Tensor, index: Tensor, dim_size: Optional[int] = None) -> Tensor: - out_mean = scatter( - inputs, - index, - dim=self.node_dim, - dim_size=dim_size, - reduce="sum", - ) + out_mean = scatter(inputs, index, dim=self.node_dim, dim_size=dim_size, reduce="sum") return out_mean def __repr__(self) -> str: return f"{self.__class__.__name__}({self.in_channels}, " f"{self.out_channels}, k={self.k})" - - -# try: -# from torch_cmspepr import knn_graph -# except ImportError: -# knn_graph = None -# -# -# class GravNetConv_cmspepr(MessagePassing): -# """ -# Gravnet implementation that uses an optimized version of knn. -# Copied from https://github.com/cms-pepr/pytorch_cmspepr/tree/main/torch_cmspepr -# """ -# -# def __init__(self, in_channels: int, out_channels: int, -# space_dimensions: int, propagate_dimensions: int, k: int, -# num_workers: int = 1, **kwargs): -# super(GravNetConv_cmspepr, self).__init__(flow='target_to_source', **kwargs) -# -# if knn_graph is None: -# raise ImportError('`GravNetConv` requires `torch-cluster`.') -# -# self.in_channels = in_channels -# self.out_channels = out_channels -# self.k = k -# self.num_workers = num_workers -# -# self.lin_s = Linear(in_channels, space_dimensions) -# self.lin_h = Linear(in_channels, propagate_dimensions) -# self.lin = Linear(in_channels + 2 * propagate_dimensions, out_channels) -# -# self.reset_parameters() -# -# def reset_parameters(self): -# self.lin_s.reset_parameters() -# self.lin_h.reset_parameters() -# self.lin.reset_parameters() -# -# def forward( -# self, x: Tensor, -# batch: OptTensor = None) -> Tensor: -# """""" -# -# assert x.dim() == 2, 'Static graphs not supported in `GravNetConv`.' -# -# b: OptTensor = None -# if isinstance(batch, Tensor): -# b = batch -# -# h_l: Tensor = self.lin_h(x) -# -# s_l: Tensor = self.lin_s(x) -# -# edge_index = knn_graph(s_l, self.k, b) -# -# edge_weight = (s_l[edge_index[1]] - s_l[edge_index[0]]).pow(2).sum(-1) -# edge_weight = torch.exp(-10. * edge_weight) # 10 gives a better spread -# -# # propagate_type: (x: OptPairTensor, edge_weight: OptTensor) -# out = self.propagate(edge_index, x=(h_l, None), -# edge_weight=edge_weight, -# size=(s_l.size(0), s_l.size(0))) -# -# return self.lin(torch.cat([out, x], dim=-1)) -# -# def message(self, x_j: Tensor, edge_weight: Tensor) -> Tensor: -# return x_j * edge_weight.unsqueeze(1) -# -# def aggregate(self, inputs: Tensor, index: Tensor, -# dim_size: Optional[int] = None) -> Tensor: -# out_mean = scatter(inputs, index, dim=self.node_dim, dim_size=dim_size, -# reduce='mean') -# out_max = scatter(inputs, index, dim=self.node_dim, dim_size=dim_size, -# reduce='max') -# return torch.cat([out_mean, out_max], dim=-1) -# -# def __repr__(self): -# return '{}({}, {}, k={})'.format(self.__class__.__name__, -# self.in_channels, self.out_channels, -# self.k) -# -# -# class EdgeConvBlock(nn.Module): -# """ -# EdgeConv implementation as an alternative to GravnetConv. -# """ -# -# def __init__(self, in_size, layer_size, k): -# super(EdgeConvBlock, self).__init__() -# -# layers = [] -# -# layers.append(nn.Linear(in_size * 2, layer_size)) -# layers.append(nn.BatchNorm1d(layer_size)) -# layers.append(nn.ReLU()) -# -# # for i in range(2): -# # layers.append(nn.Linear(layer_size, layer_size)) -# # layers.append(nn.BatchNorm1d(layer_size)) -# # layers.append(nn.ReLU()) -# -# self.edge_conv = DynamicEdgeConv(nn.Sequential(*layers), k=k, aggr="mean") -# -# def forward(self, x, batch): -# return self.edge_conv(x, batch) diff --git a/mlpf/pyg/training.py b/mlpf/pyg/training.py index c9ea2a8f3..ea46ca431 100644 --- a/mlpf/pyg/training.py +++ b/mlpf/pyg/training.py @@ -8,9 +8,9 @@ import sklearn.metrics import torch import torch_geometric -from pyg import make_plot_from_lists -from pyg.cms_utils import CLASS_NAMES_CMS -from pyg.delphes_plots import plot_confusion_matrix +from pyg import CLASS_NAMES_CMS, plot_confusion_matrix + +from .utils import make_plot_from_lists matplotlib.use("Agg") @@ -110,11 +110,6 @@ def train( t0 = time.time() pred_ids_one_hot, pred_p4 = model(batch.to(rank)) t1 = time.time() - # print( - # f"batch {i}/{len(loader)}, " - # + f"forward pass on rank {rank} = {round(t1 - t0, 3)}s, " - # + f"for batch with {batch.num_nodes} nodes" - # ) t = t + (t1 - t0) # define the target diff --git a/mlpf/pyg/utils.py b/mlpf/pyg/utils.py index a90c42d02..f690e0c4c 100644 --- a/mlpf/pyg/utils.py +++ b/mlpf/pyg/utils.py @@ -15,89 +15,16 @@ matplotlib.use("Agg") -features_delphes = [ - "Track|cluster", - "$p_{T}|E_{T}$", - r"$\eta$", - r"$Sin(\phi)$", - r"$Cos(\phi)$", - "P|E", - r"$\eta_\mathrm{out}|E_{em}$", - r"$Sin(\(phi)_\mathrm{out}|E_{had}$", - r"$Cos(\phi)_\mathrm{out}|E_{had}$", - "charge", - "is_gen_mu", - "is_gen_el", -] - -features_cms = [ - "typ_idx", - "pt", - "eta", - "phi", - "e", - "layer", - "depth", - "charge", - "trajpoint", - "eta_ecal", - "phi_ecal", - "eta_hcal", - "phi_hcal", - "muon_dt_hits", - "muon_csc_hits", - "muon_type", - "px", - "py", - "pz", - "deltap", - "sigmadeltap", - "gsf_electronseed_trkorecal", - "gsf_electronseed_dnn1", - "gsf_electronseed_dnn2", - "gsf_electronseed_dnn3", - "gsf_electronseed_dnn4", - "gsf_electronseed_dnn5", - "num_hits", - "cluster_flags", - "corr_energy", - "corr_energy_err", - "vx", - "vy", - "vz", - "pterror", - "etaerror", - "phierror", - "lambd", - "lambdaerror", - "theta", - "thetaerror", -] - -target_p4 = [ +Y_FEATURES = [ "charge", "pt", "eta", "sin_phi", "cos_phi", - "energy", + "e", ] -def one_hot_embedding(labels, num_classes): - """ - Embedding labels to one-hot form. - - Args: - labels: (LongTensor) class labels, sized [N,]. - num_classes: (int) number of classes. - Returns: - (tensor) encoded labels, sized [N, #classes]. - """ - y = torch.eye(num_classes) - return y[labels] - - def save_model(args, model_fname, outpath, model_kwargs): if not osp.isdir(outpath): @@ -112,10 +39,7 @@ def save_model(args, model_fname, outpath, model_kwargs): filelist = [f for f in os.listdir(outpath) if not f.endswith(".txt")] # don't remove the newly created logs.txt for f in filelist: - try: - os.remove(os.path.join(outpath, f)) - except IsADirectoryError: - shutil.rmtree(os.path.join(outpath, f)) + shutil.rmtree(os.path.join(outpath, f)) with open(f"{outpath}/model_kwargs.pkl", "wb") as f: # dump model architecture pkl.dump(model_kwargs, f, protocol=pkl.HIGHEST_PROTOCOL) diff --git a/mlpf/pyg_pipeline.py b/mlpf/pyg_pipeline.py index ea574fa21..fdd319d42 100644 --- a/mlpf/pyg_pipeline.py +++ b/mlpf/pyg_pipeline.py @@ -7,28 +7,22 @@ import torch import torch.distributed as dist import torch.multiprocessing as mp -from pyg import ( - MLPF, - PFGraphDataset, - features_cms, - features_delphes, - load_model, - make_file_loaders, - make_plots_cms, - make_predictions, - parse_args, - postprocess_predictions, - save_model, - target_p4, - training_loop, -) +from pyg.args import parse_args +from pyg.cms.cms_plots import make_plots_cms +from pyg.cms.cms_utils import X_FEATURES_CMS +from pyg.delphes.delphes_utils import X_FEATURES_DELPHES +from pyg.evaluate import make_predictions, postprocess_predictions +from pyg.model import MLPF +from pyg.PFGraphDataset import PFGraphDataset +from pyg.training import training_loop +from pyg.utils import load_model, make_file_loaders, save_model from torch.nn.parallel import DistributedDataParallel as DDP matplotlib.use("Agg") """ -Developing a PyTorch Geometric MLPF pipeline using DistributedDataParallel. +Developing a PyTorch Geometric supervised training of MLPF using DistributedDataParallel. Author: Farouk Mokhtar """ @@ -58,7 +52,9 @@ def setup(rank, world_size): os.environ["MASTER_PORT"] = "12355" # dist.init_process_group("gloo", rank=rank, world_size=world_size) - dist.init_process_group("nccl", rank=rank, world_size=world_size) # should be faster for DistributedDataParallel on gpus + dist.init_process_group( + "nccl", rank=rank, world_size=world_size + ) # nccl should be faster than gloo for DistributedDataParallel on gpus def cleanup(): @@ -196,22 +192,15 @@ def inference_ddp(rank, world_size, args, dataset, model, num_classes, PATH): model.eval() ddp_model = DDP(model, device_ids=[rank]) - make_predictions( - rank, - ddp_model, - file_loader_test, - args.batch_size, - num_classes, - PATH, - ) + make_predictions(rank, ddp_model, file_loader_test, args.batch_size, num_classes, PATH) cleanup() def train(device, world_size, args, dataset, model, num_classes, outpath): """ - A train() function that will load the training dataset and start a training_loop - on a single device (cuda or cpu). + A train() function that will load the training dataset and start a + training_loop on a single device (cuda or cpu). """ if device == "cpu": @@ -300,15 +289,14 @@ def inference(device, world_size, args, dataset, model, num_classes, PATH): torch.backends.cudnn.benchmark = True - # retrieve the dimensions of the PF-elements & PF-candidates - # to set the input/output dimension of the model + # retrieve the dimensions of the PF-elements & PF-candidates to set the input/output dimension of the model if args.data == "delphes": - input_dim = len(features_delphes) + input_dim = len(X_FEATURES_DELPHES) num_classes = 6 # we have 6 classes/pids for delphes elif args.data == "cms": - input_dim = len(features_cms) + input_dim = len(X_FEATURES_CMS) num_classes = 9 # we have 9 classes/pids for cms (including taus) - output_dim_p4 = len(target_p4) + output_dim_p4 = 6 # "charge, pt, eta, sin_phi, cos_phi, energy outpath = osp.join(args.outpath, args.model_prefix) @@ -357,15 +345,7 @@ def inference(device, world_size, args, dataset, model, num_classes, PATH): outpath, ) else: - train( - device, - world_size, - args, - dataset, - model, - num_classes, - outpath, - ) + train(device, world_size, args, dataset, model, num_classes, outpath) # load the best epoch state state_dict = torch.load(outpath + "/best_epoch_weights.pth", map_location=device) @@ -386,10 +366,8 @@ def inference(device, world_size, args, dataset, model, num_classes, PATH): if not os.path.exists(PATH): os.makedirs(PATH) - if not os.path.exists(f"{PATH}/predictions/"): - os.makedirs(f"{PATH}/predictions/") - if not os.path.exists(f"{PATH}/plots/"): - os.makedirs(f"{PATH}/plots/") + if not os.path.exists(pred_path): + os.makedirs(pred_path) # run the inference using DDP if more than one gpu is available dataset_test = PFGraphDataset(args.dataset_test, args.data) diff --git a/mlpf/pyg_ssl/PFGraphDataset.py b/mlpf/pyg_ssl/PFGraphDataset.py new file mode 100644 index 000000000..c854b876c --- /dev/null +++ b/mlpf/pyg_ssl/PFGraphDataset.py @@ -0,0 +1,164 @@ +import multiprocessing +import os.path as osp +import sys +from glob import glob + +import torch +from torch_geometric.data import Data, Dataset + +sys.path.append("..") +from data_clic.postprocessing import prepare_data_clic + + +def process_func(args): + self, fns, idx_file = args + return self.process_multiple_files(fns, idx_file) + + +def chunks(lst, n): + """Yield successive n-sized chunks from lst.""" + for i in range(0, len(lst), n): + yield lst[i : i + n] + + +class PFGraphDataset(Dataset): + """ + Initialize parameters of graph dataset + Args: + root (str): path + """ + + def __init__(self, root, transform=None, pre_transform=None): + super(PFGraphDataset, self).__init__(root, transform, pre_transform) + self._processed_dir = Dataset.processed_dir.fget(self) + + @property + def raw_file_names(self): + raw_list = glob(osp.join(self.raw_dir, "*")) + print("PFGraphDataset nfiles={}".format(len(raw_list))) + return sorted([raw_path.replace(self.raw_dir, ".") for raw_path in raw_list]) + + def _download(self): + pass + + def _process(self): + pass + + @property + def processed_dir(self): + return self._processed_dir + + @property + def processed_file_names(self): + proc_list = glob(osp.join(self.processed_dir, "*.pt")) + return sorted([processed_path.replace(self.processed_dir, ".") for processed_path in proc_list]) + + def __len__(self): + return len(self.processed_file_names) + + def download(self): + # Download to `self.raw_dir`. + pass + + def process_single_file(self, raw_file_name): + """ + Loads raw datafile information and generates PyG Data() objects and stores them in .pt format. + + Args + raw_file_name: raw data file name. + Returns + batched_data: a list of Data() objects of the form + clic ~ Data(x=[#, 12], ygen=[#, 5], ygen_id=[#], ycand=[#, 5], ycand_id=[#]) + """ + + events = prepare_data_clic(osp.join(self.raw_dir, raw_file_name)) + data = [] + for event in events: + Xs, ys_gen, ys_cand = event[0], event[1], event[2] + d = Data( + x=torch.tensor(Xs), + ygen=torch.tensor(ys_gen[:, 1:]), + ygen_id=torch.tensor(ys_gen[:, 0]).long(), + ycand=torch.tensor(ys_cand[:, 1:]), + ycand_id=torch.tensor(ys_cand[:, 0]).long(), + ) + data.append(d) + + return data + + def process_multiple_files(self, filenames, idx_file): + datas = [] + for fn in filenames: + x = self.process_single_file(fn) + if x is None: + continue + datas.append(x) + + datas = sum(datas, []) + p = osp.join(self.processed_dir, "data_{}.pt".format(idx_file)) + torch.save(datas, p) + print(f"saved file {p}") + + def process(self, num_files_to_batch): + idx_file = 0 + for fns in chunks(self.raw_file_names, num_files_to_batch): + self.process_multiple_files(fns, idx_file) + idx_file += 1 + + def process_parallel(self, num_files_to_batch, num_proc): + pars = [] + idx_file = 0 + for fns in chunks(self.raw_file_names, num_files_to_batch): + pars += [(self, fns, idx_file)] + idx_file += 1 + pool = multiprocessing.Pool(num_proc) + pool.map(process_func, pars) + + def get(self, idx): + p = osp.join(self.processed_dir, "data_{}.pt".format(idx)) + data = torch.load(p, map_location="cpu") + return data + + def __getitem__(self, idx): + return self.get(idx) + + +def parse_args(): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, required=True, help="Input data path") + parser.add_argument( + "--processed_dir", + type=str, + help="processed", + required=False, + default=None, + ) + parser.add_argument( + "--num-files-merge", + type=int, + default=10, + help="number of files to merge", + ) + parser.add_argument("--num-proc", type=int, default=24, help="number of processes") + args = parser.parse_args() + return args + + +if __name__ == "__main__": + + """ + e.g. to run for clic + python3 PFGraphDataset.py --dataset $sample --processed_dir $sample/processed --num-files-merge 100 + + """ + + args = parse_args() + + pfgraphdataset = PFGraphDataset(root=args.dataset) + + if args.processed_dir: + pfgraphdataset._processed_dir = args.processed_dir + + pfgraphdataset.process_parallel(args.num_files_merge, args.num_proc) diff --git a/mlpf/pyg_ssl/README.md b/mlpf/pyg_ssl/README.md new file mode 100644 index 000000000..0c6530b28 --- /dev/null +++ b/mlpf/pyg_ssl/README.md @@ -0,0 +1,28 @@ +# Setup + +Have conda installed. +```bash +conda env create -f environment.yml +conda activate mlpf +``` + +# Semi-supervised training on CLIC + +To download and process the full CLIC dataset: +```bash +cd clic/ +./get_data_clic.sh +``` +This script will download and process the data under a directory called `data/clic` under `particleflow`. + +To run a training of VICReg: +```bash +cd ../ +python ssl_pipeline.py --model_prefix_VICReg VICReg_test +``` + +To train mlpf via an ssl approach using the pre-trained VICReg model: +```bash +cd ../ +python ssl_pipeline.py --model_prefix_VICReg VICReg_test --load_VICReg --model_prefix_mlpf MLPF_test --train_mlpf +``` diff --git a/mlpf/pyg_ssl/VICReg.py b/mlpf/pyg_ssl/VICReg.py new file mode 100644 index 000000000..a3d1cf2df --- /dev/null +++ b/mlpf/pyg_ssl/VICReg.py @@ -0,0 +1,88 @@ +import torch.nn as nn +from torch_geometric.nn.conv import GravNetConv + +from .utils import CLUSTERS_X, TRACKS_X + + +# define the Encoder that learns latent representations of tracks and clusters +# these representations will be used by MLPF which is the downstream task +class ENCODER(nn.Module): + def __init__( + self, + width=126, + embedding_dim=34, + num_convs=2, + space_dim=4, + propagate_dim=22, + k=8, + ): + super(ENCODER, self).__init__() + + self.act = nn.ELU + + # 1. different embedding of tracks/clusters + self.nn1 = nn.Sequential( + nn.Linear(TRACKS_X, width), + self.act(), + nn.Linear(width, width), + self.act(), + nn.Linear(width, embedding_dim), + ) + self.nn2 = nn.Sequential( + nn.Linear(CLUSTERS_X, width), + self.act(), + nn.Linear(width, width), + self.act(), + nn.Linear(width, embedding_dim), + ) + + # 2. same GNN for tracks/clusters + self.conv = nn.ModuleList() + for i in range(num_convs): + self.conv.append( + GravNetConv( + embedding_dim, + embedding_dim, + space_dimensions=space_dim, + propagate_dimensions=propagate_dim, + k=k, + ) + ) + + def forward(self, tracks, clusters): + + embedding_tracks = self.nn1(tracks.x.float()) + embedding_clusters = self.nn2(clusters.x.float()) + + # perform a series of graph convolutions + for num, conv in enumerate(self.conv): + embedding_tracks = conv(embedding_tracks, tracks.batch) + embedding_clusters = conv(embedding_clusters, clusters.batch) + + return embedding_tracks, embedding_clusters + + +# define the decoder that expands the latent representations of tracks and clusters +class DECODER(nn.Module): + def __init__( + self, + input_dim=34, + width=126, + output_dim=200, + ): + super(DECODER, self).__init__() + + self.act = nn.ELU + + # DECODER + self.expander = nn.Sequential( + nn.Linear(input_dim, width), + self.act(), + nn.Linear(width, width), + self.act(), + nn.Linear(width, output_dim), + ) + + def forward(self, out_tracks, out_clusters): + + return self.expander(out_tracks), self.expander(out_clusters) diff --git a/mlpf/pyg_ssl/__init__.py b/mlpf/pyg_ssl/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mlpf/pyg_ssl/args.py b/mlpf/pyg_ssl/args.py new file mode 100644 index 000000000..cae6f0fc0 --- /dev/null +++ b/mlpf/pyg_ssl/args.py @@ -0,0 +1,51 @@ +import argparse + + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--outpath", type=str, default="../experiments/", help="output folder") + + # samples to be used + parser.add_argument("--samples", default=-1, help="specefies samples to use") + + # directory containing datafiles + parser.add_argument("--dataset", type=str, default="../data/clic/", help="dataset path") + + # flag to load a pre-trained model + parser.add_argument("--load_VICReg", dest="load_VICReg", action="store_true", help="loads the model without training") + + # flag to train mlpf + parser.add_argument("--train_mlpf", dest="train_mlpf", action="store_true", help="Train MLPF") + + parser.add_argument("--model_prefix_VICReg", type=str, default="VICReg_model", help="directory to hold the VICReg model") + parser.add_argument("--model_prefix_mlpf", type=str, default="MLPF_model", help="directory to hold the mlpf model") + parser.add_argument("--overwrite", dest="overwrite", action="store_true", help="overwrites the model if True") + + # training hyperparameters + parser.add_argument("--lmbd", type=float, default=25, help="the lambda term in the VICReg loss") + parser.add_argument("--u", type=float, default=25, help="the mu term in the VICReg loss") + parser.add_argument("--v", type=float, default=1, help="the nu term in the VICReg loss") + parser.add_argument("--n_epochs", type=int, default=3, help="number of training epochs") + parser.add_argument("--lr", type=float, default=1e-3, help="learning rate") + parser.add_argument("--batch_size", type=int, default=100, help="number of events to process at once") + parser.add_argument("--patience", type=int, default=30, help="patience before early stopping") + + # VICReg encoder architecture + parser.add_argument("--width_encoder", type=int, default=126, help="hidden dimension of the encoder") + parser.add_argument("--embedding_dim", type=int, default=34, help="encoded element dimension") + parser.add_argument("--num_convs", type=int, default=2, help="number of graph convolutions") + parser.add_argument("--space_dim", type=int, default=4, help="Gravnet hyperparameter") + parser.add_argument("--propagate_dim", type=int, default=22, help="Gravnet hyperparameter") + parser.add_argument("--nearest", type=int, default=8, help="k nearest neighbors") + + # VICReg decoder architecture + parser.add_argument("--width_decoder", type=int, default=126, help="hidden dimension of the decoder") + parser.add_argument("--expand_dim", type=int, default=200, help="dimension of the output of the decoder") + + # MLPF architecture + parser.add_argument("--width_mlpf", type=int, default=126, help="hidden dimension of mlpf") + + args = parser.parse_args() + + return args diff --git a/mlpf/pyg_ssl/clic/get_data_clic.sh b/mlpf/pyg_ssl/clic/get_data_clic.sh new file mode 100755 index 000000000..77265f911 --- /dev/null +++ b/mlpf/pyg_ssl/clic/get_data_clic.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +# retrieve directories (also here https://jpata.web.cern.ch/jpata/mlpf/clic/) +rsync -r fmokhtar@lxplus.cern.ch:/eos/user/j/jpata/www/mlpf/clic ./ + +# restructure the sample directories to hold parquet files under raw/ and pT files under processed/ +for sample in clic/* ; do + echo Restructuring $sample sample directory + mkdir $sample/raw + mv $sample/*.parquet $sample/raw/ + mkdir $sample/processed +done + +# make data/ directory to hold the clic/ directory of datafiles under particleflow/ +mkdir -p ../../../data + +# move the clic/ directory of datafiles there +mv clic ../../../data/ +cd .. + +# process the raw datafiles +echo ----------------------- +for sample in ../../data/clic/* ; do + echo Processing $sample sample + python3 PFGraphDataset.py --dataset $sample \ + --processed_dir $sample/processed --num-files-merge 100 --num-proc 1 +done +echo ----------------------- diff --git a/mlpf/pyg_ssl/environment.yml b/mlpf/pyg_ssl/environment.yml new file mode 100644 index 000000000..da9b68d7e --- /dev/null +++ b/mlpf/pyg_ssl/environment.yml @@ -0,0 +1,20 @@ +name: mlpf +channels: + - pyg + - pytorch + - conda-forge +dependencies: + - python=3.9 + - numpy + - pandas + - scikit-learn + - matplotlib + - pytorch + - pyg + - cpuonly + - mplhep + - tqdm + - autopep8 + - pip + - pip: + - jupyter-book diff --git a/mlpf/pyg_ssl/evaluate.py b/mlpf/pyg_ssl/evaluate.py new file mode 100644 index 000000000..ba5865f19 --- /dev/null +++ b/mlpf/pyg_ssl/evaluate.py @@ -0,0 +1,92 @@ +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import sklearn +import sklearn.metrics +import torch +from torch_geometric.nn import global_mean_pool + +from .utils import CLASS_NAMES_CLIC_LATEX, NUM_CLASSES, combine_PFelements, distinguish_PFelements + +matplotlib.use("Agg") + +# Ignore divide by 0 errors +np.seterr(divide="ignore", invalid="ignore") + + +def evaluate(device, encoder, decoder, mlpf, test_loader): + + mlpf.eval() + encoder.eval() + decoder.eval() + conf_matrix = np.zeros((6, 6)) + with torch.no_grad(): + for i, batch in enumerate(test_loader): + print(f"making predictions: {i+1}/{len(test_loader)}") + # make transformation + tracks, clusters = distinguish_PFelements(batch.to(device)) + + # ENCODE + embedding_tracks, embedding_clusters = encoder(tracks, clusters) + # POOLING + pooled_tracks = global_mean_pool(embedding_tracks, tracks.batch) + pooled_clusters = global_mean_pool(embedding_clusters, clusters.batch) + # DECODE + out_tracks, out_clusters = decoder(pooled_tracks, pooled_clusters) + + # use the learnt representation as your input as well as the global feature vector + tracks.x = embedding_tracks + clusters.x = embedding_clusters + + event = combine_PFelements(tracks, clusters) + + # make mlpf forward pass + pred_ids_one_hot = mlpf(event.to(device)) + pred_ids = torch.argmax(pred_ids_one_hot, axis=1) + target_ids = event.ygen_id + + conf_matrix += sklearn.metrics.confusion_matrix( + target_ids.detach().cpu(), + pred_ids.detach().cpu(), + labels=range(NUM_CLASSES), + ) + return conf_matrix + + +def plot_conf_matrix(cm, title, outpath): + import itertools + + cmap = plt.get_cmap("Blues") + cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis] + cm[np.isnan(cm)] = 0.0 + + plt.figure(figsize=(8, 6)) + plt.axes() + plt.imshow(cm, interpolation="nearest", cmap=cmap) + plt.colorbar() + + thresh = cm.max() / 1.5 + for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): + plt.text( + j, + i, + "{:0.2f}".format(cm[i, j]), + horizontalalignment="center", + color="white" if cm[i, j] > thresh else "black", + fontsize=15, + ) + plt.title(title, fontsize=25) + plt.xlabel("Predicted label", fontsize=15) + plt.ylabel("True label", fontsize=15) + + plt.xticks( + range(len(CLASS_NAMES_CLIC_LATEX)), + CLASS_NAMES_CLIC_LATEX, + rotation=45, + fontsize=15, + ) + plt.yticks(range(len(CLASS_NAMES_CLIC_LATEX)), CLASS_NAMES_CLIC_LATEX, fontsize=15) + + plt.tight_layout() + + plt.savefig(f"{outpath}/conf_matrix_test.pdf") diff --git a/mlpf/pyg_ssl/mlpf.py b/mlpf/pyg_ssl/mlpf.py new file mode 100644 index 000000000..6ea38e1a7 --- /dev/null +++ b/mlpf/pyg_ssl/mlpf.py @@ -0,0 +1,55 @@ +import torch.nn as nn +from torch_geometric.nn.conv import GravNetConv + +from .utils import NUM_CLASSES + + +# downstream model +class MLPF(nn.Module): + def __init__( + self, + input_dim=34, + width=126, + num_convs=2, + k=8, + ): + super(MLPF, self).__init__() + + self.act = nn.ELU + + # GNN that uses the embeddings learnt by VICReg as the input features + self.conv = nn.ModuleList() + for i in range(num_convs): + self.conv.append( + GravNetConv( + input_dim, + input_dim, + space_dimensions=4, + propagate_dimensions=22, + k=k, + ) + ) + + # DNN that acts on the node level to predict the PID + self.nn = nn.Sequential( + nn.Linear(input_dim, width), + self.act(), + nn.Linear(width, width), + self.act(), + nn.Linear(width, NUM_CLASSES), + ) + + def forward(self, batch): + + # unfold the Batch object + input_ = batch.x.float() + batch = batch.batch + + # perform a series of graph convolutions + for num, conv in enumerate(self.conv): + embedding = conv(input_, batch) + + # predict the PIDs + preds_id = self.nn(embedding) + + return preds_id diff --git a/mlpf/pyg_ssl/training_VICReg.py b/mlpf/pyg_ssl/training_VICReg.py new file mode 100644 index 000000000..e04774930 --- /dev/null +++ b/mlpf/pyg_ssl/training_VICReg.py @@ -0,0 +1,256 @@ +import json +import pickle as pkl +import time + +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn.functional as F +from torch_geometric.nn import global_mean_pool + +from .utils import distinguish_PFelements + +matplotlib.use("Agg") + +# Ignore divide by 0 errors +np.seterr(divide="ignore", invalid="ignore") + + +# VICReg loss function +def criterion(x, y, device="cuda", lmbd=25, u=25, v=1, epsilon=1e-3): + bs = x.size(0) + emb = x.size(1) + + std_x = torch.sqrt(x.var(dim=0) + epsilon) + std_y = torch.sqrt(y.var(dim=0) + epsilon) + var_loss = torch.mean(F.relu(1 - std_x)) + torch.mean(F.relu(1 - std_y)) + + invar_loss = F.mse_loss(x, y) + + xNorm = (x - x.mean(0)) / x.std(0) + yNorm = (y - y.mean(0)) / y.std(0) + crossCorMat = (xNorm.T @ yNorm) / bs + cross_loss = (crossCorMat * lmbd - torch.eye(emb, device=torch.device(device)) * lmbd).pow(2).sum() + + loss = u * var_loss + v * invar_loss + cross_loss + + return loss + + +@torch.no_grad() +def validation_run( + device, + encoder, + decoder, + train_loader, + valid_loader, + lmbd, + u, + v, +): + with torch.no_grad(): + optimizer = None + ret = train( + device, + encoder, + decoder, + train_loader, + valid_loader, + optimizer, + lmbd, + u, + v, + ) + return ret + + +def train( + device, + encoder, + decoder, + train_loader, + valid_loader, + optimizer, + lmbd, + u, + v, +): + """ + A training/validation run over a given epoch that gets called in the training_loop() function. + When optimizer is set to None, it freezes the model for a validation_run. + """ + + is_train = not (optimizer is None) + + if is_train: + print("---->Initiating a training run") + encoder.train() + decoder.train() + loader = train_loader + else: + print("---->Initiating a validation run") + encoder.eval() + decoder.eval() + loader = valid_loader + + # initialize loss counters + losses = 0 + + for i, batch in enumerate(loader): + + # make transformation + tracks, clusters = distinguish_PFelements(batch.to(device)) + + # ENCODE + embedding_tracks, embedding_clusters = encoder(tracks, clusters) + # POOLING + pooled_tracks = global_mean_pool(embedding_tracks, tracks.batch) + pooled_clusters = global_mean_pool(embedding_clusters, clusters.batch) + # DECODE + out_tracks, out_clusters = decoder(pooled_tracks, pooled_clusters) + + # compute loss + loss = criterion(out_tracks, out_clusters, device, lmbd, u, v) + + # update parameters + if is_train: + for param in encoder.parameters(): + param.grad = None + for param in decoder.parameters(): + param.grad = None + loss.backward() + optimizer.step() + + losses += loss.detach() + + # if i == 20: + # break + + losses = losses.cpu().item() / len(loader) + + return losses + + +def training_loop_VICReg( + device, + encoder, + decoder, + train_loader, + valid_loader, + n_epochs, + patience, + optimizer, + outpath, + lmbd, + u, + v, +): + """ + Main function to perform training. Will call the train() and validation_run() functions every epoch. + + Args: + encoder: the encoder part of VICReg + decoder: the decoder part of VICReg + train_loader: a pytorch Dataloader for training + valid_loader: a pytorch Dataloader for validation + patience: number of stale epochs allowed before stopping the training + optimizer: optimizer to use for training (by default: Adam) + outpath: path to store the model weights and training plots + """ + + t0_initial = time.time() + + losses_train, losses_valid = [], [] + + best_val_loss = 99999.9 + stale_epochs = 0 + + for epoch in range(n_epochs): + t0 = time.time() + + if stale_epochs > patience: + print("breaking due to stale epochs") + break + + # training step + losses = train( + device, + encoder, + decoder, + train_loader, + valid_loader, + optimizer, + lmbd, + u, + v, + ) + + losses_train.append(losses) + + # validation step + losses = validation_run( + device, + encoder, + decoder, + train_loader, + valid_loader, + lmbd, + u, + v, + ) + + losses_valid.append(losses) + + # early-stopping + if losses < best_val_loss: + best_val_loss = losses + stale_epochs = 0 + + try: + encoder_state_dict = encoder.module.state_dict() + except AttributeError: + encoder_state_dict = encoder.state_dict() + try: + decoder_state_dict = decoder.module.state_dict() + except AttributeError: + decoder_state_dict = decoder.state_dict() + + torch.save(encoder_state_dict, f"{outpath}/encoder_best_epoch_weights.pth") + torch.save(decoder_state_dict, f"{outpath}/decoder_best_epoch_weights.pth") + + with open(f"{outpath}/VICReg_best_epoch.json", "w") as fp: # dump best epoch + json.dump({"best_epoch": epoch}, fp) + else: + stale_epochs += 1 + + t1 = time.time() + + epochs_remaining = n_epochs - (epoch + 1) + time_per_epoch = (t1 - t0_initial) / (epoch + 1) + eta = epochs_remaining * time_per_epoch / 60 + + print( + f"epoch={epoch + 1} / {n_epochs} " + + f"train_loss={round(losses_train[epoch], 4)} " + + f"valid_loss={round(losses_valid[epoch], 4)} " + + f"stale={stale_epochs} " + + f"time={round((t1-t0)/60, 2)}m " + + f"eta={round(eta, 1)}m" + ) + + fig, ax = plt.subplots() + ax.plot(range(len(losses_train)), losses_train, label="training") + ax.plot(range(len(losses_valid)), losses_valid, label="validation") + ax.set_xlabel("Epochs") + ax.set_ylabel("Loss") + ax.legend(title="VICReg", loc="best", title_fontsize=20, fontsize=15) + plt.savefig(f"{outpath}/VICReg_loss.pdf") + + with open(f"{outpath}/VICReg_loss_train.pkl", "wb") as f: + pkl.dump(losses_train, f) + with open(f"{outpath}/VICReg_loss_valid.pkl", "wb") as f: + pkl.dump(losses_valid, f) + + print("----------------------------------------------------------") + print(f"Done with training. Total training time is {round((time.time() - t0_initial)/60,3)}min") diff --git a/mlpf/pyg_ssl/training_mlpf.py b/mlpf/pyg_ssl/training_mlpf.py new file mode 100644 index 000000000..2c75b9ef8 --- /dev/null +++ b/mlpf/pyg_ssl/training_mlpf.py @@ -0,0 +1,224 @@ +import json +import math +import pickle as pkl +import time + +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import torch + +from .utils import combine_PFelements, distinguish_PFelements + +matplotlib.use("Agg") + +# Ignore divide by 0 errors +np.seterr(divide="ignore", invalid="ignore") + + +def compute_weights(device, target_ids, num_classes): + """ + computes necessary weights to accomodate class imbalance in the loss function + """ + + vs, cs = torch.unique(target_ids, return_counts=True) + weights = torch.zeros(num_classes).to(device=device) + for k, v in zip(vs, cs): + weights[k] = 1.0 / math.sqrt(float(v)) + # weights[2] = weights[2] * 3 # emphasize nhadrons + return weights + + +@torch.no_grad() +def validation_run( + device, + encoder, + mlpf, + train_loader, + valid_loader, +): + with torch.no_grad(): + optimizer = None + ret = train( + device, + encoder, + mlpf, + train_loader, + valid_loader, + optimizer, + ) + return ret + + +def train( + device, + encoder, + mlpf, + train_loader, + valid_loader, + optimizer, +): + """ + A training/validation run over a given epoch that gets called in the training_loop() function. + When optimizer is set to None, it freezes the model for a validation_run. + """ + + is_train = not (optimizer is None) + + if is_train: + print("---->Initiating a training run") + mlpf.train() + loader = train_loader + else: + print("---->Initiating a validation run") + mlpf.eval() + loader = valid_loader + + # initialize loss counters + losses = 0 + + for i, batch in enumerate(loader): + + # make transformation + tracks, clusters = distinguish_PFelements(batch.to(device)) + + # ENCODE + embedding_tracks, embedding_clusters = encoder(tracks, clusters) + + tracks.x = embedding_tracks + clusters.x = embedding_clusters + + event = combine_PFelements(tracks, clusters) + + # make mlpf forward pass + pred_ids_one_hot = mlpf(event.to(device)) + target_ids = event.to(device).ygen_id + + weights = compute_weights(device, target_ids, num_classes=6) # to accomodate class imbalance + loss = torch.nn.functional.cross_entropy(pred_ids_one_hot, target_ids, weight=weights) # for classifying PID + + # update parameters + if is_train: + for param in mlpf.parameters(): + param.grad = None + loss.backward() + optimizer.step() + + losses += loss.detach() + + # if i == 20: + # break + + losses = losses.cpu().item() / len(loader) + + return losses + + +def training_loop_mlpf( + device, + encoder, + mlpf, + train_loader, + valid_loader, + n_epochs, + patience, + optimizer, + outpath, +): + """ + Main function to perform training. Will call the train() and validation_run() functions every epoch. + + Args: + encoder: the encoder part of VICReg + mlpf: the mlpf downstream task + train_loader: a pytorch Dataloader for training + valid_loader: a pytorch Dataloader for validation + patience: number of stale epochs allowed before stopping the training + optimizer: optimizer to use for training (by default: Adam) + outpath: path to store the model weights and training plots + """ + + t0_initial = time.time() + + losses_train, losses_valid = [], [] + + best_val_loss = 99999.9 + stale_epochs = 0 + + for epoch in range(n_epochs): + t0 = time.time() + + if stale_epochs > patience: + print("breaking due to stale epochs") + break + + # training step + losses = train( + device, + encoder, + mlpf, + train_loader, + valid_loader, + optimizer, + ) + + losses_train.append(losses) + + # validation step + losses = validation_run( + device, + encoder, + mlpf, + train_loader, + valid_loader, + ) + + losses_valid.append(losses) + + # early-stopping + if losses < best_val_loss: + best_val_loss = losses + stale_epochs = 0 + + try: + mlpf_state_dict = mlpf.module.state_dict() + except AttributeError: + mlpf_state_dict = mlpf.state_dict() + + torch.save(mlpf_state_dict, f"{outpath}/mlpf_best_epoch_weights.pth") + + with open(f"{outpath}/mlpf_best_epoch.json", "w") as fp: # dump best epoch + json.dump({"best_epoch": epoch}, fp) + else: + stale_epochs += 1 + + t1 = time.time() + + epochs_remaining = n_epochs - (epoch + 1) + time_per_epoch = (t1 - t0_initial) / (epoch + 1) + eta = epochs_remaining * time_per_epoch / 60 + + print( + f"epoch={epoch + 1} / {n_epochs} " + + f"train_loss={round(losses_train[epoch], 4)} " + + f"valid_loss={round(losses_valid[epoch], 4)} " + + f"stale={stale_epochs} " + + f"time={round((t1-t0)/60, 2)}m " + + f"eta={round(eta, 1)}m" + ) + + fig, ax = plt.subplots() + ax.plot(range(len(losses_train)), losses_train, label="training") + ax.plot(range(len(losses_valid)), losses_valid, label="validation") + ax.set_xlabel("Epochs") + ax.set_ylabel("Loss") + ax.legend(title="SSL-based MLPF", loc="best", title_fontsize=20, fontsize=15) + plt.savefig(f"{outpath}/mlpf_loss.pdf") + + with open(f"{outpath}/mlpf_loss_train.pkl", "wb") as f: + pkl.dump(losses_train, f) + with open(f"{outpath}/mlpf_loss_valid.pkl", "wb") as f: + pkl.dump(losses_valid, f) + + print("----------------------------------------------------------") + print(f"Done with training. Total training time is {round((time.time() - t0_initial)/60,3)}min") diff --git a/mlpf/pyg_ssl/utils.py b/mlpf/pyg_ssl/utils.py new file mode 100644 index 000000000..55ed13877 --- /dev/null +++ b/mlpf/pyg_ssl/utils.py @@ -0,0 +1,164 @@ +import json +import os +import os.path as osp +import pickle as pkl +import shutil +import sys + +import matplotlib +import torch +from torch_geometric.data import Batch + +matplotlib.use("Agg") + +# define input/output dimensions +CLUSTERS_X = 6 +TRACKS_X = 11 +COMMON_X = 11 +NUM_CLASSES = 6 +CLASS_NAMES_CLIC_LATEX = [ + "none", + "chhad", + "nhad", + r"$\gamma$", + r"$e^\pm$", + r"$\mu^\pm$", +] + + +# function that takes an event~Batch() and splits it into two Batch() objects representing the tracks/clusters +def distinguish_PFelements(batch): + + track_id = 1 + cluster_id = 2 + + tracks = Batch( + x=batch.x[batch.x[:, 0] == track_id][:, 1:].float(), # remove the first input feature which is not needed anymore + ygen=batch.ygen[batch.x[:, 0] == track_id], + ygen_id=batch.ygen_id[batch.x[:, 0] == track_id], + ycand=batch.ycand[batch.x[:, 0] == track_id], + ycand_id=batch.ycand_id[batch.x[:, 0] == track_id], + batch=batch.batch[batch.x[:, 0] == track_id], + ) + clusters = Batch( + x=batch.x[batch.x[:, 0] == cluster_id][:, 1:].float()[ + :, :CLUSTERS_X + ], # remove the first input feature which is not needed anymore + ygen=batch.ygen[batch.x[:, 0] == cluster_id], + ygen_id=batch.ygen_id[batch.x[:, 0] == cluster_id], + ycand=batch.ycand[batch.x[:, 0] == cluster_id], + ycand_id=batch.ycand_id[batch.x[:, 0] == cluster_id], + batch=batch.batch[batch.x[:, 0] == cluster_id], + ) + + return tracks, clusters + + +# conversly, function that combines the learned latent representations back into one Batch() object +def combine_PFelements(tracks, clusters): + + # zero padding + # clusters.x = torch.cat([clusters.x, torch.from_numpy(np.zeros([clusters.x.shape[0],TRACKS_X-CLUSTERS_X]))], axis=1) + + event = Batch( + x=torch.cat([tracks.x, clusters.x]), + ygen=torch.cat([tracks.ygen, clusters.ygen]), + ygen_id=torch.cat([tracks.ygen_id, clusters.ygen_id]), + ycand=torch.cat([tracks.ycand, clusters.ycand]), + ycand_id=torch.cat([tracks.ycand_id, clusters.ycand_id]), + batch=torch.cat([tracks.batch, clusters.batch]), + ) + + return event + + +def load_VICReg(device, outpath): + + encoder_state_dict = torch.load(f"{outpath}/encoder_best_epoch_weights.pth", map_location=device) + decoder_state_dict = torch.load(f"{outpath}/decoder_best_epoch_weights.pth", map_location=device) + + print("Loading a previously trained model..") + with open(f"{outpath}/encoder_model_kwargs.pkl", "rb") as f: + encoder_model_kwargs = pkl.load(f) + with open(f"{outpath}/decoder_model_kwargs.pkl", "rb") as f: + decoder_model_kwargs = pkl.load(f) + + return ( + encoder_state_dict, + encoder_model_kwargs, + decoder_state_dict, + decoder_model_kwargs, + ) + + +def save_VICReg(args, outpath, encoder_model_kwargs, decoder_model_kwargs): + + if not osp.isdir(outpath): + os.makedirs(outpath) + + else: # if directory already exists + if not args.overwrite: # if not overwrite then exit + print("model already exists, please delete it") + sys.exit(0) + + print("model already exists, deleting it") + + filelist = [f for f in os.listdir(outpath) if not f.endswith(".txt")] # don't remove the newly created logs.txt + for f in filelist: + shutil.rmtree(os.path.join(outpath, f)) + + with open(f"{outpath}/encoder_model_kwargs.pkl", "wb") as f: # dump model architecture + pkl.dump(encoder_model_kwargs, f, protocol=pkl.HIGHEST_PROTOCOL) + with open(f"{outpath}/decoder_model_kwargs.pkl", "wb") as f: # dump model architecture + pkl.dump(decoder_model_kwargs, f, protocol=pkl.HIGHEST_PROTOCOL) + + with open(f"{outpath}/hyperparameters.json", "w") as fp: # dump hyperparameters + json.dump( + { + "n_epochs": args.n_epochs, + "lr": args.lr, + "batch_size": args.batch_size, + "width_encoder": args.width_encoder, + "embedding_dim": args.embedding_dim, + "num_convs": args.num_convs, + "space_dim": args.space_dim, + "propagate_dim": args.propagate_dim, + "k": args.nearest, + "input_dim": args.embedding_dim, + "width_decoder": args.width_decoder, + "output_dim": args.expand_dim, + "lmbd": args.lmbd, + "u": args.u, + "v": args.v, + }, + fp, + ) + + +def save_MLPF(args, outpath, mlpf_model_kwargs): + + if not osp.isdir(outpath): + os.makedirs(outpath) + + else: # if directory already exists + filelist = [f for f in os.listdir(outpath) if not f.endswith(".txt")] # don't remove the newly created logs.txt + for f in filelist: + shutil.rmtree(os.path.join(outpath, f)) + + with open(f"{outpath}/mlpf_model_kwargs.pkl", "wb") as f: # dump model architecture + pkl.dump(mlpf_model_kwargs, f, protocol=pkl.HIGHEST_PROTOCOL) + + with open(f"{outpath}/hyperparameters.json", "w") as fp: # dump hyperparameters + json.dump( + { + "n_epochs": args.n_epochs, + "lr": args.lr, + "batch_size": args.batch_size, + "width": args.width_mlpf, + "num_convs": args.num_convs, + "space_dim": args.space_dim, + "propagate_dim": args.propagate_dim, + "k": args.nearest, + }, + fp, + ) diff --git a/mlpf/ssl_pipeline.py b/mlpf/ssl_pipeline.py new file mode 100644 index 000000000..5044d0cc0 --- /dev/null +++ b/mlpf/ssl_pipeline.py @@ -0,0 +1,223 @@ +import glob +import os +import os.path as osp +import pickle as pkl +import random +import sys + +import matplotlib +import numpy as np +import torch +import torch_geometric +from pyg_ssl.args import parse_args +from pyg_ssl.evaluate import evaluate, plot_conf_matrix +from pyg_ssl.mlpf import MLPF +from pyg_ssl.training_mlpf import training_loop_mlpf +from pyg_ssl.training_VICReg import training_loop_VICReg +from pyg_ssl.utils import load_VICReg, save_MLPF, save_VICReg +from pyg_ssl.VICReg import DECODER, ENCODER + +matplotlib.use("Agg") + + +""" +Developing a PyTorch Geometric semi-supervised (VICReg-based https://arxiv.org/abs/2105.04906) pipeline +for particleflow reconstruction on CLIC datasets. + +Author: Farouk Mokhtar +""" + + +# Ignore divide by 0 errors +np.seterr(divide="ignore", invalid="ignore") + +# define the global base device +if torch.cuda.device_count(): + device = torch.device("cuda:0") + print(f"Will use {torch.cuda.get_device_name(device)}") +else: + device = "cpu" + print("Will use cpu") + + +if __name__ == "__main__": + + args = parse_args() + + world_size = torch.cuda.device_count() + + torch.backends.cudnn.benchmark = True + + # load the clic dataset + if args.samples == -1: # use all samples + samples = [ + "gev380ee_pythia6_higgs_bbar_full201", + "gev380ee_pythia6_higgs_zz_4l_full201", + "gev380ee_pythia6_qcd_all_rfull201", + "gev380ee_pythia6_higgs_gamgam_full201", + "gev380ee_pythia6_ttbar_rfull201", + "gev380ee_pythia6_zpole_ee_rfull201", + ] + else: + samples = args.samples.split(",") + + data = [] + for sample in samples: + if sample not in os.listdir(args.dataset): + print(f"no processed files found for sample {sample}") + continue + + files = glob.glob(f"{args.dataset}/{sample}/processed/*") + data_per_sample = [] + for file in files: + data_per_sample += torch.load(f"{file}") + + print(f"Number of events for sample {sample} is: {len(data_per_sample)}") + data += data_per_sample + + # shuffle datafiles belonging to different samples + random.shuffle(data) + + if len(data) == 0: + print("failed to load dataset, check --dataset path") + sys.exit(0) + else: + print(f"---> Total number of events at hand: {len(data)}") + + data_VICReg = data[: round(0.9 * len(data))] + data_mlpf = data[round(0.9 * len(data)) :] + print(f"Will use {len(data_VICReg)} events for VICReg with an 80/20 split") + print(f"Will use {len(data_mlpf)} for mlpf with a 50/25/25 split") + + # setup the directory path to hold all models and plots + outpath = osp.join(args.outpath, args.model_prefix_VICReg) + + # load a pre-trained VICReg model + if args.load_VICReg: + ( + encoder_state_dict, + encoder_model_kwargs, + decoder_state_dict, + decoder_model_kwargs, + ) = load_VICReg(device, outpath) + + encoder = ENCODER(**encoder_model_kwargs) + decoder = DECODER(**decoder_model_kwargs) + + encoder.load_state_dict(encoder_state_dict) + decoder.load_state_dict(decoder_state_dict) + + decoder = decoder.to(device) + encoder = encoder.to(device) + + else: + encoder_model_kwargs = { + "width": args.width_encoder, + "embedding_dim": args.embedding_dim, + "num_convs": args.num_convs, + "space_dim": args.space_dim, + "propagate_dim": args.propagate_dim, + "k": args.nearest, + } + + decoder_model_kwargs = { + "input_dim": args.embedding_dim, + "width": args.width_decoder, + "output_dim": args.expand_dim, + } + + encoder = ENCODER(**encoder_model_kwargs) + decoder = DECODER(**decoder_model_kwargs) + + print("Encoder", encoder) + print("Decoder", decoder) + print(f"VICReg model name: {args.model_prefix_VICReg}") + + # save model_kwargs and hyperparameters + save_VICReg(args, outpath, encoder_model_kwargs, decoder_model_kwargs) + + print("Training over {} epochs".format(args.n_epochs)) + + data_train = data_VICReg[: int(0.8 * len(data))] + data_valid = data_VICReg[int(0.8 * len(data)) :] + + train_loader = torch_geometric.loader.DataLoader(data_train, args.batch_size) + valid_loader = torch_geometric.loader.DataLoader(data_valid, args.batch_size) + + decoder = decoder.to(device) + encoder = encoder.to(device) + + optimizer = torch.optim.SGD( + list(encoder.parameters()) + list(decoder.parameters()), + lr=args.lr, + momentum=0.9, + weight_decay=1.5e-4, + ) + + training_loop_VICReg( + device, + encoder, + decoder, + train_loader, + valid_loader, + args.n_epochs, + args.patience, + optimizer, + outpath, + args.lmbd, + args.u, + args.v, + ) + + if args.train_mlpf: + + data_train = data_mlpf[: round(0.5 * len(data_mlpf))] + data_valid = data_mlpf[round(0.5 * len(data_mlpf)) : round(0.75 * len(data_mlpf))] + data_test = data_mlpf[round(0.75 * len(data_mlpf)) :] + + print(f"Will use {len(data_train)} events for train") + print(f"Will use {len(data_valid)} events for valid") + print(f"Will use {len(data_test)} events for test") + + train_loader = torch_geometric.loader.DataLoader(data_train, args.batch_size) + valid_loader = torch_geometric.loader.DataLoader(data_valid, args.batch_size) + + mlpf_model_kwargs = { + "input_dim": encoder.conv[1].out_channels, + "width": args.width_mlpf, + } + + mlpf = MLPF(**mlpf_model_kwargs) + mlpf = mlpf.to(device) + print(mlpf) + print(f"MLPF model name: {args.model_prefix_mlpf}") + + # make mlpf specific directory + outpath = osp.join(f"{outpath}/MLPF/", args.model_prefix_mlpf) + save_MLPF(args, outpath, mlpf_model_kwargs) + + optimizer = torch.optim.SGD(mlpf.parameters(), lr=args.lr) + + print("Training MLPF") + + training_loop_mlpf( + device, + encoder, + mlpf, + train_loader, + valid_loader, + args.n_epochs, + args.patience, + optimizer, + outpath, + ) + + # test + test_loader = torch_geometric.loader.DataLoader(data_test, args.batch_size) + + conf_matrix = evaluate(device, encoder, decoder, mlpf, test_loader) + + plot_conf_matrix(conf_matrix, "SSL based MLPF", outpath) + + with open(f"{outpath}/conf_matrix_test.pkl", "wb") as f: + pkl.dump(conf_matrix, f) diff --git a/notebooks/ssl-VICreg.ipynb b/notebooks/ssl-VICreg.ipynb new file mode 100644 index 000000000..f315ee2ad --- /dev/null +++ b/notebooks/ssl-VICreg.ipynb @@ -0,0 +1,4800 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "import torchvision.transforms as tr\n", + "\n", + "import torch_geometric\n", + "from torch_geometric.nn import global_mean_pool\n", + "from torch_geometric.data import Batch\n", + "\n", + "from typing import Optional, Union\n", + "\n", + "from torch import Tensor\n", + "from torch.nn import Linear\n", + "from torch_geometric.nn.conv import MessagePassing, GravNetConv\n", + "from torch_geometric.typing import OptTensor, PairOptTensor, PairTensor\n", + "from torch_scatter import scatter\n", + "\n", + "from tqdm.notebook import tqdm\n", + "\n", + "import numpy as np\n", + "\n", + "import json\n", + "import math\n", + "import os\n", + "import time\n", + "import pickle as pkl\n", + "\n", + "import sklearn\n", + "import sklearn.metrics\n", + "\n", + "import matplotlib\n", + "import matplotlib.pyplot as plt\n", + "import mplhep as hep\n", + "\n", + "plt.style.use(hep.style.CMS)\n", + "plt.rcParams.update({\"font.size\": 20})" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# VICReg loss function\n", + "def criterion(x, y, device=\"cuda\", lmbd=25, u=25, v=1, epsilon=1e-3):\n", + " bs = x.size(0)\n", + " emb = x.size(1)\n", + "\n", + " std_x = torch.sqrt(x.var(dim=0) + epsilon)\n", + " std_y = torch.sqrt(y.var(dim=0) + epsilon)\n", + " var_loss = torch.mean(F.relu(1 - std_x)) + torch.mean(F.relu(1 - std_y))\n", + "\n", + " invar_loss = F.mse_loss(x, y)\n", + "\n", + " xNorm = (x - x.mean(0)) / x.std(0)\n", + " yNorm = (y - y.mean(0)) / y.std(0)\n", + " crossCorMat = (xNorm.T @ yNorm) / bs\n", + " cross_loss = (\n", + " (\n", + " crossCorMat * lmbd\n", + " - torch.eye(emb, device=torch.device(device)) * lmbd\n", + " )\n", + " .pow(2)\n", + " .sum()\n", + " )\n", + "\n", + " loss = u * var_loss + v * invar_loss + cross_loss\n", + "\n", + " return loss" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# CLIC" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data_0.pt\r\n" + ] + } + ], + "source": [ + "! ls ../data/clic/gev380ee_pythia6_higgs_bbar_full201/processed/data_0.pt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "\n", + "all_files = glob.glob(\n", + " f\"../data/clic/gev380ee_pythia6_higgs_bbar_full201/processed/data_0.pt\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# load the clic dataset\n", + "data = []\n", + "for f in all_files:\n", + " data += torch.load(f\"{f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "A single event: \n", + " Batch(x=[103, 12], ygen=[103, 5], ygen_id=[103], ycand=[103, 5], ycand_id=[103], batch=[103], ptr=[2])\n" + ] + } + ], + "source": [ + "loader = torch_geometric.loader.DataLoader(data, batch_size=1, shuffle=True)\n", + "for batch in loader:\n", + " print(f\"A single event: \\n {batch}\")\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "num of clic events 9233\n" + ] + } + ], + "source": [ + "print(f\"num of clic events {len(loader)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## From event to tracks/clusters" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "CLUSTERS_X = 6\n", + "TRACKS_X = 11\n", + "COMMON_X = 11" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# function that takes an event~Batch() and splits it into two Batch() objects representing the tracks/clusters\n", + "def distinguish_PFelements(batch):\n", + "\n", + " track_id = 1\n", + " cluster_id = 2\n", + "\n", + " tracks = Batch(\n", + " x=batch.x[batch.x[:, 0] == track_id][\n", + " :, 1:\n", + " ].float(), # remove the first input feature which is not needed anymore\n", + " ygen=batch.ygen[batch.x[:, 0] == track_id],\n", + " ygen_id=batch.ygen_id[batch.x[:, 0] == track_id],\n", + " ycand=batch.ycand[batch.x[:, 0] == track_id],\n", + " ycand_id=batch.ycand_id[batch.x[:, 0] == track_id],\n", + " batch=batch.batch[batch.x[:, 0] == track_id],\n", + " )\n", + " clusters = Batch(\n", + " x=batch.x[batch.x[:, 0] == cluster_id][:, 1:].float()[\n", + " :, :CLUSTERS_X\n", + " ], # remove the first input feature which is not needed anymore\n", + " ygen=batch.ygen[batch.x[:, 0] == cluster_id],\n", + " ygen_id=batch.ygen_id[batch.x[:, 0] == cluster_id],\n", + " ycand=batch.ycand[batch.x[:, 0] == cluster_id],\n", + " ycand_id=batch.ycand_id[batch.x[:, 0] == cluster_id],\n", + " batch=batch.batch[batch.x[:, 0] == cluster_id],\n", + " )\n", + "\n", + " return tracks, clusters\n", + "\n", + "\n", + "# conversly, function that combines the learned latent representations back into one Batch() object\n", + "def combine_PFelements(tracks, clusters):\n", + "\n", + " # zero padding\n", + " # clusters.x = torch.cat([clusters.x, torch.from_numpy(np.zeros([clusters.x.shape[0],TRACKS_X-CLUSTERS_X]))], axis=1)\n", + "\n", + " event = Batch(\n", + " x=torch.cat([tracks.x, clusters.x]),\n", + " ygen=torch.cat([tracks.ygen, clusters.ygen]),\n", + " ygen_id=torch.cat([tracks.ygen_id, clusters.ygen_id]),\n", + " ycand=torch.cat([tracks.ycand, clusters.ycand]),\n", + " ycand_id=torch.cat([tracks.ycand_id, clusters.ycand_id]),\n", + " batch=torch.cat([tracks.batch, clusters.batch]),\n", + " )\n", + "\n", + " return event" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "event: Batch(x=[103, 12], ygen=[103, 5], ygen_id=[103], ycand=[103, 5], ycand_id=[103], batch=[103], ptr=[2])\n", + "tracks: Batch(x=[34, 11], ygen=[34, 5], ygen_id=[34], ycand=[34, 5], ycand_id=[34], batch=[34])\n", + "clusters: Batch(x=[69, 6], ygen=[69, 5], ygen_id=[69], ycand=[69, 5], ycand_id=[69], batch=[69])\n" + ] + } + ], + "source": [ + "tracks, clusters = distinguish_PFelements(batch)\n", + "\n", + "print(f\"event: {batch}\")\n", + "print(f\"tracks: {tracks}\")\n", + "print(f\"clusters: {clusters}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# VICreg" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "# define the Encoder that learns latent representations of tracks and clusters\n", + "# these representations will be used by MLPF which is the downstream task\n", + "class Encoder(nn.Module):\n", + " def __init__(\n", + " self,\n", + " input_dim=11,\n", + " embedding_dim=34,\n", + " num_convs=2,\n", + " ):\n", + " super(Encoder, self).__init__()\n", + "\n", + " self.act = nn.ELU\n", + "\n", + " ### 1. different embedding of tracks/clusters\n", + " self.nn1 = nn.Sequential(\n", + " nn.Linear(TRACKS_X, 126),\n", + " self.act(),\n", + " nn.Linear(126, 126),\n", + " self.act(),\n", + " nn.Linear(126, embedding_dim),\n", + " )\n", + " self.nn2 = nn.Sequential(\n", + " nn.Linear(CLUSTERS_X, 126),\n", + " self.act(),\n", + " nn.Linear(126, 126),\n", + " self.act(),\n", + " nn.Linear(126, embedding_dim),\n", + " )\n", + "\n", + " ### 2. same GNN for tracks/clusters\n", + " self.conv = nn.ModuleList()\n", + " for i in range(num_convs):\n", + " self.conv.append(\n", + " GravNetConv(\n", + " embedding_dim,\n", + " embedding_dim,\n", + " space_dimensions=4,\n", + " propagate_dimensions=22,\n", + " k=16,\n", + " )\n", + " )\n", + "\n", + " def forward(self, tracks, clusters):\n", + "\n", + " embedding_tracks = self.nn1(tracks.x.float())\n", + " embedding_clusters = self.nn2(clusters.x.float())\n", + "\n", + " # perform a series of graph convolutions\n", + " for num, conv in enumerate(self.conv):\n", + " embedding_tracks = conv(embedding_tracks, tracks.batch)\n", + " embedding_clusters = conv(embedding_clusters, clusters.batch)\n", + "\n", + " return embedding_tracks, embedding_clusters" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# define the decoder that expands the latent representations of tracks and clusters\n", + "class Decoder(nn.Module):\n", + " def __init__(\n", + " self,\n", + " embedding_dim=34,\n", + " output_dim=200,\n", + " ):\n", + " super(Decoder, self).__init__()\n", + "\n", + " self.act = nn.ELU\n", + "\n", + " ############################ DECODER\n", + " self.expander = nn.Sequential(\n", + " nn.Linear(embedding_dim, 126),\n", + " self.act(),\n", + " nn.Linear(126, 126),\n", + " self.act(),\n", + " nn.Linear(126, output_dim),\n", + " )\n", + "\n", + " def forward(self, out_tracks, out_clusters):\n", + "\n", + " return self.expander(out_tracks), self.expander(out_clusters)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loss is: 6351292.0\n" + ] + } + ], + "source": [ + "# retrieve a batch with batch_size>1\n", + "loader = torch_geometric.loader.DataLoader(data, batch_size=2)\n", + "for batch in loader:\n", + " break\n", + "\n", + "# retrieve the tracks and clusters\n", + "tracks, clusters = distinguish_PFelements(batch)\n", + "\n", + "# setup VICReg\n", + "encoder = Encoder(embedding_dim=34)\n", + "decoder = Decoder(embedding_dim=34)\n", + "\n", + "# make encoder forward pass\n", + "embedding_tracks, embedding_clusters = encoder(tracks, clusters)\n", + "\n", + "# pooling\n", + "pooled_tracks = global_mean_pool(embedding_tracks, tracks.batch)\n", + "pooled_clusters = global_mean_pool(embedding_clusters, clusters.batch)\n", + "\n", + "# make decoder forward pass\n", + "out_tracks, out_clusters = decoder(pooled_tracks, pooled_clusters)\n", + "\n", + "# compute the loss between the two latent representations\n", + "loss = criterion(out_tracks, out_clusters, device=\"cpu\")\n", + "print(\"loss is: \", loss.item())" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "# train the encoder\n", + "def train_VICReg(encoder, decoder, data, batch_size, lr, epochs, lmbd, u, v):\n", + "\n", + " data_train = data[: int(0.8 * len(data))]\n", + " data_valid = data[int(0.8 * len(data)) :]\n", + "\n", + " train_loader = torch_geometric.loader.DataLoader(data_train, batch_size)\n", + " valid_loader = torch_geometric.loader.DataLoader(data_valid, batch_size)\n", + "\n", + " optimizer = torch.optim.SGD(\n", + " list(encoder.parameters()) + list(decoder.parameters()),\n", + " lr=lr,\n", + " momentum=0.9,\n", + " weight_decay=1.5e-4,\n", + " )\n", + "\n", + " patience = 20\n", + " best_val_loss = 99999.9\n", + " stale_epochs = 0\n", + "\n", + " losses_train, losses_valid = [], []\n", + "\n", + " for epoch in tqdm(range(epochs)):\n", + "\n", + " encoder.train()\n", + " decoder.train()\n", + " loss_train = 0\n", + "\n", + " for batch in tqdm(train_loader):\n", + " # make transformation\n", + " tracks, clusters = distinguish_PFelements(batch)\n", + "\n", + " ### ENCODE\n", + " embedding_tracks, embedding_clusters = encoder(tracks, clusters)\n", + " ### POOLING\n", + " pooled_tracks = global_mean_pool(embedding_tracks, tracks.batch)\n", + " pooled_clusters = global_mean_pool(\n", + " embedding_clusters, clusters.batch\n", + " )\n", + " ### DECODE\n", + " out_tracks, out_clusters = decoder(pooled_tracks, pooled_clusters)\n", + "\n", + " # compute loss\n", + " loss = criterion(out_tracks, out_clusters, \"cpu\", lmbd, u, v)\n", + "\n", + " # update parameters\n", + " for param in encoder.parameters():\n", + " param.grad = None\n", + " for param in decoder.parameters():\n", + " param.grad = None\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " loss_train += loss.detach()\n", + " print(loss)\n", + " encoder.eval()\n", + " decoder.eval()\n", + " loss_valid = 0\n", + " with torch.no_grad():\n", + " for batch in tqdm(valid_loader):\n", + " # make transformation\n", + " tracks, clusters = distinguish_PFelements(batch)\n", + "\n", + " ### ENCODE\n", + " embedding_tracks, embedding_clusters = encoder(\n", + " tracks, clusters\n", + " )\n", + " ### POOLING\n", + " pooled_tracks = global_mean_pool(\n", + " embedding_tracks, tracks.batch\n", + " )\n", + " pooled_clusters = global_mean_pool(\n", + " embedding_clusters, clusters.batch\n", + " )\n", + " ### DECODE\n", + " out_tracks, out_clusters = decoder(\n", + " pooled_tracks, pooled_clusters\n", + " )\n", + "\n", + " # compute loss\n", + " loss = criterion(out_tracks, out_clusters, \"cpu\", lmbd, u, v)\n", + "\n", + " loss_valid += loss.detach()\n", + "\n", + " print(\n", + " f\"epoch {epoch} - loss_train: {round(loss_train.item(),3)} - loss_valid: {round(loss_valid.item(),3)}\"\n", + " )\n", + "\n", + " losses_train.append(loss_train / len(train_loader))\n", + " losses_valid.append(loss_valid / len(valid_loader))\n", + "\n", + " return losses_train, losses_valid" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cc4ed3494d064ba19b03f112e02208ac", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f7bd1fb89fe14e1da085c4ade59e1b6c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=74.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(17.1143, grad_fn=)\n", + "tensor(17.5056, grad_fn=)\n", + "tensor(12.6601, grad_fn=)\n", + "tensor(11.7078, grad_fn=)\n", + "tensor(13.9646, grad_fn=)\n", + "tensor(12.2038, grad_fn=)\n", + "tensor(10.7236, grad_fn=)\n", + "tensor(9.5616, grad_fn=)\n", + "tensor(9.4808, grad_fn=)\n", + "tensor(13.6069, grad_fn=)\n", + "tensor(9.4234, grad_fn=)\n", + "tensor(7.3646, grad_fn=)\n", + "tensor(12.9122, grad_fn=)\n", + "tensor(9.6944, grad_fn=)\n", + "tensor(11.4006, grad_fn=)\n", + "tensor(8.4398, grad_fn=)\n", + "tensor(11.6232, grad_fn=)\n", + "tensor(12.7801, grad_fn=)\n", + "tensor(7.9718, grad_fn=)\n", + "tensor(17.4086, grad_fn=)\n", + "tensor(11.2642, grad_fn=)\n", + "tensor(8.0566, grad_fn=)\n", + "tensor(7.0279, grad_fn=)\n", + "tensor(10.0929, grad_fn=)\n", + "tensor(9.9815, grad_fn=)\n", + "tensor(8.2206, grad_fn=)\n", + "tensor(7.3969, grad_fn=)\n", + "tensor(7.1252, grad_fn=)\n", + "tensor(10.4497, grad_fn=)\n", + "tensor(8.8574, grad_fn=)\n", + "tensor(10.4324, grad_fn=)\n", + "tensor(7.3432, grad_fn=)\n", + "tensor(7.3111, grad_fn=)\n", + "tensor(7.6107, grad_fn=)\n", + "tensor(6.3035, grad_fn=)\n", + "tensor(7.9172, grad_fn=)\n", + "tensor(5.8502, grad_fn=)\n", + "tensor(9.7417, grad_fn=)\n", + "tensor(7.0346, grad_fn=)\n", + "tensor(9.5749, grad_fn=)\n", + "tensor(7.7997, grad_fn=)\n", + "tensor(6.3195, grad_fn=)\n", + "tensor(8.2451, grad_fn=)\n", + "tensor(6.5764, grad_fn=)\n", + "tensor(6.5125, grad_fn=)\n", + "tensor(12.9500, grad_fn=)\n", + "tensor(9.5524, grad_fn=)\n", + "tensor(6.9098, grad_fn=)\n", + "tensor(7.5910, grad_fn=)\n", + "tensor(9.1961, grad_fn=)\n", + "tensor(5.9542, grad_fn=)\n", + "tensor(7.5825, grad_fn=)\n", + "tensor(6.8488, grad_fn=)\n", + "tensor(9.8842, grad_fn=)\n", + "tensor(9.3080, grad_fn=)\n", + "tensor(5.6855, grad_fn=)\n", + "tensor(7.4296, grad_fn=)\n", + "tensor(12.7053, grad_fn=)\n", + "tensor(11.9499, grad_fn=)\n", + "tensor(6.1810, grad_fn=)\n", + "tensor(6.5556, grad_fn=)\n", + "tensor(6.9351, grad_fn=)\n", + "tensor(11.4989, grad_fn=)\n", + "tensor(5.6947, grad_fn=)\n", + "tensor(8.6101, grad_fn=)\n", + "tensor(6.7843, grad_fn=)\n", + "tensor(6.3333, grad_fn=)\n", + "tensor(5.5171, grad_fn=)\n", + "tensor(5.6945, grad_fn=)\n", + "tensor(9.2005, grad_fn=)\n", + "tensor(9.8258, grad_fn=)\n", + "tensor(4.8552, grad_fn=)\n", + "tensor(9.6258, grad_fn=)\n", + "tensor(7.4179, grad_fn=)\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1ecde16403674f6a9e3a31ff598371a4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=19.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 0 - loss_train: 674.87 - loss_valid: 160.031\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1744347efbca4194b1d1820ddec4ad0a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=74.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(7.9516, grad_fn=)\n", + "tensor(5.9534, grad_fn=)\n", + "tensor(5.6981, grad_fn=)\n", + "tensor(5.6513, grad_fn=)\n", + "tensor(11.8971, grad_fn=)\n", + "tensor(5.3641, grad_fn=)\n", + "tensor(8.9031, grad_fn=)\n", + "tensor(6.7965, grad_fn=)\n", + "tensor(15.5520, grad_fn=)\n", + "tensor(9.5906, grad_fn=)\n", + "tensor(6.6310, grad_fn=)\n", + "tensor(7.4972, grad_fn=)\n", + "tensor(17.0604, grad_fn=)\n", + "tensor(6.1025, grad_fn=)\n", + "tensor(17.8324, grad_fn=)\n", + "tensor(6.6777, grad_fn=)\n", + "tensor(5.8728, grad_fn=)\n", + "tensor(7.1072, grad_fn=)\n", + "tensor(6.2918, grad_fn=)\n", + "tensor(10.2833, grad_fn=)\n", + "tensor(8.7856, grad_fn=)\n", + "tensor(7.2900, grad_fn=)\n", + "tensor(4.9020, grad_fn=)\n", + "tensor(6.7362, grad_fn=)\n", + "tensor(6.5049, grad_fn=)\n", + "tensor(8.0068, grad_fn=)\n", + "tensor(6.0271, grad_fn=)\n", + "tensor(5.1885, grad_fn=)\n", + "tensor(6.7262, grad_fn=)\n", + "tensor(10.1436, grad_fn=)\n", + "tensor(7.2118, grad_fn=)\n", + "tensor(5.6213, grad_fn=)\n", + "tensor(6.1551, grad_fn=)\n", + "tensor(5.7825, grad_fn=)\n", + "tensor(6.2268, grad_fn=)\n", + "tensor(4.9155, grad_fn=)\n", + "tensor(5.7651, grad_fn=)\n", + "tensor(7.2264, grad_fn=)\n", + "tensor(7.3183, grad_fn=)\n", + "tensor(10.0508, grad_fn=)\n", + "tensor(8.1063, grad_fn=)\n", + "tensor(6.3154, grad_fn=)\n", + "tensor(7.6218, grad_fn=)\n", + "tensor(4.6924, grad_fn=)\n", + "tensor(5.4368, grad_fn=)\n", + "tensor(15.5651, grad_fn=)\n", + "tensor(6.0718, grad_fn=)\n", + "tensor(5.3050, grad_fn=)\n", + "tensor(5.4519, grad_fn=)\n", + "tensor(14.0827, grad_fn=)\n", + "tensor(8.2136, grad_fn=)\n", + "tensor(7.6124, grad_fn=)\n", + "tensor(4.9019, grad_fn=)\n", + "tensor(9.9467, grad_fn=)\n", + "tensor(8.9388, grad_fn=)\n", + "tensor(4.4099, grad_fn=)\n", + "tensor(5.7022, grad_fn=)\n", + "tensor(12.1328, grad_fn=)\n", + "tensor(11.7464, grad_fn=)\n", + "tensor(6.6856, grad_fn=)\n", + "tensor(5.2094, grad_fn=)\n", + "tensor(5.2663, grad_fn=)\n", + "tensor(10.0325, grad_fn=)\n", + "tensor(5.5281, grad_fn=)\n", + "tensor(8.3660, grad_fn=)\n", + "tensor(7.3165, grad_fn=)\n", + "tensor(5.4561, grad_fn=)\n", + "tensor(5.4833, grad_fn=)\n", + "tensor(5.6383, grad_fn=)\n", + "tensor(7.8111, grad_fn=)\n", + "tensor(8.1833, grad_fn=)\n", + "tensor(4.8039, grad_fn=)\n", + "tensor(9.6894, grad_fn=)\n", + "tensor(5.9399, grad_fn=)\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6481513ac7d24ec2a4180fe1b79c66e2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=19.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 1 - loss_train: 564.962 - loss_valid: 153.639\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "01899ef4c3e2479c9cdafe8198d35af8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=74.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(6.8071, grad_fn=)\n", + "tensor(5.2029, grad_fn=)\n", + "tensor(5.6389, grad_fn=)\n", + "tensor(4.9276, grad_fn=)\n", + "tensor(10.4237, grad_fn=)\n", + "tensor(5.6532, grad_fn=)\n", + "tensor(5.5398, grad_fn=)\n", + "tensor(5.2076, grad_fn=)\n", + "tensor(11.2870, grad_fn=)\n", + "tensor(10.4238, grad_fn=)\n", + "tensor(4.8681, grad_fn=)\n", + "tensor(6.1919, grad_fn=)\n", + "tensor(11.9445, grad_fn=)\n", + "tensor(5.2906, grad_fn=)\n", + "tensor(16.6916, grad_fn=)\n", + "tensor(6.1618, grad_fn=)\n", + "tensor(5.8024, grad_fn=)\n", + "tensor(5.7333, grad_fn=)\n", + "tensor(6.0102, grad_fn=)\n", + "tensor(10.3797, grad_fn=)\n", + "tensor(8.0717, grad_fn=)\n", + "tensor(5.0374, grad_fn=)\n", + "tensor(4.3454, grad_fn=)\n", + "tensor(5.1559, grad_fn=)\n", + "tensor(6.6478, grad_fn=)\n", + "tensor(7.0174, grad_fn=)\n", + "tensor(5.5751, grad_fn=)\n", + "tensor(5.0200, grad_fn=)\n", + "tensor(5.5644, grad_fn=)\n", + "tensor(8.2627, grad_fn=)\n", + "tensor(5.5606, grad_fn=)\n", + "tensor(5.3229, grad_fn=)\n", + "tensor(4.7672, grad_fn=)\n", + "tensor(5.7150, grad_fn=)\n", + "tensor(5.8910, grad_fn=)\n", + "tensor(4.4840, grad_fn=)\n", + "tensor(5.3659, grad_fn=)\n", + "tensor(6.0494, grad_fn=)\n", + "tensor(6.3314, grad_fn=)\n", + "tensor(10.1213, grad_fn=)\n", + "tensor(7.6837, grad_fn=)\n", + "tensor(6.0136, grad_fn=)\n", + "tensor(6.5478, grad_fn=)\n", + "tensor(4.9268, grad_fn=)\n", + "tensor(5.6020, grad_fn=)\n", + "tensor(13.6902, grad_fn=)\n", + "tensor(5.0092, grad_fn=)\n", + "tensor(5.4188, grad_fn=)\n", + "tensor(5.1546, grad_fn=)\n", + "tensor(14.8861, grad_fn=)\n", + "tensor(6.6189, grad_fn=)\n", + "tensor(6.2592, grad_fn=)\n", + "tensor(5.0902, grad_fn=)\n", + "tensor(9.7983, grad_fn=)\n", + "tensor(7.2085, grad_fn=)\n", + "tensor(4.0844, grad_fn=)\n", + "tensor(5.1873, grad_fn=)\n", + "tensor(10.7753, grad_fn=)\n", + "tensor(9.3382, grad_fn=)\n", + "tensor(6.6640, grad_fn=)\n", + "tensor(4.5954, grad_fn=)\n", + "tensor(5.2793, grad_fn=)\n", + "tensor(11.3696, grad_fn=)\n", + "tensor(4.9745, grad_fn=)\n", + "tensor(6.8131, grad_fn=)\n", + "tensor(6.5148, grad_fn=)\n", + "tensor(4.3709, grad_fn=)\n", + "tensor(5.5442, grad_fn=)\n", + "tensor(5.3998, grad_fn=)\n", + "tensor(8.5223, grad_fn=)\n", + "tensor(7.6515, grad_fn=)\n", + "tensor(4.6986, grad_fn=)\n", + "tensor(9.2258, grad_fn=)\n", + "tensor(5.3591, grad_fn=)\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d9340f64fabf4260a750358902247ffb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=19.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 2 - loss_train: 506.768 - loss_valid: 143.7\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9d54822ad1df4c479d06202ace8495da", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=74.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(7.0911, grad_fn=)\n", + "tensor(4.6317, grad_fn=)\n", + "tensor(4.8988, grad_fn=)\n", + "tensor(4.8732, grad_fn=)\n", + "tensor(10.4070, grad_fn=)\n", + "tensor(4.7845, grad_fn=)\n", + "tensor(4.8295, grad_fn=)\n", + "tensor(4.8145, grad_fn=)\n", + "tensor(11.7755, grad_fn=)\n", + "tensor(9.4923, grad_fn=)\n", + "tensor(4.6781, grad_fn=)\n", + "tensor(5.4411, grad_fn=)\n", + "tensor(10.8914, grad_fn=)\n", + "tensor(4.7082, grad_fn=)\n", + "tensor(15.1668, grad_fn=)\n", + "tensor(6.2218, grad_fn=)\n", + "tensor(4.8269, grad_fn=)\n", + "tensor(5.8758, grad_fn=)\n", + "tensor(5.7826, grad_fn=)\n", + "tensor(8.3913, grad_fn=)\n", + "tensor(6.9584, grad_fn=)\n", + "tensor(4.6140, grad_fn=)\n", + "tensor(4.2160, grad_fn=)\n", + "tensor(4.5247, grad_fn=)\n", + "tensor(5.6797, grad_fn=)\n", + "tensor(8.0428, grad_fn=)\n", + "tensor(5.4294, grad_fn=)\n", + "tensor(4.3584, grad_fn=)\n", + "tensor(5.3933, grad_fn=)\n", + "tensor(8.3035, grad_fn=)\n", + "tensor(4.5199, grad_fn=)\n", + "tensor(4.6374, grad_fn=)\n", + "tensor(4.4694, grad_fn=)\n", + "tensor(5.5490, grad_fn=)\n", + "tensor(5.5473, grad_fn=)\n", + "tensor(4.4320, grad_fn=)\n", + "tensor(5.2649, grad_fn=)\n", + "tensor(5.4595, grad_fn=)\n", + "tensor(5.9638, grad_fn=)\n", + "tensor(9.1573, grad_fn=)\n", + "tensor(7.2642, grad_fn=)\n", + "tensor(6.0178, grad_fn=)\n", + "tensor(6.7047, grad_fn=)\n", + "tensor(4.7553, grad_fn=)\n", + "tensor(5.0288, grad_fn=)\n", + "tensor(13.2631, grad_fn=)\n", + "tensor(5.1826, grad_fn=)\n", + "tensor(5.2226, grad_fn=)\n", + "tensor(4.6285, grad_fn=)\n", + "tensor(13.5988, grad_fn=)\n", + "tensor(6.2188, grad_fn=)\n", + "tensor(5.7213, grad_fn=)\n", + "tensor(4.5609, grad_fn=)\n", + "tensor(12.5483, grad_fn=)\n", + "tensor(6.3327, grad_fn=)\n", + "tensor(4.0822, grad_fn=)\n", + "tensor(4.3821, grad_fn=)\n", + "tensor(8.5776, grad_fn=)\n", + "tensor(8.0046, grad_fn=)\n", + "tensor(6.2709, grad_fn=)\n", + "tensor(4.3207, grad_fn=)\n", + "tensor(4.4026, grad_fn=)\n", + "tensor(10.2983, grad_fn=)\n", + "tensor(4.4839, grad_fn=)\n", + "tensor(6.4809, grad_fn=)\n", + "tensor(7.3265, grad_fn=)\n", + "tensor(4.2703, grad_fn=)\n", + "tensor(5.3037, grad_fn=)\n", + "tensor(5.1118, grad_fn=)\n", + "tensor(7.6732, grad_fn=)\n", + "tensor(7.7848, grad_fn=)\n", + "tensor(4.3186, grad_fn=)\n", + "tensor(8.6014, grad_fn=)\n", + "tensor(5.4774, grad_fn=)\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "422acbcade4c477db6ce488c4a944a8f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=19.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 3 - loss_train: 476.323 - loss_valid: 145.527\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5ff0704b81fb40ca8bf4dd2646a60697", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=74.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(5.8876, grad_fn=)\n", + "tensor(4.2191, grad_fn=)\n", + "tensor(5.0486, grad_fn=)\n", + "tensor(4.2879, grad_fn=)\n", + "tensor(8.7557, grad_fn=)\n", + "tensor(4.2077, grad_fn=)\n", + "tensor(5.4648, grad_fn=)\n", + "tensor(4.2296, grad_fn=)\n", + "tensor(11.3488, grad_fn=)\n", + "tensor(11.4836, grad_fn=)\n", + "tensor(5.2353, grad_fn=)\n", + "tensor(4.7876, grad_fn=)\n", + "tensor(10.4500, grad_fn=)\n", + "tensor(4.1362, grad_fn=)\n", + "tensor(14.3191, grad_fn=)\n", + "tensor(5.5210, grad_fn=)\n", + "tensor(4.2720, grad_fn=)\n", + "tensor(5.6319, grad_fn=)\n", + "tensor(6.6013, grad_fn=)\n", + "tensor(7.5529, grad_fn=)\n", + "tensor(6.5096, grad_fn=)\n", + "tensor(4.9283, grad_fn=)\n", + "tensor(4.2874, grad_fn=)\n", + "tensor(4.7524, grad_fn=)\n", + "tensor(5.7788, grad_fn=)\n", + "tensor(7.6269, grad_fn=)\n", + "tensor(4.9446, grad_fn=)\n", + "tensor(3.8344, grad_fn=)\n", + "tensor(5.0345, grad_fn=)\n", + "tensor(6.4814, grad_fn=)\n", + "tensor(4.8209, grad_fn=)\n", + "tensor(5.0424, grad_fn=)\n", + "tensor(4.3128, grad_fn=)\n", + "tensor(6.0068, grad_fn=)\n", + "tensor(5.3745, grad_fn=)\n", + "tensor(4.4638, grad_fn=)\n", + "tensor(4.9197, grad_fn=)\n", + "tensor(4.7087, grad_fn=)\n", + "tensor(6.2417, grad_fn=)\n", + "tensor(8.1741, grad_fn=)\n", + "tensor(5.8463, grad_fn=)\n", + "tensor(7.2341, grad_fn=)\n", + "tensor(5.5280, grad_fn=)\n", + "tensor(4.5958, grad_fn=)\n", + "tensor(4.8016, grad_fn=)\n", + "tensor(14.7080, grad_fn=)\n", + "tensor(4.6235, grad_fn=)\n", + "tensor(4.3479, grad_fn=)\n", + "tensor(5.6104, grad_fn=)\n", + "tensor(15.9084, grad_fn=)\n", + "tensor(6.7198, grad_fn=)\n", + "tensor(5.4069, grad_fn=)\n", + "tensor(4.4367, grad_fn=)\n", + "tensor(13.7286, grad_fn=)\n", + "tensor(5.2922, grad_fn=)\n", + "tensor(4.3664, grad_fn=)\n", + "tensor(4.4716, grad_fn=)\n", + "tensor(7.1443, grad_fn=)\n", + "tensor(7.3815, grad_fn=)\n", + "tensor(5.4717, grad_fn=)\n", + "tensor(4.5381, grad_fn=)\n", + "tensor(4.4019, grad_fn=)\n", + "tensor(10.3668, grad_fn=)\n", + "tensor(4.2622, grad_fn=)\n", + "tensor(6.4107, grad_fn=)\n", + "tensor(6.6816, grad_fn=)\n", + "tensor(4.2128, grad_fn=)\n", + "tensor(5.0786, grad_fn=)\n", + "tensor(4.6933, grad_fn=)\n", + "tensor(7.3246, grad_fn=)\n", + "tensor(6.8033, grad_fn=)\n", + "tensor(4.7040, grad_fn=)\n", + "tensor(9.1092, grad_fn=)\n", + "tensor(5.2673, grad_fn=)\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f2c0baa94c3c4773b7a917da92f1a311", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=19.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 4 - loss_train: 463.162 - loss_valid: 143.295\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c4d0b22e5240470fa45c52edbf508f66", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=74.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(5.9518, grad_fn=)\n", + "tensor(3.9519, grad_fn=)\n", + "tensor(4.7261, grad_fn=)\n", + "tensor(4.3481, grad_fn=)\n", + "tensor(8.9319, grad_fn=)\n", + "tensor(4.0497, grad_fn=)\n", + "tensor(5.4410, grad_fn=)\n", + "tensor(3.9214, grad_fn=)\n", + "tensor(10.8746, grad_fn=)\n", + "tensor(10.2818, grad_fn=)\n", + "tensor(4.7808, grad_fn=)\n", + "tensor(4.3973, grad_fn=)\n", + "tensor(8.7418, grad_fn=)\n", + "tensor(3.7351, grad_fn=)\n", + "tensor(14.0835, grad_fn=)\n", + "tensor(4.7531, grad_fn=)\n", + "tensor(4.1437, grad_fn=)\n", + "tensor(5.4919, grad_fn=)\n", + "tensor(7.3417, grad_fn=)\n", + "tensor(7.7751, grad_fn=)\n", + "tensor(5.2229, grad_fn=)\n", + "tensor(4.8788, grad_fn=)\n", + "tensor(5.2837, grad_fn=)\n", + "tensor(4.5515, grad_fn=)\n", + "tensor(5.5976, grad_fn=)\n", + "tensor(9.2228, grad_fn=)\n", + "tensor(5.7124, grad_fn=)\n", + "tensor(4.0172, grad_fn=)\n", + "tensor(5.1693, grad_fn=)\n", + "tensor(5.7995, grad_fn=)\n", + "tensor(5.4830, grad_fn=)\n", + "tensor(6.4520, grad_fn=)\n", + "tensor(4.2117, grad_fn=)\n", + "tensor(5.6656, grad_fn=)\n", + "tensor(5.1777, grad_fn=)\n", + "tensor(4.5500, grad_fn=)\n", + "tensor(4.8177, grad_fn=)\n", + "tensor(4.2737, grad_fn=)\n", + "tensor(5.5633, grad_fn=)\n", + "tensor(8.1340, grad_fn=)\n", + "tensor(4.7180, grad_fn=)\n", + "tensor(7.6666, grad_fn=)\n", + "tensor(4.8805, grad_fn=)\n", + "tensor(4.4371, grad_fn=)\n", + "tensor(5.1130, grad_fn=)\n", + "tensor(14.3936, grad_fn=)\n", + "tensor(4.3380, grad_fn=)\n", + "tensor(4.1308, grad_fn=)\n", + "tensor(5.1693, grad_fn=)\n", + "tensor(15.2911, grad_fn=)\n", + "tensor(6.1610, grad_fn=)\n", + "tensor(6.4991, grad_fn=)\n", + "tensor(4.0511, grad_fn=)\n", + "tensor(15.6772, grad_fn=)\n", + "tensor(6.5448, grad_fn=)\n", + "tensor(5.3772, grad_fn=)\n", + "tensor(4.3625, grad_fn=)\n", + "tensor(9.4824, grad_fn=)\n", + "tensor(6.9498, grad_fn=)\n", + "tensor(5.1893, grad_fn=)\n", + "tensor(4.7492, grad_fn=)\n", + "tensor(4.2938, grad_fn=)\n", + "tensor(7.7459, grad_fn=)\n", + "tensor(4.1500, grad_fn=)\n", + "tensor(5.8344, grad_fn=)\n", + "tensor(5.2791, grad_fn=)\n", + "tensor(4.2294, grad_fn=)\n", + "tensor(4.3810, grad_fn=)\n", + "tensor(4.6488, grad_fn=)\n", + "tensor(7.4250, grad_fn=)\n", + "tensor(6.0616, grad_fn=)\n", + "tensor(4.1789, grad_fn=)\n", + "tensor(10.1346, grad_fn=)\n", + "tensor(4.4683, grad_fn=)\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a22455275e174992ad94c1724540e3fe", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=19.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 5 - loss_train: 455.519 - loss_valid: 138.543\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "235589a9941147ccb14a420709b96fcd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=74.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(6.2601, grad_fn=)\n", + "tensor(3.9866, grad_fn=)\n", + "tensor(5.0616, grad_fn=)\n", + "tensor(4.1970, grad_fn=)\n", + "tensor(7.8287, grad_fn=)\n", + "tensor(3.8438, grad_fn=)\n", + "tensor(5.2745, grad_fn=)\n", + "tensor(3.8651, grad_fn=)\n", + "tensor(9.0998, grad_fn=)\n", + "tensor(10.5051, grad_fn=)\n", + "tensor(4.2912, grad_fn=)\n", + "tensor(3.8415, grad_fn=)\n", + "tensor(6.6632, grad_fn=)\n", + "tensor(4.2922, grad_fn=)\n", + "tensor(16.8637, grad_fn=)\n", + "tensor(5.2063, grad_fn=)\n", + "tensor(4.6022, grad_fn=)\n", + "tensor(5.2184, grad_fn=)\n", + "tensor(8.3808, grad_fn=)\n", + "tensor(5.6825, grad_fn=)\n", + "tensor(6.4019, grad_fn=)\n", + "tensor(5.4236, grad_fn=)\n", + "tensor(4.5919, grad_fn=)\n", + "tensor(5.2249, grad_fn=)\n", + "tensor(4.6262, grad_fn=)\n", + "tensor(10.6848, grad_fn=)\n", + "tensor(5.8857, grad_fn=)\n", + "tensor(4.6118, grad_fn=)\n", + "tensor(5.4918, grad_fn=)\n", + "tensor(7.2597, grad_fn=)\n", + "tensor(5.9254, grad_fn=)\n", + "tensor(7.6503, grad_fn=)\n", + "tensor(4.4205, grad_fn=)\n", + "tensor(6.1311, grad_fn=)\n", + "tensor(5.3046, grad_fn=)\n", + "tensor(3.9042, grad_fn=)\n", + "tensor(4.8742, grad_fn=)\n", + "tensor(4.0804, grad_fn=)\n", + "tensor(7.1725, grad_fn=)\n", + "tensor(10.6229, grad_fn=)\n", + "tensor(5.4148, grad_fn=)\n", + "tensor(7.1883, grad_fn=)\n", + "tensor(4.7707, grad_fn=)\n", + "tensor(4.7888, grad_fn=)\n", + "tensor(6.2957, grad_fn=)\n", + "tensor(12.6100, grad_fn=)\n", + "tensor(4.4697, grad_fn=)\n", + "tensor(4.2302, grad_fn=)\n", + "tensor(4.9086, grad_fn=)\n", + "tensor(14.3046, grad_fn=)\n", + "tensor(5.7869, grad_fn=)\n", + "tensor(6.7747, grad_fn=)\n", + "tensor(3.9423, grad_fn=)\n", + "tensor(14.9057, grad_fn=)\n", + "tensor(5.8263, grad_fn=)\n", + "tensor(4.1720, grad_fn=)\n", + "tensor(4.3226, grad_fn=)\n", + "tensor(7.4884, grad_fn=)\n", + "tensor(5.6814, grad_fn=)\n", + "tensor(5.3383, grad_fn=)\n", + "tensor(3.9539, grad_fn=)\n", + "tensor(4.4185, grad_fn=)\n", + "tensor(4.1187, grad_fn=)\n", + "tensor(4.1605, grad_fn=)\n", + "tensor(5.8989, grad_fn=)\n", + "tensor(4.1717, grad_fn=)\n", + "tensor(3.9611, grad_fn=)\n", + "tensor(3.8483, grad_fn=)\n", + "tensor(4.2664, grad_fn=)\n", + "tensor(6.0762, grad_fn=)\n", + "tensor(7.9711, grad_fn=)\n", + "tensor(4.2189, grad_fn=)\n", + "tensor(12.6181, grad_fn=)\n", + "tensor(3.9903, grad_fn=)\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "827debf854eb4a8781fc6c688a721fe4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=19.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 6 - loss_train: 452.145 - loss_valid: 141.36\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e0296fb736e3460aa9412e3260a2f2c3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=74.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tensor(5.3423, grad_fn=)\n", + "tensor(3.9018, grad_fn=)\n", + "tensor(4.2212, grad_fn=)\n", + "tensor(4.4765, grad_fn=)\n", + "tensor(9.8165, grad_fn=)\n", + "tensor(3.8228, grad_fn=)\n", + "tensor(4.7231, grad_fn=)\n", + "tensor(4.0622, grad_fn=)\n", + "tensor(9.0704, grad_fn=)\n", + "tensor(10.3249, grad_fn=)\n", + "tensor(4.3950, grad_fn=)\n", + "tensor(3.7458, grad_fn=)\n", + "tensor(5.0440, grad_fn=)\n", + "tensor(4.1995, grad_fn=)\n", + "tensor(14.3149, grad_fn=)\n", + "tensor(5.0848, grad_fn=)\n", + "tensor(4.0687, grad_fn=)\n", + "tensor(5.6892, grad_fn=)\n", + "tensor(7.7981, grad_fn=)\n", + "tensor(5.5955, grad_fn=)\n", + "tensor(6.9694, grad_fn=)\n", + "tensor(4.8980, grad_fn=)\n", + "tensor(4.0116, grad_fn=)\n", + "tensor(4.3682, grad_fn=)\n", + "tensor(4.6536, grad_fn=)\n", + "tensor(5.7745, grad_fn=)\n", + "tensor(4.9864, grad_fn=)\n", + "tensor(4.4262, grad_fn=)\n", + "tensor(5.5547, grad_fn=)\n", + "tensor(6.4315, grad_fn=)\n", + "tensor(5.3349, grad_fn=)\n", + "tensor(9.3308, grad_fn=)\n", + "tensor(4.6466, grad_fn=)\n", + "tensor(7.4837, grad_fn=)\n", + "tensor(4.2786, grad_fn=)\n", + "tensor(3.7314, grad_fn=)\n", + "tensor(4.3199, grad_fn=)\n", + "tensor(4.0516, grad_fn=)\n", + "tensor(5.6179, grad_fn=)\n", + "tensor(9.3199, grad_fn=)\n", + "tensor(4.4288, grad_fn=)\n", + "tensor(6.6905, grad_fn=)\n", + "tensor(4.8315, grad_fn=)\n", + "tensor(4.3175, grad_fn=)\n", + "tensor(5.3807, grad_fn=)\n", + "tensor(9.9398, grad_fn=)\n", + "tensor(3.9714, grad_fn=)\n", + "tensor(3.9950, grad_fn=)\n", + "tensor(5.5337, grad_fn=)\n", + "tensor(12.8955, grad_fn=)\n", + "tensor(6.4362, grad_fn=)\n", + "tensor(5.6140, grad_fn=)\n", + "tensor(3.9075, grad_fn=)\n", + "tensor(13.0102, grad_fn=)\n", + "tensor(5.5787, grad_fn=)\n", + "tensor(3.6340, grad_fn=)\n", + "tensor(4.2888, grad_fn=)\n", + "tensor(6.2230, grad_fn=)\n", + "tensor(6.2438, grad_fn=)\n", + "\n", + "\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0mencoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membedding_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m34\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mdecoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDecoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membedding_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m34\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mlosses_train_VICRreg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlosses_valid_VICRreg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_VICReg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecoder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlmbd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mu\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_VICReg\u001b[0;34m(encoder, decoder, data, batch_size, lr, epochs, lmbd, u, v)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0;31m### ENCODE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m \u001b[0membedding_tracks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding_clusters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mencoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtracks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclusters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 34\u001b[0m \u001b[0;31m### POOLING\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 35\u001b[0m \u001b[0mpooled_tracks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mglobal_mean_pool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membedding_tracks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, tracks, clusters)\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclusters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m \u001b[0membedding_tracks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtracks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 42\u001b[0m \u001b[0membedding_clusters\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclusters\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 140\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m 1845\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhas_torch_function_variadic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1846\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mhandle_torch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1847\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1848\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "batch_size = 100\n", + "lr = 1e-4\n", + "epochs = 100\n", + "lmbd, u, v = 25, 25, 1 # VICReg paper\n", + "lmbd, u, v = 5e-3, 1, 1 # stable\n", + "\n", + "lmbd, u, v = 0.1, 10, 10\n", + "lmbd, u, v = 0.1, 1, 0.1 # VICReg paper\n", + "\n", + "\n", + "encoder = Encoder(embedding_dim=34)\n", + "decoder = Decoder(embedding_dim=34)\n", + "losses_train_VICRreg, losses_valid_VICRreg = train_VICReg(\n", + " encoder, decoder, data, batch_size, lr, epochs, lmbd, u, v\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "# save the VICReg\n", + "torch.save(encoder.state_dict(), \"ssl/encoder.pth\")\n", + "torch.save(decoder.state_dict(), \"ssl/decoder.pth\")" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe4AAAF1CAYAAAAutgnWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAAsTAAALEwEAmpwYAABWgElEQVR4nO3deXiU1fn/8ffJvu8QIOybgKgoiKAg4r6hUndrFa3SulT8amttbdXaX6titWpbq9BWrcUVcYlarRsqIiII7iD7FiAkkH2d5Pz+OJOQlSxMZjLM53Vdc03yPPM8czKl3nOf5T7GWouIiIgEh7BAN0BERETaT4FbREQkiChwi4iIBBEFbhERkSCiwC0iIhJEIgLdgH0xxmjKu4iIhCRrrWnpuDJuERGRIBIUgdta69PH2LFjfXq/q6++2udtDJZ76rPsnp9jsPzdwfBvUp9l6P3dgf4s29Ktu8rrzJw5s9mxadOmMW3atAC0RkRExDeys7PJzs7u0DX7FbiNMYOArdba6v25T1vmzJnTlbcXEREJiJaS0Llz5+7zmnZ3lRtj+hlj5htjTvP+/h9gHZBjjDm8480VERGRjurIGPcjwAhgszFmKHA+cArwBnBPF7RNREREmuhIV/lk4Dxr7TfGmNuAt621bxtjKoGXuqZ5waErxtqD5Z6+Fix/tz7L7nm/rqLP0neC4e/u7p+lac8MNgBjTD5wvrX2PWPMB8Ar1toHjDFTgNestYk+b5x3HXd729he48aNY9myZT69Z6jSZ+kb+hx9R5+l7+iz9J2OfJbGuOXbtpV13B3JuD8G/mCMeQUYD5xvjOkL3AB824H7iIiISCd1ZIz7RiAcuA2Yba3NBWYBJwC3+r5pIiIi0lS7M25r7Xpcpt3QPcBvrLWVPm2ViIiItKhDldOMMYcYY/p5f54MPABcY+o65EVERKRLdWQd90+BlcAEY0wk8BwwGPg18NsuaV0XaakSm3SOPkvf0OfoO/osfUefpe/48rPsyKzytcBTwF3Aebhu8qHApcAd1tqhPmvV3ve0AFdffXWzcwd0ydOKIqgsguS+gW6JiIh0oZZKntZVTmttVnlHAnc5MMlau9wYMweostZe762atthaG7s/jW/lPbtkOVi399Zt8P2b8LPlgW6JiIj4WVvLwToyxr0ZONYYEwtMB172Hj8M2N35JkozpXlQkhvoVohIiFm0aBHGGCZPnrzP13300UcYYzjhhBMAmDFjBsYYFi5c2OLrFy9ezI9+9CMGDBhATEwMAwYM4KSTTuKll15qMTGru19Lj8TERMaMGcNdd91FaWnpfv/Nwagj67j/CjyEG8/eZa19xxhzJfAw8EQXtC10eSqgqhSsBc37ExE/Ofroo+nfvz+LFi1i27ZtZGVltfi6BQsWAHDhhRfu837WWm699VZmz55NeHg4Y8aMYerUqezcuZMPP/yQd955h/PPP5/nnnuOluY4n3POOY3aUFtby/bt2/nggw+44447+N///scHH3xAeHj4fvzVwacjy8H+YoxZB4wEFngPlwO/Bx70fdNCmKcSbA3UVEFEdKBbIyIhIiwsjIsuuojZs2czf/58Zs2a1ew11lpeeuklwsPDmT59+j7vN3v2bGbPns2IESN47bXXGDJkSP25nTt3cu655/LCCy8wYcIEbrrppmbXz5o1i+OOO67Z8by8PI488kg+/vhjFixYwPnnn9/xPzaIdWg5mLX2DWvt/dbaDd7fn7HW3qt13D7mqXDP1WWBbYeIhJxLLrkEgOeee67F8ytWrGDTpk2ccMIJ9OjRo9X7bNmyhTvuuIOUlBQ+/vjjRkEbIDMzkxdeeIGIiAgeeuihDs1lysjI4MorrwTg888/b/d1B4qOruM+0xjzgTEm1xiz2xizyBhzVlc1LmR5vN+DqhS4RcS/Dj30UEaOHMknn3zC5s2bm52v6ya/4IIL9nmfxx9/nMrKSm644QbS0tJafE3v3r255ppr6NevHxs2bOhQO+vuWVJS0uh4ZWUlv/nNbxg3bhzx8fEMHjyYm2++mV27djW7x44dO7j88ssZPHgwWVlZXHbZZezcuZMzzzyToUN9vlDKZzqyjvsC3C5ga4CbgZ8B3wHzvefac48wY8zLTY6lGmPeN8Z84R0zl/qMuzyw7RCRkGOMqc+658+f3+z8Sy+9REREBOecc84+7/PZZ58BezP41jz88MMsWrSIwYMHd6idX3zxBQDjx+8t6FlSUsLEiRP5wx/+AMB5551HWloaDzzwAEcffTQ7d+6sf+369esZO3Ys//73v+nfvz9Tpkzh3XffZerUqeTmdvPJwdbadj2Az4FftnD8l8CKdlw/APgGWNXk+PXADNyXiCVNzlnXxBDzt4nW3pFkbc7KQLdERELQ2rVrLWCPOuqoRsdXrVplAXvaaac1On755ZdbwL7//vv1x0aMGGEBW1FR0eH3b+l+1lpbW1trd+zYYf/2t7/Z8PBwO3ny5Eb3v/322y1gH3nkEVtbW1t/zV//+lcL2B//+Mf1r73kkkssYJ9//vn6YwUFBXbChAkWsEOGDOlwu32lQexrMZ52pKt8GPBWC8ffwhViactm3NKxTU2OlwFpQLy3sVKXcaurXEQCYMiQIYwfP55PP/2UjRs31h9/6aWXgLa7yQE2bdpEz549iY7u/ATbqVOnNloKFhYWRq9evbjuuuu48MILeeutt+rvb63loYce4phjjuGaa66pn6VujOHaa6/lqKOOYv78+dTW1pKXl8czzzzDiSee2GhiW3JyMnfffXen2+svHVkOtgk4Elf2tKEjcUF5n7zfIjx1RVUaWACsxe069kBL144bN67djZw5c2bwl+mrG+PW5DQRCZBLLrmEpUuX8sILL/CLX/wCcOPbkZGRbXaTg5tAtn37djweDxERHQk1ezVdDmatJT8/nzfffJPnn3+e0047jUsvvRRw49WFhYUUFRVx/fXXN7tXUVERhYWF5OTksHnzZqy1Lc5YnzRpEmFhHZr+1S5z5sxhzpw5PrlXRz7NOcADxpiewEfeY5NxXeW/2Y823A38AFgKZBtjHrPW5jV8Qcht5O7xjm0rcItIgFxwwQXcdNNNPP/88/ziF79gy5YtfPbZZ5x55pmkpKS0ef2wYcPYsmULW7ZsYdCgQa2+7pNPPuHFF1/k+OOP5/TTT290rrXlYBs3bmTkyJH86le/4oc//CHGmPqJdF999RVfffVVq+9XXFzMli1bADezvamIiIh9zpbvrI4klW3t29WRrxV/wQXoa4CF3se1wJ24oN5ZaUAuUAmUAkn7ca8Dg2aVi0iA9e7dm6lTp7Js2TLWr1/Pyy+/DLSvmxxg9OjRALz66qv7fN0//vEP7r//fsrL2z8Zd+DAgUyZMoWtW7dSXFxc316AG2+8cZ/zrUaOHEmvXr0AWpyEVlNTQ35+frvbEgjtDtzeMfOHrLV9gRQgxVqbBXyMC7gdYoy53RgzCvgDbvOSFcA31u37Hdq0jltEuoG6GeHPP/88CxYsICoqirPPPrtd115xxRUA/PGPf6wPrk3l5+fzyiuvYIxhypQpHWpbXbZcWFgIQN++fYmJieGzzz5rcU34448/ziOPPALA8OHDAfjwww+bvW7p0qV4PJ4OtcXfOtWRb60tstYWdfLaU73Pd1lrv7XWfmmtPdJaO8Zae1tn7nlAqfFArfcfjQK3iATQD37wA6KiovjnP//Jhx9+yGmnnUZSUvs6RceMGcM111xDbm4ukyZNYs2aNY3O7969mxkzZpCfn8/NN99MRkZGh9oWGRkJuLFtcFXfZs6cyccff8y//vWvRq996623uPLKK1m5ciXgsvOzzjqLt956ixdffLH+dSUlJdx6660dakcgdG7GgHSdmgZF6NRVLiIBlJKSwumnn17fTd5WbfKmHnzwQQoKCnjmmWc4+OCDOeSQQxg9ejR5eXl88skn7Nmzh+OOO47f//73HW7byJEjAVizZg1HHnkkAL/97W954403uOqqq5g7dy6jRo1i69atvPPOOwwcOLDR+9x77718/PHHnH/++UyZMoU+ffrw0UcfkZWVxYQJE6iuru5wm/zF91PnZP94GgRuZdwiEmB13eUxMTGceeaZHbo2KiqKefPm8frrr3PGGWewfft2nn32Wb799lvGjRvHvHnzeO+994iJielwu4444ggAfv7zn5OTkwO4meyff/45s2bNory8nGeffZYNGzZw/fXXs2TJkkaT0UaMGMHy5cs577zzWLVqFZ9++innnHMO77zzDh6Pp34cvDtq937crd7AmKNw+3H7fHuWkNyPuygHHnDfJDnqp3DavYFtj4jIAcZay7p164iLi6NPnz6NzlVWVtKrVy/OO+885s6dG5D2tbUf9z67yo0x/9rXeS/fz5sPZXUT08Bt7SkiIj5ljOH000+npKSE1atXk5iYWH/uoYceoqCgoN2z5wOhrTHu1hffNdZ8ap50TqOuctUqFxHpCrfddhszZszg0EMP5eSTTyYrK4ulS5fy+uuvM27cOE488cRAN7FV+wzc1tqp/mqIeDXMuDXGLSLSJS6//HLS0tK4//77WbBgAeXl5QwdOpRrr72W3/3ud20WQQmkoJhV3lK1mWnTpjFt2rQAtKaLNcy41VUuItJlukMcyc7OJjs7u0PX7PfktK4UkpPT1i+Ef58N4dHQ+zC46u1At0hERPyorclpWg7W3dRl3HFp6ioXEZFmFLi7m7ox7tg0dZWLiEgzCtzdTaOMW7PKRUSkMQXu7qY+405VV7mIiDQTWoHbWijNg4pO7Y/iHw0z7qpS12YRERGv0ArcNdVw3xBY+ligW9K6hhm3rXFtFhER8QqpwF1tIqgJi2b37rxAN6V1DSenAVRrgpqIhLY777yzQwVRBg4cyIwZM7quQQEWUoEbYHdNDFu35wa6Ga3zVIIJhxjvnrfa2lNEgty4ceO48847O339DTfc0Gw/731ZuHAh99574G7QFFKV0yLDwygzcdjKQl81zfeqyyEiBiLj9v4uIhLC0tLSSEtLa/frBw4c2HWN8bHOVE7DWtttH4B1TfSdVb87wn5574k+vadPvXaztfcMtPbbbGvvSLI2Z2WgWyQi0mlTpkyxdf8tHzBggJ0yZYq955577PXXX2+TkpLsrl27bElJib3qqqtsZmamjYqKsoMGDbL33HOPra2ttdZa+/jjj9vk5GRrrbUbNmywgP3666/t2WefbVNSUuzgwYPt008/3eg9Z82aZa219o477rCnnHKKfe211+zYsWNtfHy8Pf744+369evrX79y5Up77LHH2qSkJDthwgS7fPlyC9gVK1b462NqpEHsazE2BkXG7UtVEQlEVpcEuhmt81R4M+5Y97u6ykWkA36X/Q3f5nTNyplRfZK4Y9rBHbpm3rx5nHLKKRx//PHcdNNNzJgxgz//+c9ceOGF/Pe//yU1NZXbbruNBQsW8MgjjzB48GDeffddbr31ViZOnMixxx7b4n2vvPJKrrvuOm666SbuuusuLrvsMk499VRSU1Obvfbbb7/l4Ycf5sEHH2TTpk1cf/313HLLLbzwwgvk5uYyefJkzjvvPO677z7WrFnDueee26nPx19CLnDXRiUSVbop0M1onacSIqIhKt79rrXcIhLEsrKyiImJIS0trb4Le8CAATz44IP1E87Gjh3LlClTOO200wAYM2YMd911F+vXr281cF9wwQVcdtllAPzxj3/kqKOOYtOmTS0G7u3bt7N06VJ69erFpEmT+Pjjj1m8eDEAf//73+nVqxdz584lPDyc8ePHk5+fz6xZs3z9UfhM6AXu6ERii7vxTO36jLtujFuBW0Tar6MZcSCMGTOm0Szx888/n6+//pqnnnqKL774gvfee4/y8n3P75k4cWL9zz179tznawcNGkSvXr1afP1XX33F0UcfTXh4eP2xSZMmtftvCYSQm1UeFpNMPGWUV9UEuiktq8u46wK3uspF5AATGxvb6Pdbb72VKVOm8MEHH3D44Yczb968FjPnhuLi4tr9fvt6rcfjaXasYRDvjkIu446ITSaBCrYVl9MvPSHQzWmuLuOOUsYtIge+kpISHnjgAV599VVOPfVUAAoLCykuLvbL+48aNYoXX3yR2tpawsJcLlvXjd5dhVzGHZWQTJix7N6zO9BNaVnTjFuBW0SCXHh4OGvWrGHr1q3NzkVFRREREcGTTz7J0qVLeeqppzj99NOpra1lxYoVFBV1bYnqa6+9li1btjBz5kw+++wznnvuOf7yl7/Ut7s7CrnAHZPgul8KC7pr4NYYt4gcWGbOnMnbb7/N6aef3uxcVFQUzzzzDJ9//jlTp07lL3/5C7feeiv33Xcf8+bN47PPPuvStvXt25f333+fr7/+mhNOOIHHHnuMefPmAW2PnQeKsd14EwtjjFvM7cM27l76LGlv/ITXJ7/EGScc77P7+szfJkDGMLjwKbgrHY6+AU68I9CtEhE5IK1evZqNGzdyyimn1B9btGgRJ510EiUlJQHJuusm7llrW6zzGhRj3L6qnAaQkOSq75QVdfOMGyAyXhm3iEgXysvL44wzzuAf//gHp556Ktu3b2fWrFnMmDHDL0G7M5XTgiJwz5kzx2f3iop3XeUVxXt8dk+fqhvjBleERYFbRKTLHHPMMTzxxBPcfffd/PSnP6VHjx6cccYZfqt13lISOnfu3H1eExSB26eiEwGoKuum9cobZtxRcVoOJiLSxS699FIuvfTSQDej3UJuclrdrls15d01cDfMuNVVLiIijYVe4I52gdtWdNfA3XCMW13lIiLSWOgF7qh4agnDVHbDjUZqPGBrIFJd5SIi0rLQC9zGUBURT6SnhCpPbaBb05inwj1rVrmIiLQi9AI34IlMJMmUkV9aGeimNObxtkdd5SIi0gq/Bm5jTJgx5uUmx8KNMY8aY1YYY27wRztsVAIJlJNXXOWPt2u/+ozbOzlNXeUiItKE3wK3MWYA8BUwosmpE4EiYCxwkjGm61e8xySTSBl5Jd0t41ZXuYhInY0bN2KMYeXKlQAMHDiQGTNmtPr6J554gpSUlE6/34wZM+r3DO/O/LmOezNwGPB6k+NTgI+stbXGGL8spIuITSbR7GJrtw3cKsAiItLUwoULm20Juj9+/vOfs2zZMhYuXAjAvffe2+Y+4N2B3wK3dQXHPXX1xxvIAKYbY34NvGat/UPTa8eNG9fu95k5c2aLJVIbioxLJoFy8ku6a1d5g1nltR7wVEFEVODaJSLSDXR1NpyZmdll954zZ47vqoBaa/36AN5s8vtDwEwgEsgGhjc4Z10TfSz7Rpt3e197V/Y3vr/3/tj4sbV3JFm77n33++K/ud/Ldge0WSIinXXuuefak08+udGx7Oxsa4yxGzdutLNnz7aDBw+2UVFRNjMz015xxRW2pKTEWmvthg0bLGBXrFhhrbV2ypQpdtasWdZaa2tqauy9995rhw4dalNSUuz06dPtfffdZ5OTk+vf55NPPrGTJ0+2iYmJNiEhwR5zzDF28eLF1lpr77jjDlsXYwC7YcMGe8cdd9jDDjus/vrPP//cTp061SYlJdkBAwbY6667zhYXF9efHzBggJ03b579v//7P5uVlWUzMjLsLbfcYmtra/frM2sQ+1qMo92h5OlKoNRaW22MKcYf4+7RSSSaYBjj9nYJVZdDbGpg2iQiweW/t8KOr7rm3r0OgdPu6dAlF110ERdffDF79uwhNdX9d+zZZ5/lhBNO4LvvvuOWW25h9uzZTJ06lVWrVnHDDTcwcOBAbr/99n3e94EHHuD2229n9uzZHH300SxYsIBf//rXxMW5LZGttUyfPp0jjjiCV199lerqau6//35++MMfsn79em644Qa2bNnC119/zbx588jKymp0/23btjFp0iR+8IMfcPfdd7Nr1y5uvPFGcnJyWLBgQf3r7rrrLi666CJefPFF5s2bx+zZszn22GM544wzOvQ5dUTAArcx5nZgPvA88Jwx5lbgfWvtqi5/8+hEovBQUFzc5W/VIfXLwepmlce7Z80sF5EgdfrppxMTE0N2djaXXXYZ5eXlvPLKKzz66KOkpqby6KOP8pOf/ARww6JPPvkk69evb/O+999/Pz//+c+54YYb6q/98ssvWbRoEQDl5eXMmjWLSy+9lL59+wKQm5tbX5M8LS2N1NRUYmNjGTp0aLP7P/LII/Tu3Zsnnniifpew5ORkjj32WNauXVt/zbBhw7jzzjsBGDNmDP/+979Zs2bNfnxibfN74LbWnup9vqvB4TP92oiYZADKiwr8+rZtapZxu2+OVJcGpj0iEnw6mBF3tbi4OM4++2xefPFFLrvsMt544w3CwsKYPn06cXFxZGVlsWDBAr788kuWLFnCu+++y2WXXbbPe+7Zs4cdO3ZwwgknNDo+derU+sAdFxfHrFmz+PTTT1mwYAHLly/njTfeaHe7v/vuOyZNmtRoa8+JEycSERHBqlWr6gP3xIkT689HR0fv16z29grJAix19cqrSgsC246mmmbcDbvKRUSC1EUXXcRbb71FcXExzz77LBdeeCFxcXG89NJLjB49mn//+99kZmZy1113tauLubV9ssPC9oa03NxcjjrqKGbNmkV+fj4XX3wxs2fPbneb3VBzY8YYjDF4PJ76Y3Vd8/7UHca4/c+7tWdNRSGemloiwrvJ95dms8rrusqVcYtI8Dr55JOJjY3lhRde4PXXX+fdd98F4L777uOnP/0p99zjegmsteTm5pKenr7P+yUlJZGVlcW7777L1KlT649/+OGH9T+//PLL5OXlsWXLlvpA//e//73dbR45ciTz58+npqam/volS5ZQXV3NqFGj2n2frhCagdu7tWcC5ewpq6ZHYnSAG+TVUslT0FpuEQlqUVFRnHvuudxyyy3069ePCRMmABAfH88bb7zBqaeeSllZGc888wzLly8nIiKCnJycfd7z5ptv5le/+hWZmZkcc8wxvPrqq/XrsevuvX37dp588kkOPvhg3n//fe6//37ABfhjjz2W8PBwtm/fzvr16+nXr1+j+1977bU8+OCDXHnllVx33XXk5uZy4403ctZZZzF8+HDffkAd1E1STT/zdpUndbfqac0KsHgzbnWVi0iQu+iii8jPz+eKK67AGAPAww8/TFJSEqeffjo333wzo0eP5q233mLDhg3861//2uf9Zs2axe9//3sefvhhjj/+eFauXMnjjz9ef/7CCy/k2muv5Re/+AVnnnkmK1asYMmSJUyePLm+1scFF1yAtZZDDjmEHTt2NLp/3759+eijj9i8eTMnnXQS1157Laeccgr/+c9/fPzJdJxpqR+/u6gr1uLzNu7eAA+P4efVP+HsGb9g8rAevr1/Z30wG97/A/w2H8IjoCgHHhgJZz4I464IdOtERMQP6r7YWGtNS+eDoqu8pUpo06ZNY9q0aZ27oXdWeQLl3S/jDotwQRvUVS4icoDLzs4mOzu7Q9cEReD2WZm4Ot7JaYmUda8dwjyVe8e3oUFXuQK3iMiBqKUkdO7cufu8JjTHuMMjsRGxpIR3w4w7osFEuYgol4GrAIuIiHiFZuAGTEwSGZFV7Op2gTum8bHIOGXcIiJSL2QDN9FJpIdXkNeddgjzVDbOuEGBW0REGgnhwJ1Iclg5+d0u426y12xUnLrKRUSkXugG7pgkEk13G+NWxi0iIvsWuoE7OokEysgvqaK2tpusZdcYt4iItCGkA3dsbSmeWktheXWgW+NUVzTPuNVVLiIiDYRu4I5JIqrGbd7RbbrLW824VfJURESc0A3c0UlEekoJo7b7LAlrdYxbu4OJiIgTFJXTfF7yFOqrp7myp91kSVhLGbe6ykVEDlgqedoRDbb27DZLwlrNuNVVLiJyIFLJ047wbu3ZrcqetjrGra5yERFxQjhwu67yvrHV7CjsLoG7hYw7Kg5qPeDpJt35IiISUKEbuL1be47OCGPJ+nzf7/ndUda2nnGD1nKLiAgQyoHb21V+ZK9wthWUs3pncWDbU1MN2JbHuEGBW0REgJAO3K6r/OB0A8C73+UGsjUu24YWZpV79+TWzHIRESGUA7d3VnmSKeeQrGTeWxXowO0dZ2+WcXs3HVHGLSIihHLgjowDEw6VRRw/oicrNu9hd2kAJ4C1lnGrq1xERBoI3cBtjOsuryjihJE9qbXwwfcBzLrrM+7Wusq1JExERIKkAEuXVE4D111eWczoPslkJETz7ne5TD+87/7ds7PqM+7WuspVhEVE5ECjymkdFZ0MlUWEhRmOH9GD/369g+qaWiLDA9AR0VrGHenNuNVVLiJywFHltI7ydpUDHD8ik+IKD8s27glMW9rKuNVVLiIihHrgjkmCShe4Jw3LICo8jPdW7QxMW1pdDlY3OU1d5SIiEuqBO3pv4E6IjuCowWmBWxbW6nKwuq5yZdwiIhLygXtvVznA8SN6sm5XKRvzAhAk6zLuuq7xOuGRbtmaCrCIiAihHrjrusq9dcqPH9ETIDBZd2sZtzFuSZi6ykVEhFAP3NFJ3p23XLY7ID2eoT0TeH91IAK3NzA3HeMGbe0pIiL1/Bq4jTFhxpiXWzl3gTHmVn+2p65eecPu8klDM/hs426qPLV+bUqrGTe47nN1lYuICH4M3MaYAcBXwIgWzkUCt/urLfW8W3vWTVADOGpQGhXVtXy1rdC/bWltVjmoq1xEROr5M+PeDBwGbGrh3EzgLT+2xfFu7dkwcB85KA2ATzfk+7ctdRl3eEsZt7rKRUTE8VvlNGutBTzGGNvwuDEmEZgK/A04qqVrx40b1+73mTlzZoslUlvUQld5RkI0Q3smsHTDbq49rt1vu/88FRAeBWEtfJdSV7mISFCbM2eOz6qAdoeSpzcBfwYiW3vBsmXLuuadY5pn3OC6y19ZmYOnppYIf5U/9VS23E0Orqu8bLd/2iEiIj7XkaTSGLPP891hVvlBwB+AB4GZxpjj/fbO9V3lxY0Ojx+URkmlh++2F7dwURfxVLQ8MQ3UVS4iIvUClnEbY24H5ltrL/H+fhwwwVr7nt8a0UJXOcCEwemAG+c+pG+yf9qyr4xbXeUiIuLl94zbWnuq9/kua+23DY4vtNbe49fGtDA5DSAzKYaB6XEsWe/H7ul9ZdyaVS4iIl7doas8cMIjXC3wyuZd4uMHpfHZxt3U1toWLuwC+8y41VUuIiJOaAdu8NYrL2h2+KhB6RSWV7N6p5/Gudsa4671gKfKP20REZFuS4E7dQDkrW12+KjB3vXc6/20nnufs8rrtvbUOLeISKhT4O5zBGz/Amo8jQ73TY0jKyWWpRv9NM69z4w7du9rREQkpClwZx3hNvjYtarZqaMGpbF0w26s9cM4t6di32PcoIxbRES6RQGWNrW0aH3atGlMmzZt/2/e5wj3nPM59Brd6NRRg9NYsGIb63aVMrRnwv6/1754KtvOuDWzXETkgJKdnU12dnaHrgmKwO2rMnEtShvsloVt+xyOuKzRqfGD9q7n7vrA3Z6MW4FbRORA0lISOnfu3H1eo67ysDDoMwZyVjQ7NTA9jp6J0Xzqj/Xc7cq41VUuIhLqFLjBdZfv/GbvDl1exhiOGpzO0g0+DtzfvuomxDW0z4xbXeUiIuIocIOboFZbDTu+bnbqiP4p7CiqYHuhD4Nm9iz4YHbjY/vMuDU5TUREHAVuaDxBrYkx/VIAWLm5wDfvVVUK5bsbz2K31ptxx7Z8jTJuERHxUuAGSO4L8T3cBLUmRvVJIio8jJVbCnzzXoXb3PPu9Xu75uuelXGLiEgbFLgBjIE+h7eYcUdHhDOqTxIrfBW4i7a6Z1sLeWvcz3WFVVob4647roxbRCTkKXDX6XME7FoNlSXNTo3pl8JXWwvx1NTu//vUZdywt7u8zYxbXeUiIuIocNfJOgKwzWd7A4f3T6G8uobvdzYP6h1WuBUwYMLcFwVoO+MOj4SwSHWVi4hIcBRg6dLKaXUaTlAbeEyjU/UT1LYUMKpP0v69T9FWSMiE6ATY9Z071lbGDd6tPZVxi4gcSFQ5bX8k9IDkfi1OUOufFkdafBQrNu/hkqP679/7FG6D5CxI7N3+jBtcd7kybhGRA4oqp+2vViaoGWM4rG+yb2aWF251s9h7jID8dW6P7fqMu63ArYxbRCTUKXA31Odw2LMRyppXShvTL5W1u0oorqju/P2thaJtkOQN3LYG8tc2yLjVVS4iIvumwN1QVt04d/O65WP6p2AtfLm1sPP3L9/juruTs6DHQe7YrlUdyLjVVS4iEuoUuBvqPcY9t1RBrW8KwP51lxd613An94WMYXtnlrcr41ZXuYiIKHA3FpsC6UNbnKCWHBfJ4B7xrNif0qdF3jXcSX1dIE4d6GaWt2tyWpwybhERUeBupt9RsPkTqG1ebGVMvxRWbinAWtu5e9dn3FnuucdIb8bdnuVgyrhFRESBu7lBU9xY9I4vm506vF8KeSWVbN3TyQBauNUVUonv6X7vcZCbnFblLezSZsatwC0iEuoUuJsaPMU9r1/Y7NSYfqnAfoxzF22DpD4Q5v3Ye4yAWo/bCxzakXGrq1xEJNQpcDeV2MsF1A0fNDs1onci0RH7sVNY4TY3Ma1OzxHuua7Mapuzyis6974iInLACIrKaX4pedrQ4ONg+ZNu7LlBFhwZHsborP0oxFK4FQZM3Pt7+jDAQO637vc213GXubXgxnTu/UVEpFtRyVNfGTQFPn0UtiyFQZMbnRrTL4X/LNlERXUNMZHh7b9nbQ0U50BS1t5jUXGQOsAVfQmP3ndAjowFrPsyEbmPzFxERIKGSp76ysBjwIS32F1+8qhMKj21zF++tWP3LMl149nJWY2P9xjpnvfVTQ4u4waNc4uIhDgF7pbEJLsqai1MUBs/KI3D+qXwj4/WU1PbgWVh9UvB+jU+XldBbV/d5KA9uUVEBFDgbt3g41whlorGJU6NMfz02MFszC/jf9/saP/9iryBO6lpxu2doNZW93d9xq3ALSISyhS4WzNoitsEZOPHzU6dfHAvBqbH8eiH69tfjKXQWzWtaVd53czyNrvK6zJudZWLiIQyBe7W9BsPEbEtjnOHhxmumjyYL7YUsHRD853EWlS4FaISICal8fGM4e5ZXeUiItIOfg3cxpgwY8zLTY7FG2PeNsZ8YYyZa0w3WesUEe2WbrUwzg1w3ti+pMdH8diH69t3v6Ktrpu86Z8XFQ8p/TU5TURE2sVvgdsYMwD4ChjR5NS5wPvAGMACx/irTW0afJzbdrO4+Vh2TGQ4lx89kPdW5bJ6R3Hb9yrc1rybvM7BP4ABbfzZyrhFRAT/ZtybgcOATU2OrwGesW6wuAOzvfxgUF350+bd5QA/mjCA2Mhw5rQn6y7c2rhqWkMn/c499kVj3CIigh8LsHgDs8cYY5sc/8Q4l+Ky7d83vXbcuHHtfp+ZM2e2WGmtU3odCrGpbpz7sAubnU6Nj+LCI/sx79NN/Oz4oQzMiG/5Pp5KKM1123l2ljJuEZGgNWfOHJ8VEwt45TTvmPa9QCZwjrW2uulrli1b5vd2AW4zkEFT4Pu3oGw3xKU1e8lPpgzmxc+3Muu5lcz/6UQiw1voxCjKcc+tdZW3h5aDiYgErY4klW1N9eoOs8qne59nWGvbMVjsZ5NudGu5X/2ZqxPeRO/kWO75waF8saWAB9/5vuV71Bdf8UXGra5yEZFQFrDAbYy53RgzChgPnAC8b4xZaIyZFKg2tajP4W78edVrsLTl+rFnHNqbC8f145GF61i8Lq/5C4q8a7j3p6s8Ql3lIiISgMBtrT3V+3yXtfZba+2t1tqx1trjvI9F/m5TmyZcC8NOgf/dtncLzibuOGsUg9Ljuem5L9hTWtX4ZF3GndSn820IC3NLxpRxi4iEtO7QVd79GQPn/B3iMuCFK6CyeY9+XFQED198OPmlldzy4peNK6oVboXYNLcb2P6IjFXGLSIS4hS42ys+Hc6dC3s2wOs3tzjePTormV+eOoK3v93Jvz7euPdE0bb9G9+uExmnwC0iEuIUuDti4CSYcit8+RwseaTFl1x5zCBOHpXJH9/4bu94d6GvAnesuspFREKcAndHHfsLGDkN3roNVv+32emwMMP9FxzGoIx4rpv3OVt2l7mu8qa7gnWGuspFREKeAndHhYXB9Meg92Ew/8ew46tmL0mMiWTOj8biqbXc/cSLUFkIGcP2/70j45Rxi4iEOAXuzoiKh4ufhZhkePoiKN7Z7CWDeyTw8EWHM2XPfKpMNPaQ8/f/fSNjwVOx//cREZGgFfDKae3RUrWZadOmMW3atAC0xiupN1z8DDx+Gjx7Mcx4AyIb7/A1ta/BE7GYZ6snw5fFXDqheeW1DomMg7L8/buHiIh0G9nZ2WRnZ3foGmNbmB3dXdTVNe/ObeS7bHjuUjf2ffxvGp9beA8svJubev6D9/OSWfiLqSTHRnb+veZf6daR/2z5/rVZRES6rbqSp9baFmufqqt8f42cBodeBIv+DDu/3Xu8ugI++wcMO5kfn3MyBeXV/O39tfv3XpqcJiIS8hS4feGUP7rx7uwboLbGHfv6RSjdBROu5eA+yZw/ti9PfLyRTfmlnX8fTU4TEQl5Cty+EJ8Op9wNWz+Dz/7pirMseQR6joLBxwFw88kHERFuuPfNVZ1/H2XcIiIhT4HbVw69AIacAO/+Dr54FnZ+DROuceVSgcykGH46ZQhvfLWDzzbu7tx7RMa5WeW1tT5suIiIBBMFbl8xBs58AGwtvHyNq2t+yAWNXnL15MH0Sorh/732LbW1nZhwV7e1p0dZt4hIqFLg9qXUgTD114CFI3/cbHlYbFQ4vzjlIL7YWsirX+R0/P6R3k1K1F0uIhKyFLh9bcK1cMFTMOn/Wjw9/fAshvSI5/llWzp+77qMWxPURERClgK3r4WFw6iz9gbZpqfDDCeOzGTZxj2UVno6du/6wK2MW0QkVKlyWgAcO7wHj324niXr8zlhZGb7L6zvKlfGLSJyIFDltCBR6alhzO/e5oJxffnd2aPbf+G69+Gpc+CK/8KAo7usfSIiEjiqnNYNRUeEM3FIOh+uyevYhcq4RURCngJ3gBw7LIMNeaVszu9AENYYt4hIyFPgDpBjh/cA4IM1u9p/kZaDiYiEPAXuABmUEU/f1Fg+/L4jgVvLwUREQp0Cd4AYY5gyvAeL1+ZR5WlnCVN1lYuIhDwF7gA6dngPSqtq+HzznvZdoMlpIiIhT4E7gI4ekk5EmGnUXW6t5U9vream51Y2vyAiGjDKuEVEQpgCdwAlxkRyRP9UPvAGbmst/+/17/jr+2tZsGIbuUUVjS8wxrsntwK3iEioUuW0AJtyUA/ue2s1u4or+cdH6/nnog1MPagH76/exSfr8zl7TFbjC7Qnt4jIAUOV04LQV1sLmfbXRRzeP4UVmwu4bOIAbj9zFEf8/m1OP6Q395x7aOML/nwIDJwE0/8emAaLiEiXaqtyWlBk3Aeyg/skkR4fxYrNBfzwqP787qyDMcYwflA6n6zPb35BZKwmp4mIhDAF7gALCzPceNJwdhVXcuMJw+q/aR09JJ13vtvJtoJyslIa7DSmrnIRkZCmwN0N/GjCgGbHJg5JB+CTdfmcN7bv3hORccq4RURCmGaVd1MHZSaSFh/F4nVNNiJRxi0iEtIUuLupsDDDhMFpLFmX33hyngK3iEhIU+DuxiYOySCnsIJNDXcQU1e5iEhI82vgNsaEGWNebnIsxhgz3xiz2BhzhT/b091NHOwd5244u1wZt4hISPNb4DbGDAC+AkY0OTUdWARMBmYYYzRhzmtIj3h6JkazeF3DwK3KaSIiocyfGfdm4DBgU5PjY4Fl1toaYAvQ349t6taMMUwcks4nDce5tY5bRCSk+S27tS7yeOqqoTWQDOR4f84BUpteO27cuHa/z8yZM1sskRqsjh6Szisrc1ibW8KwzESXcddWQ001hEcGunkiItIOc+bMYc6cOT65V3foli4E+gLrgX5AQdMXLFu2zM9N6j4mDs4A3Di3C9wN9uRW4BYRCQodSSrrCnG1pjvMKl8GjDXGhANZuC518eqXFktWSiyL13rHuRsGbhERCTkBC9zGmNuNMaOAl4FjcBPU/mWtrQ5Um7qjunHuJRvyqa21DQK3xrlFREKR37vKrbWnep/vanD4PH+3I5iMH5TG/OVbWZ9XwlBl3CIiIa07dJVLGw7vlwLAis0FbnIaKHCLiIQoBe4gMKRHAonREazcUqCuchGREKfAHQTCwgyH9UvxBm5l3CIioUyBO0iM6ZfCqh3FVBDtDijjFhEJSQrcQWJMvxRqai2r8z3uQEcz7jXvwFfzobzA520TERH/6Q4FWNrU0qL1adOmMW3atAC0JjDG9E8B4MvcSg4D8HQwcC+4Csr3QFgE9J8IB50Oo8+FxExfN1VERNopOzub7OzsDl1jGu313M3UlUftzm30p0n3vsf43uE8sP4sOOWPMPG69l1YUQT39IMjLoO4DPj+Tcj9FgZNgctf7dpGi4hIh9RVTrPWtlhCTV3lQWRMvxQ+21rhfunIGHfhFvc8eCqceAdc+wmMudQFbxERCSoK3EFkTL8UthR5sGERHRvjLvAG7pQGG6/1GA6luzTmLSISZBS4g8jh3nFuT1hMBwO3t/x7cr+9x9KHuef8tb5pnIiI+IUCdxA5uE8ykeGGShPdwa7yzRARAwk99x7LGO6e89b4tpEiItKlFLiDSExkOCN7J1FaG9XxrvLkvtBwq7jUAW6Ged73vm+oiIh0GQXuIDOmXwqFnghsVQcnpzXsJge3l3fqIMhXxi0iEkwUuIPMmH4plNkoSkuL239RwRZI6df8eMZwyNMYt4hIMFHgDjJj+qVQbqMpaxq4PZUtX1BdDqW5kNy/+bmMobB7HdTW+L6hIiLSJRS4g8ygjHiqw6OpLC/de3Dnt3B3P9i4qPkFhVvdc0sZd/owqKmCgk1d01gREfE5lTwNMsYYYmITqK3M23vw00ehphI2fQIDJzW+oG4pWEpLGbd3SVjeWkgb3DUNFhGRVnWm5GlQBO45c+YEugndSnxCEmFlFZRWeoivLYYvn3cncr9p/uK6qmlNJ6fB3iVh+WuAk7ukrSIi0rqWktC5c+fu8xp1lQehtJRkYqjk47V5sGKe23AkfajrMm+qYAuYcEjs3fxcXBrEpmlJmIhIEFHgDkI901OJM1W8/sU2+Owfbrevg6e7KmhNJ6kVboGkLAhvpXMlY5hmlouIBBEF7iAUHhVHLFVUrvof7NkAR14FPUeBrYFdqxu/uGBzyxPT6mQM01puEZEgosAdjCJiCKOWS202FdE9YORZkHmwO9d0x6+CFoqvNJQ+DEp2QkVh17VXRER8RoE7GEXGATAp/BveiTsNIqIgbQiER8HOBhPUaqqhOKflGeV1Gs4sFxGRbk+BOxhFxgJQQzj37ppASaXHjWFnHNQ44y7KAVu7767y+l3C1F0uIhIMFLiDkTfjLhx4Kls8Kbzz7U53PHNU45nl+1oKVidtkDYbEREJIgrcwSgxE4CU435G7+QYXvsyxx3vOcp1jZfvcb/vq/hKnfBISB2o7T1FRIJEUBRgUeW0JgZNgVlfEpY6gDMO+ZYnP9lIYVk1yXUT1HZ+CwOPcRPTwC0H25f0YW4pmYiI+FVnKqcZa20XNWf/GWMsQHduY6B9saWAs//2MbPPO5QLhoXBn0fB6X+C8VfDK9fBmrfh5210g//vt/DpY3DbdggL90/DRUSkRcYYAKy1pqXz6ioPcof2TaZ/WhzZX+RAUh+ISd47s7xgy767yetkDHO1zuu61kVEpNtS4A5yxhjOPLQ3i9flk19aBT0P3juzvLCNNdx16meWq7tcRKS7U+A+AJw9JouaWsujH6xzM8tzv3N7bBdu3fdSsDp1m41ogpqISLenwH0AOKhXIpcc1Z9/LtrA5siBUFkE2z53e223J+OOT4fYVC0JExEJAgrcB4hfnTaCzKQY7l/pXSjw/ZvuuT1j3KCZ5SIiQUKB+wCRGBPJ3T84hPd2Z7gD37/lntuTcQP0OgQ2L4GVT3dNA0VExCf8FriNMTHGmPnGmMXGmCsaHE81xrxvjPnCGHOlv9pzIDruoJ6cMnY4OTYddn7lDrZnjBvg+N/AgInw8jXwv9+4MXIREel2/JlxTwcWAZOBGcaYuuIvPwSeBA4HmldakQ757RmjWB82AAAbmwrRie27MC4NLl0AR14Ni/8Cz1ykHcNERLohfwbuscAya20NsAWoG3wtA9KAeECVVvZTclwkWcPHArArrGfHLg6PhDP+BGf+Gda9B/86FarLu6CVIiLSWf4seZoMeItqkwOken9eAKwFbgMeaOnCcePGtftNZs6c2WKJ1FAy6ODxsHouK4sT6bW1gEP7pnTsBuOuhMQ+8MyFrqLapBu7opkiIiFjzpw5zJkzxyf38lvJU2PMn4BXrbUfGmOeAX5jrV1njPk78AywFMgGLrbW5nmvUcnTztjxNTx6DM+Fn8GcuJm8fsNkYiI7Ucr06Qth0ycwa6XrShcRkS7XnUqeLgPGGmPCgSygrr5mGpALVAKlQJIf23RgyhgOvQ7hkGPOZN2uUu59c1Xn7nPinVBVDB/e59PmiYhI5/kz444B/oML2o/hxrjn47rr/wlEAq9ba29rcI0y7v1056vf8MTijTx91VEcPTSj4zd49Wew8hm4/jO3d7eIiHSptjJu7Q52gCuvquGMv3xERVUNb/7fsSTFRHbsBkXb4S9HwEGnwXn/6ppGiohIve7UVS4BEBsVzgMXjGFncSW3vPAltbUd/BKU1BsmXg9fvwjblndNI0VEpN0UuEPAmH4p/Pr0kbz5zY7OjXcfcwPEZcD/bgf1foiIBJQCd4i48piBXDZxAI99uJ55n27q2MXRiXDcrbBpkcu8RUQkYDTGHUI8NbXMfGo5H3y/i39ePo7jDupAgZbaGvjnSbBnI1y3FOI7MdFNRETapDFuqRcRHsZfLj6cgzITuW7e53ybU9T+i8PC4ey/QWUxvPHzrmukiIjsU1Bk3FdffXWzc9OmTWPatGl+b9OBYEdhBdMf+ZiSCg+/P2c05xye1f6LP7wP3vt/cMFTMOqsrmukiEgIyM7OJjs7u9GxuXPnAloOJk1s3VPGjc+uZNmmPZwzpg93nTO6fUvFaqph7vFQvAOu+1QV1UREfEzruKVVnppaHlm4jofeXUPv5BgevHAM4wa2IxDv+ArmHAejz4MfPNbl7RQRCSUa45ZWRYSHccMJw3j+JxMxBs5/7BPufPUbSis9+76w1yEw+Wb48llY845/GisiIoAybvEqqfRw35ur+PeSTfRJjuX/TR/N1H3NOvdUwt+PgVoPXLsEImP811gRkQOYMm5pl4ToCH539mjm/3QisVHhXPH4Z9zwzAp2FlW0fEFENJw+G/ZsgMV/ad+blO2G0vz9a2jeWhWBEZGQpoxbmqn01PD3het4ZOE6IsIM100dyo8nDWp5a9DnL4fv33Rru1MHtH7Tou3wjxOgNA+O+BEcfcO+X9+Sde/BU9NdzfTR53bsWhGRIKHJadJpm/PL+OMb3/HmNzvomxrLb84YySkH96r/RwVA4Tb465Ew+Di4+OmWb1RZAo+fBvnrYOQ0V33N1sKhF8Dkn0PG0PY16MlpsOFD6H80XPnf/f77RES6I3WVS6f1T4/j0R+N5emrjiI+KoKf/udzfvqf5ewqrtz7ouQsmHILrH4dvn+r+U1qa+DFH8POr+H8J9ws9FlfwFE/gW9fcVl40fa2G5OzwgXt9KGweTHkdnKPcRGRIKfALW06emgGr98wiV+dNoL3V+/i5D9/wGtf5ux9wYRrIWM4/PcWqCrbe9xaePNW15V+2mwYfrI7npwFp94NP/nQTXLLvqHtceuPH4boJLjkeQiLhOVP+PzvFBEJBkHRVa7Kad3Hmp3F/PyFL/hiayFnHNKb358zmrT4KFi/EP59tguqGcOgx0EQEQtfPO22BT3lDy3fcMmj8OYv4ay/urHvluze4PYEP/pncNJd8MIVsO5duHk1RMZ22d8qItLVVDlN/MJTU8tjH67nwXe+JyUuitnnHeqWjq15BzZ+BLtWQe53ULDZTSL7wVwIa6Vzp7YW/n0W5KyEaxdDSv/mr3n95y7DvvErtz/4hg/dePc5j8KYi7vyTxUR8TtNTpMu821OEf/33EpW7yzmRxMG8KvTRxAXFbH3BZ4qiIhq+0Z7Nro14Vlj4UcvNw7ypfnw54PdF4Bz/uaOWQt/HQdx6fDj//nyTxIRCThNTpMuM6pPEq9cfwxXTx7Efz7dxJkPL2L5pt17X9CeoA2QOhBO/j1s+AA++0fjc5/NBU+56yavYwyMnQFbPoWd3+7vnyEiElSUcYtPLF6Xxy9e+JKcwnIuGd+fW04dQXJsOzYtqWMt/OcHbq12Uhb0HQdZ42DRn6HfeLjkucavL82HB0a4AH76fT79W0REAkld5eI3pZUeHnj7ex7/eAMZCdHcedbBnDa6ybrvfakohJVPw9bPYOsyKNjkjl/xXxhwdPPXv3gVfP8/uHkVRMX57g8REQkgBW7xu6+2FvKrl77k621FjB+Uxk+OHczUg3oSFtbOAF6nZBeU7IReo1s+v/FjeOJ0OO7XcNwv97/hIiLdgAK3BISnppZ5n27msQ/WkVNYwdCeCcycPJizD+9DdEQLpVM7w1qYfyV8swBOuw+Omumb+4qIBJACtwRUdU0tr3+5ncc+XM9324tIjIngxJGZnDa6F8cO79Fy/fOOqKl29dJXvw7THnJj3iIiQUyBW7oFay0fr83n5ZXbePvbnRSWVxMXFc6koRkcOTCNsQNTGd0nmaiITix08FTCsz+Ete/AOX/X2m4RCWoK3NLtVNfUsmR9Pm98tYNFa3exZXc5ANERYYzpl8IxQzOYNCyDQ7OSiQhvZyCvLoenL3QFYI7+GYw6G/oc4ZaOiYgEkQMicKvk6YEtt6iCZZv2sHzTHpasz+ebnCIAEmMiOO6gnvy/s0eTHNeOpWVVpfDST2DV6273scTecNBpMO7HrU9wExEJIJU8lQNCfkkli9fls2hNHi+t2MaY/in8+8rx7R8PL9vtdipb/QasfRfCIuDHb0HPkV3bcBERHzggMu7u3EbpWq9+kcMNz6zgjEN785eLDu/4krKCLfCPEyE8Eq56BxJ7dU1DRUR8RCVPJaiddVgffn36CF7/cjt/fOO7jt8gpR/88HmXhT99AVSW+L6RIiJ+pMAt3d7Vkwcz4+iB/GPRBv65aEPHb9D7MDj/CdjxtVv3XePxeRtFRPxFgVu6PWMMvz1zFKccnMn/e/1b3v1uZ8dvMvxkOON+WPMWvHYj1Nb4vJ0iIv6gwC1BITzM8NBFhzOqdxI3Pf8FW3aXdfwm466AY2+BFU/Bc5e6WegiIkHGb4HbGBNjjJlvjFlsjLmiwfFwY8yjxpgVxpgb/NUeCT4xkeE88sMjqK21XP/051R5ajt+k+Nvg9P/BN+/CY+fBkXbfd9QEZEu5M+MezqwCJgMzDDGRHiPnwgUAWOBk4wxPipkLQeiAenx3Hf+oXyxtbBzk9UAxl8NFz8LeWvdjPPtX7i65yIiQcCfgXsssMxaWwNsAfp7j08BPrLW1gKX+rE9EqROHd2bK48ZxBOLN/L6l53MmIefAlf+F2wNPHYs/LEP/PVIeGo6/PdWKM3zbaNFRHwkou2X+EwykOP9OQdI9f6cAUw3xvwaeM1a+4emF44bN67dbzJz5kxmztQuUQe6W08bwYote/jli18yLDOB4ZmJHb9J78Ng5kL4egEUboXCLe552T/h21fgvH+2vA+4iEgHzZkzhzlz5vjkXn4rwGKM+RPwqrX2Q2PMM8BvrLXrjDEPAd8AjwMLgJuttd97r1EBFmnVtoJyzv7rx1RU1/DghWM4cVSmb268/Ut44XLYs8mNiR/zfxCmeZwi4h/dqQDLMmCsdww7C9jsPb4SKLDWVgPFfm6TBLGslFhevf4YBmXEc9W/l/HQO2uorfXBl7zeh8LMD9xGJe/eBf+ZDp/8DVY+A9//D7Yth6pOzGoXEfEBf2bcMcB/cEH7MdwY93xgE/Ac0Ad431p7c4NrlHFLmyqqa/j1gq9YsGIbJ4/K5E8XHEZSTDs2JWmLtbD8cfjfb6GqScW18CjoeyQMnAyDjnXd7tEJ+/+eIhLyVKtcQoK1lsc/3sgf3viOuMhwzh/Xj8uPHsCA9Hhf3BwqCqF8tyudWrwDtnzqthDd/oXbiQwgsQ9kDIWM4RCXAWHhbltREw4JmTBoMqT03/d7iUjIU+CWkPL1tkLmfrSe17/cTo21HH9QTy4/eiCThmZ0fIOS9igvgM2fwM5vIG8N5K9xy8wqC1t+feogGDwFBnkf8emNz5fsgm9ecvc87GJX8U1EQooCt4SknUUVzFuyiXmfbia/tIr+aXFcPL4/54/rS0ZCdNc3wFpXVtXWuiVnezbC+g9gwwewcRFUFgHGjacPngppg+C712Dde+710UnuNUNPhFP+CD0O6vo2i0i3oMAtIa3SU8Nb3+xk3pJNfLphN5HhhuNH9OSEEZlMOagHmUkx/m9UjQdyVsD692Hd+7B1KdR6ILkfHHIeHHIBpA+Fz+bCwnvd+Pr4q+GYWZDUx//tFRG/UuAW8VqbW8zTn27hv19vZ3thBQAjeycx9aAenDgqkzF9U7qmO70tlcVu3/AeI5ovOyvNg/f/AMufcL8PPREOvxSGnwYRUX5vqoh0PQVukSastazeWcz7q3axcHUuyzbtoabWkpEQzQkjenL8yJ4cOTCNtPhuFBh3r4cV82Dl01CcA7Fp0HMURMbufdRUuTH3igL3bMIgsTck9XbPqQMh6wjIHA3hPph1LyJdQoFbpA2FZdUs/D6Xd77LZeGqXIor3X7dgzPiOWJAKkf0T+XQvskMy0wgOiLApfRra1z3+pfPQuE28JRDdQVUl7lgHJMCsSnu2dZC8XYoynHPNVXuHhEx0HsMZI5y9/NUQHU51FS7JW0xye4RneRmxddUu9fVetzD1uwdv49Ng/Qhrms/fYj7spC/DvK+d5P1inPAUwU1le79q8vdl4ry3VC+x62HTxvkvoRkjoIeIyE60TsjPxzCItwEvqQsiPDD3ASRbuCACNxXX311s3PTpk1j2rRpfm+THNiqPLWs3FLA8k17WL5pD59v3sPuUhfwIsMNwzMTGd0nmYN6JTIsM4FhPRPJTIqu/z9at2UtFGyGbctg63L3nPe9W48eEeMy9vBIqCxxGXtF4d5lbg2FRXgDargL0k3Xt2MAu/fn+B7u/uGRLvBGRENsqveR5n7PXwe537j2tcq4JXUp/SBlAKQN9j4Gud8Tero2iQSZ7OxssrOzGx2bO3cuEOSBuzu3UQ5s1lo25ZfxdU4h3+QU8fU291wXzAESYyIY1tPVSx+WmciwngkMy0ygV1JM9w/orbHWG5SNC7phES0Hxqoy142fv9YthautgYxhbi172hCIimv/e1YWuy8T1RV7s/paD5TkujryBVugcLOboV+4tfEXCxMOib3c5L2U/tBzpBsS6DnK/R6s/ztISDogMu7u3EYJPdZa8kur+H5nMWtzS/h+ZzFrdpawJrekUUBPiI5gSI94hvRIYEjPBAamxzMgPY6BGfEkRPtzf58DkKfKZei717ugXpTjHRLIcccaZu+R8S6oJ/ZyWXtyXxh+KvSfqBr00i0pcIv4UV5JJd/vLGZdbglrc0tYt6uUtbkl7CiqaPS6HonRDEyPY0B6PAO9wXxIjwQGZcQTE6ku3/1WUQS537ku+Lw1boy/eCeU7HBzA2oqXaW70T9wj96HK4hLt6HALdINlFZ62Jhfysa8Mjbml7Ihr5TN+e7n3OLK+tcZA31TYxmckUDf1Fj6pMSSleKeB6TH0TMxCMbTu7uqUlj9X/j6RVjzNtRWu4l4fQ6HrLHuEZPsjtfWuMl5SX28s/HVUyJdT4FbpJurC+rrd5WybpfL1NfvKiWnsJyCsupGr42LCmdgejyDMuLpnRxDRmI0GQnRZCRE0Ss5hj4psb7ZYCVUlO+B79+CLUvdhL2d37hx9ZZEJbig3n8C9DrULbNLynIT8DQxTnxIgVskiJVVecgpqGBbQTmbvJn6xjz3vLOokvLqmmbXJMZE1GfpmUnR9EyMITMphsykaDKTYuiZFE1GfHRgis10d9XlLnhXl7sJeeGRbvb87vWweQlsWeLON50YF5fuXU8fB5ExEBHrfW74iG7wHO0mANZNwsNCUl/IPNgti4tNDdhHIIGnwC1yACut9JBXUsmu4kp2FFWQU1DOtj3lbCuoYHthOTuLKskvraTp/4UiwgwZCdGkJ0SRFh9FenwU6d7f0+OjSI93P6fGuUdiTIQCfZ2KIti9Doq2Q9E2N35emrd3PXzdw1MBnsq9a+1rKt2kOo/3Z4z7UlCXrdfsndhIYh/oNRp6HeIemYdAQg83i7+6zHX311a7Lwp1y/mikzo2i1+6LQVukRBXXVNLXkklOworyC2uJLeogh1FFewsqmR3aRX5pVXsLq0kr7iqxQweIMxAcmwkKXFRJMVGup9jI0mNiyQ13gX/uiCfEufOp8ZHER8VrjH5lljbeImate4LwM5vYefXLqvf+Q3krW69674lSVne5XgHQY/hbjlcz1GuKI8EDQVuEWm38qoa8ksryS+pIr+0kj2l1RSUV1NQVsWesioKyz0UlldTWFblPV5NYXl1q/eLCDP1gb5hwHdfAtxzcv3vUY2Oa3Y9LmPftQp2fOUqzkXFueVtUXGuK78us68ud9Xo8ta6YJ+3pnFxnKQs1w0/4Gi3nWzvwzQu340dEIFbldNEui9PTW19cN9d6p4LyqspLKv2Bvtqiir2BvzC8ur6R+0+/vMTExlGSqzL4Osz+vjI+sw+Nd4Fe/e7+zkxWl36gMvgi7a5DD73G/e8/QsX1MGVxB002VWdi4xzXwSiEty5Wo/rtq+pdmPtfQ53QV8lZ7uEKqeJSNCorbUUV3oo8mbuBeVV9Rl8ofeLQEFZNXvKvF8KyqrYU+q+FLT2n4QwgzeYewN8vPs5LT6atPi9zxkJ0fRIjCY9PpqoiBBav128EzZ8CBsWun3hS/PceDlt/Dc2LNJNmus7HoafAgMnu8l30iUOiIy7O7dRRPyrttZSVFHN7tKq+qBe91zgzfILyurOV9U/V9e0/N+RlLhIeiRE09M7A7+nd4ldeoJ3wl58VP0kvoBvMtMVrHVd7VWl7vfwCBeowyOheIfbOz5nBeR8DluXuclxkfEwZKp7pAxw69wTe7sMXXMa9psCt4iEPGstJZWe+sl4ecWV5JVU1c/I31VcSW6xd/JecSVVnhY2WMGVsU2Lj6pfN99wqV3PxBhv8I8mOTbywJyUV10OGz6C7990j6Jtjc9HxLogntTHjasn9XEbwaQNcbvHJWQqsLeDAreISAfUBfm8kirySyrJK6kkv9R107sZ+FXsKq5kZ1EFuUWV9dvANhQdEUav5Bh6J8fQOznW++yCfO/kWHolx5AeHxXc4/HWus1e6mrEFzV8bNu7nWzDWfFRCd5APtgF87TBbivZum1iba3b1jV1oHtExTd+zxrv1rIHeDe9AreISBcqrfSws6jCm7VX1i+5yymsYHtBOdsLK9hZVIGnyUy8qIgw+iTHkJUaS5/kWLJSXXnbvqlx9E2NJTMpJvjH32s8bhOY3esgf7173r3ebeVasKntpW7xPV33e2WRWz9f7e3Oj0lxm8UkZbkKdjHJLuBHJ7lHXJorilP3iE4MqkxfgVtEJMBqai35Ja5IzvYGAX1bQbkrmlNQTm5x40I5xkB6fHR9pt4nxZW0dfXr3c89E2MID9asvcbjtmmtLm+8x3tFIezZ4LZv3b3BBe3oJG9wTnKbwRR7N4sp2up+rijyFrVpRWScd4e43u45dZDberZuC9roBL/92e2hwC0iEgQqPTXsKKxg6x5X/S6nsJwdha5YznZvJbyiisYZakSYoVdyTKNM3T3cz72SY4gMD/Ksvb08lVBZAhUFrgZ9Wb57lO5ye7oXb3fV7opz3N7utkGxoYbZeVy6y+Z7H+aWwmUM9/uadwVuEZEDREmlh+3eDH1bfXlb97x1Tzk7iysaZe1hBjKTYupr19dl672T636POXAn0u2Lp8pl9btWu7XtRTneQL/bPe/ZtLdbPjIOeoyA+AyITXPd8PEZbjZ92mD38HFlOgVuEZEQUeWpZXuhC+Jb95TV163fVlDGtgKXwTddFhcbGe4mz6XE0CvJbUxTN2O+7njIbUpTW+Oqz21f6ZbC7VrtKtOV7XHPDavSgQvo0x91a9x9oK3AHRSby86cObPZMVVOExFpLCoijAHp8QxIj2/xfG2tJa+0kpwCtyFNw/H2nMJyPlmXR25xZbOJdJHhrks+MzGGtLra9N7NaXomxdDL++iZFH1glKoNC4eeI9zjsIuan68q847Br9/7SO7XqbdqqXJaW5Rxi4hIvbrgvrOwbjJdOTn1u81VUFBWXb88rmmAB7cZTY9Et569Z2I0aQ12mkvzlqytC/ypcVHBO7muC6mrXEREfM5aS2F5NbnFlfWT6HKL6nag21vQZk9pFaVVLe86Z7y7zqXF7a1Jnxa/t2JdRkI0afF7d5xLiQ2NLWYVuEVEJKAqqmu8mbrbca6u7nx+aZV3cxpXlja/ZO9zS9k87N1itm4LWffszebjo7xfAupq1O/9OSKIZtcrcIuISFCx1lJU7iGv1O0ZX1jmtpctbLDF7J4ytwPdbm/w31NW3ep+8gBJMRGkJ0TXb0CTXBf0YyNJS3DZfUZCdH2t+tiowI3VK3CLiEhIqKiuqc/eCxoE9d2l1ewurWR3mXveU7r3S0Br3fgxkWGkxUWRlhBFerwL6HV16TOTYuibGke/tFgSYyJ9/ncocIuIiLSi0lPDntJqdhW7uvS7vPXp95S6gL+nzG1Gk1vkztU06cJPjYukX1ocvzptJBOHpPukTQfEcjAREZGuEB0RTq/kcHolt71xSU2tJb/UTcbbsrucLXvK2Ly7jC27y/zate63jNsYEwP8B+gDzLXWPt7k/AXAYGvtPQ2OKeMWEZGQ0lbG7c9pdtOBRcBkYIYxpj7bN8ZEArf7sS0iIiJByZ+BeyywzFpbA2wB+jc4NxN4y18NmTNnjr/e6oCnz9I39Dn6jj5L39Fn6Tu+/Cz92VU+F7jbWrveGDMbeM5au9wYkwg8DvwNOKqlrvKxY8e2+31mzpzZYonUhsaNG8eyZcs68VdIU/osfUOfo+/os/QdfZa+M2DAAHr06NGu1y5fvhzoHpPTCoG+wHqgH1DgPX4T8Geg1Tn1+ocjIiLBrEePHu2OZW3t1ubPwL0MGGuM+RjIAjZ7jx8ETAVSgCRjzFJr7Xt+bJeIiEjQ8Gfgfhk3q/wC4DHgV8aY+dbaSwCMMccBExS0RUREWue3wG2trQDO28f5hcBCf7VHREQkGAVP1fVurKN7qR5I9/S1YPm79Vl2z/t1FX2WvhMMf3d3/ywVuH0gWP7hdPd/jBA8f7c+y+55v66iz9J3guHv7u6fpQK3iIhIEFHgFhERCSIK3CIiIkEkKLb1FBERCTXdYZMRERER2U/dOuMWERGRxpRxi4iIBBEFbhERkSCiwC0iIhJEFLhFRESCSEgFbmNMjDFmvjFmsTHmikC3J9gYY+KNMW8bY74wxsw1xgwxxnzo/TwPD3T7gpEx5gJjzK3GmPHGmE+MMR8YY/oHul3BxBgTbox51Bizwhhzgz7LzjPGpBpj3vf+f/xKfZadY4wJM8a87P252WdojLnLGLPIGPM309bm2y0IqcANTAcWAZOBGcYYf25reiA4F3gfGANYYA5wg/f47YFrVnAyxkSy93O7A5gG/BK4KWCNCk4nAkXAWOAk4Hfos+ysHwJPAocDM9G/yw4zxgwAvgJGeA81+gyNMVnAaGvtJCAcOKyj7xFqgXsssMxaWwNsAfQNsmPWAM9Yt4ZwBzAV+Mpaux3IDGjLgtNM4C3vz+nW2jxgBe6LkbTfFOAja20tcCmQoc+y08qANCAe9+Vc/y47bjMuGG/y/t70Mzwc+Nx7bhlwREffINQCdzKQ4/05B0gNYFuCjrX2E2CjMeZS4Bjgfe+XIHD/h5d2MsYk4r74vOY9VApgra0EPIFqV5DKAKYbYz4Brsdl3/osO2cB8GtgI+7fpv5ddpB1PLgvPtD8M9zvOBRqgbsQ6Ov9uR9QELimBB/vWMy9uO7Ic4C4BsMNSYFqV5C6Cfgze//PnQhgjIkFogLVqCBVDiwBjgUm4DIafZadczfwAyALOA4YBPos91PT/2/vdxwKtTHeZcBYY8zHuH+YmwPcnmAz3fs8w1prjTFfAIcYY3YC2wLYrmB0EC7jTsF96eltjOkBDGVvN5q0z0qg1FpbbYwpBlbos+y0NCAXqMRliqv1We63XU0+wxXAj73njgD+3tEbhlrgfhn4D3AB8Ji1tjqwzQk644ETgPe9EyF/CzyE+3d0bQDbFXSstZcAGGOOw2WJ7wKv4v6DeVnAGhacngeeM8bcips8+Wf0WXbWH4CngEjgddx/M/VZ7p87afAZWmu3GWO+NMYswn3p/KKjN1StchERkSASamPcIiIiQU2BW0REJIgocIuIiAQRBW4REZEgosAtIiISRBS4RUTkgGKMecIYY1t5FPipDQuNMQ92xb1DbR23iIiEhqW4TVOaqvV3Q3xNgVtERA5E5dbatYFuRFdQV7mIiIQUY8xAb7f5RGPMu8aYQmPM58aYMxu8Jsy7v/s3xpgS737vFza5z8HGmLe91281xvzRGBPe+CXmLmPMZmPMHmPM3Lr9HYwxcd79uHOMMWXGmGXGmCntab8Ct4iIhKr5wAu4Pd0XAdnGmHHecz/DbbryF9wGNs8B84wxZwF4648vxu15cRLwe2AWjfctn4Hb/es84E/AVcDF3nN3en+e5X3/zcCLTQJ/i9RVLiIiB6JjjTEVLRz/L/B/3p//Za19FMAYswy3s9ws4EfALcC9deeBz40xw4Bf4GqPXwvsBGZ6tzdeaoxJBvo3eK8NwA3W1RZf6t0SeYj33CjcxlfzvZs2XYPbeCgKt+NdqxS4RUTkQLSMljdGKQXqstr36g56g+e7wJneANwH+LDJtR8AZ3t/PgRY7A3adfeY3eT1S23jDUFKG/w8F3gcWGWMeRXIBp6z7dhARIFbREQORGXW2lUtnTDGDPT+2HSGeQ37jou1Dc5HtnB9sza0dsJa+4oxpi9wPnAyLovfYIw51Vq7c1831Ri3iIiEqqOb/H488J21thDYDkxucn4K8K3352+BCcaY+jhqjJlljHmlPW9sjPktMN5a+6S19ofAQOAg4Mx9XogybhEROTDFGmOGtnKuLvb90hizHfgKuBQ4DpjgPXcf8HtjzE7gM1xWfAUuQwZ4BDce/qgxZi4wArgdN5mtPY4BLjbG/BLYgfvSEAN82daFCtwiInIgGg+saeXckd7n64GfAGOAdcBZ1tpPveceAixuIlt/YC3wI2vtSwDW2i3GmGOBB3Bj5QXAY8Af2tm+a3BB/klcwF4NXGKt/aytC007xsFFREQOGN4x7g3A4dbalYFtTcdpjFtERCSIKHCLiIgEEXWVi4iIBBFl3CIiIkFEgVtERCSIKHCLiIgEEQVuERGRIKLALSIiEkQUuEVERILI/wcBWdSbph7y2QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(figsize=(8, 6))\n", + "ax.plot(\n", + " range(len(losses_train_VICRreg[1:])),\n", + " losses_train_VICRreg[1:],\n", + " label=\"training\",\n", + ")\n", + "ax.plot(\n", + " range(len(losses_valid_VICRreg[1:])),\n", + " losses_valid_VICRreg[1:],\n", + " label=\"validation\",\n", + ")\n", + "ax.set_xlabel(\"Epochs\", fontsize=15)\n", + "ax.set_ylabel(\"Loss\", fontsize=15)\n", + "ax.legend(title=\"VICReg\", loc=\"best\", title_fontsize=20, fontsize=15);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train MLPF" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "class MLPF(nn.Module):\n", + " def __init__(\n", + " self,\n", + " # input_dim=COMMON_X + 34 + 200,\n", + " input_dim=34,\n", + " embedding_dim=34,\n", + " num_classes=6,\n", + " num_convs=2,\n", + " k=8,\n", + " ):\n", + " super(MLPF, self).__init__()\n", + "\n", + " self.act = nn.ELU\n", + "\n", + " # GNN that uses the embeddings learnt by VICReg as the input features\n", + " self.conv = nn.ModuleList()\n", + " for i in range(num_convs):\n", + " self.conv.append(\n", + " GravNetConv(\n", + " input_dim,\n", + " input_dim,\n", + " space_dimensions=4,\n", + " propagate_dimensions=22,\n", + " k=k,\n", + " )\n", + " )\n", + "\n", + " # DNN that acts on the node level to predict the PID\n", + " self.nn = nn.Sequential(\n", + " nn.Linear(input_dim, 126),\n", + " self.act(),\n", + " nn.Linear(126, 126),\n", + " self.act(),\n", + " nn.Linear(126, num_classes),\n", + " )\n", + "\n", + " def forward(self, batch):\n", + "\n", + " # unfold the Batch object\n", + " input_ = batch.x.float()\n", + " batch = batch.batch\n", + "\n", + " # embedding = self.nn0(input_)\n", + " # perform a series of graph convolutions\n", + " for num, conv in enumerate(self.conv):\n", + " embedding = conv(input_, batch)\n", + "\n", + " # predict the PIDs\n", + " preds_id = self.nn(embedding)\n", + "\n", + " return preds_id" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "def compute_weights(target_ids, num_classes):\n", + " \"\"\"\n", + " computes necessary weights to accomodate class imbalance in the loss function\n", + " \"\"\"\n", + "\n", + " vs, cs = torch.unique(target_ids, return_counts=True)\n", + " weights = torch.zeros(num_classes)\n", + " for k, v in zip(vs, cs):\n", + " weights[k] = 1.0 / math.sqrt(float(v))\n", + " # weights[2] = weights[2] * 3 # emphasize nhadrons\n", + " return weights\n", + "\n", + "\n", + "def train_mlpf(data, batch_size, model, with_VICReg, epochs):\n", + "\n", + " data_train = data[:4000]\n", + " data_val = data[4000:5000]\n", + " data_test = data[5000:]\n", + "\n", + " train_loader = torch_geometric.loader.DataLoader(data_train, batch_size)\n", + " val_loader = torch_geometric.loader.DataLoader(data_val, batch_size)\n", + " test_loader = torch_geometric.loader.DataLoader(data_test, batch_size)\n", + "\n", + " lr = 1e-3\n", + " optimizer = torch.optim.SGD(\n", + " model.parameters(), lr=lr\n", + " ) # , momentum= 0.9, weight_decay=1.5e-4)\n", + "\n", + " patience = 20\n", + " best_val_loss = 99999.9\n", + " stale_epochs = 0\n", + "\n", + " losses_train, losses_valid = [], []\n", + "\n", + " encoder.eval()\n", + " decoder.eval()\n", + "\n", + " for epoch in tqdm(range(epochs)):\n", + "\n", + " model.train()\n", + " loss_train = 0\n", + " for batch in tqdm(train_loader):\n", + " if with_VICReg:\n", + " # make transformation\n", + " tracks, clusters = distinguish_PFelements(batch)\n", + "\n", + " ### ENCODE\n", + " embedding_tracks, embedding_clusters = encoder(\n", + " tracks, clusters\n", + " )\n", + " ### POOLING\n", + " pooled_tracks = global_mean_pool(\n", + " embedding_tracks, tracks.batch\n", + " )\n", + " pooled_clusters = global_mean_pool(\n", + " embedding_clusters, clusters.batch\n", + " )\n", + " ### DECODE\n", + " out_tracks, out_clusters = decoder(\n", + " pooled_tracks, pooled_clusters\n", + " )\n", + "\n", + " # use the learnt representation as your input as well as the global feature vector\n", + " # tracks.x = torch.cat([tracks.x, embedding_tracks, out_tracks[tracks.batch]], axis=1)\n", + " tracks.x = embedding_tracks\n", + " # clusters.x = torch.cat([clusters.x, embedding_clusters, out_clusters[clusters.batch]], axis=1)\n", + " clusters.x = embedding_clusters\n", + "\n", + " event = combine_PFelements(tracks, clusters)\n", + "\n", + " else:\n", + " event = batch\n", + "\n", + " # make mlpf forward pass\n", + " pred_ids_one_hot = model(event)\n", + " pred_ids = torch.argmax(pred_ids_one_hot, axis=1)\n", + " target_ids = event.ygen_id\n", + "\n", + " weights = compute_weights(\n", + " target_ids, num_classes=6\n", + " ) # to accomodate class imbalance\n", + " loss = torch.nn.functional.cross_entropy(\n", + " pred_ids_one_hot, target_ids, weight=weights\n", + " ) # for classifying PID\n", + "\n", + " # update parameters\n", + " for param in model.parameters():\n", + " param.grad = None\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " loss_train += loss.detach()\n", + "\n", + " model.eval()\n", + " loss_valid = 0\n", + " with torch.no_grad():\n", + " for batch in tqdm(val_loader):\n", + " time.time()\n", + " if with_VICReg:\n", + " # make transformation\n", + " tracks, clusters = distinguish_PFelements(batch)\n", + "\n", + " ### ENCODE\n", + " embedding_tracks, embedding_clusters = encoder(\n", + " tracks, clusters\n", + " )\n", + " ### POOLING\n", + " pooled_tracks = global_mean_pool(\n", + " embedding_tracks, tracks.batch\n", + " )\n", + " pooled_clusters = global_mean_pool(\n", + " embedding_clusters, clusters.batch\n", + " )\n", + " ### DECODE\n", + " out_tracks, out_clusters = decoder(\n", + " pooled_tracks, pooled_clusters\n", + " )\n", + "\n", + " # use the learnt representation as your input as well as the global feature vector\n", + " # tracks.x = torch.cat([tracks.x, embedding_tracks, out_tracks[tracks.batch]], axis=1)\n", + " tracks.x = embedding_tracks\n", + " # clusters.x = torch.cat([clusters.x, embedding_clusters, out_clusters[clusters.batch]], axis=1)\n", + " clusters.x = embedding_clusters\n", + "\n", + " event = combine_PFelements(tracks, clusters)\n", + "\n", + " else:\n", + " event = batch\n", + "\n", + " # make mlpf forward pass\n", + " pred_ids_one_hot = model(event)\n", + " pred_ids = torch.argmax(pred_ids_one_hot, axis=1)\n", + " target_ids = event.ygen_id\n", + "\n", + " weights = compute_weights(\n", + " target_ids, num_classes=6\n", + " ) # to accomodate class imbalance\n", + " loss = torch.nn.functional.cross_entropy(\n", + " pred_ids_one_hot, target_ids, weight=weights\n", + " ) # for classifying PID\n", + "\n", + " loss_valid += loss.detach()\n", + "\n", + " print(\n", + " f\"epoch {epoch} - train: {round(loss_train.item(),3)} - valid: {round(loss_valid.item(), 3)} - stale={stale_epochs}\"\n", + " )\n", + "\n", + " losses_train.append(loss_train / len(train_loader))\n", + " losses_valid.append(loss_valid / len(val_loader))\n", + "\n", + " # early-stopping\n", + " if losses_valid[epoch] < best_val_loss:\n", + " best_val_loss = losses_valid[epoch]\n", + " stale_epochs = 0\n", + " else:\n", + " stale_epochs += 1\n", + "\n", + " fig, ax = plt.subplots()\n", + " ax.plot(range(len(losses_train[1:])), losses_train[1:], label=\"training\")\n", + " ax.plot(range(len(losses_valid[1:])), losses_valid[1:], label=\"validation\")\n", + " ax.set_xlabel(\"Epochs\", fontsize=15)\n", + " ax.set_ylabel(\"Loss\", fontsize=15)\n", + " if with_VICReg:\n", + " ax.legend(title=\"ssl MLPF\", loc=\"best\", title_fontsize=20, fontsize=15)\n", + " else:\n", + " ax.legend(\n", + " title=\"native MLPF\", loc=\"best\", title_fontsize=20, fontsize=15\n", + " )\n", + " return losses_train, losses_valid" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Num of model paramaters: 28366\n" + ] + } + ], + "source": [ + "# train ssl version of MLPF\n", + "# model_ssl = MLPF(COMMON_X + 34 + 200)\n", + "model_ssl = MLPF(34)\n", + "print(\n", + " \"Num of model paramaters: \",\n", + " sum(p.numel() for p in model_ssl.parameters() if p.requires_grad),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "20055cb02ad64c2d9432787e2a20ada5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=50.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1a454a9249964d6db2ebfec640d90d06", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8176702d26c441848893c0cb728fce22", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 0 - train: 135.347 - valid: 31.53 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e92302255ec64adbac37e191f042509d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6dac2834b5d943ec9db0ac0a19b4e8c9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 1 - train: 124.318 - valid: 30.721 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "010d941f662746b7a67ec83d89bfaa6c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f469cdf4146c46f6a0b6e167be899929", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 2 - train: 121.797 - valid: 30.199 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "37d43e0681924cb0818bedaf623204db", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6f96819c59914635aab4bfe3f787c237", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 3 - train: 119.916 - valid: 29.769 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b9c7729c156b43cda1ea5f4cb30ee594", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "060e53f2de284b93824ff3698c516b92", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 4 - train: 118.319 - valid: 29.394 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7e0a477cd55d4ad58191b0be3f983cbf", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "17c03e8428ee4cc6b569a81840f29b28", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 5 - train: 116.897 - valid: 29.051 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7f89fa99da3b4dd396a03a8713b237f9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e6f49cbc279d4732afa76fec9389bf71", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 6 - train: 115.576 - valid: 28.729 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2fb8d1872bac48759a19a88bc6db4a81", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2bf9747fe15c44cc9b0190ec40710be6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 7 - train: 114.32 - valid: 28.42 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b29471d57aeb40dd80fd4914165dcc20", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7ffba070f04a42e996b548b7548d7582", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 8 - train: 113.114 - valid: 28.122 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c9d18340cb9b429fa74a186ac372f796", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7241e81f606a49bbbb548678b981d6d4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 9 - train: 111.95 - valid: 27.834 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ac5b1cbe751b4945bc1231a8e7b9cbd4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c254bf770d80472aa104cc77e845ec13", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 10 - train: 110.814 - valid: 27.551 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2042727cc0a4432f9138de714681e5ec", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ad09057511034adeb2a9b21f0b61d610", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 11 - train: 109.695 - valid: 27.272 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e3790b3385a540b8a6023cf520162afa", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8f611d9decc04797afbde92af3ce660b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 12 - train: 108.584 - valid: 26.995 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2215a149e7834c6fbcfacacf94f90be6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9a61c084953f4e42bf53455c4c17e86c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 13 - train: 107.491 - valid: 26.724 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b01e64f526c9447ab50cd806f9bba991", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d7112253072d4955a11d2e752dfc5cd6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 14 - train: 106.422 - valid: 26.459 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cf96677568c54650b8d3de2d9f58d8ac", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d6f19233dd454bc187b18aebf8240a34", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 15 - train: 105.372 - valid: 26.199 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d53ccb048f7443b5ae51586105ef0aa5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "776a6ed88ef64ab8b0b3aa8cc9d88d43", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 16 - train: 104.341 - valid: 25.943 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "259dc19594a3443fb3758676ceddfdee", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f28cb7cd327d476d8ed3fe44fb6a1d62", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 17 - train: 103.326 - valid: 25.691 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f2c5731c250844d7b8a1b37647d31a61", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7808e4fe4e474757a4ff0706a05ad727", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 18 - train: 102.322 - valid: 25.442 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8c2fc6b4e9c54b57a952f9b1cffb3cb9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6d7e98f4a5aa4401bc92bc53afebdeca", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 19 - train: 101.327 - valid: 25.194 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e8dae3d67b6443d19e65203f40cd3591", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "870e52a4125242f5a5835c6bb00eca32", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 20 - train: 100.339 - valid: 24.949 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4cab2059e43f4d1fbd6ee0992e8cf0c6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7bc234882374409eb35f7f4177f36819", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 21 - train: 99.358 - valid: 24.706 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ddad47874d654c4a88a1d9f29b673a21", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7ad253599fb2486681e9907a7a0d1ab9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 22 - train: 98.385 - valid: 24.464 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7bcc641fdb754db5b63496ec91695124", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5d4f154deec141acaeee6d01700223c8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 23 - train: 97.419 - valid: 24.225 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1c1fa30c4666477fb46385a17c41d081", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cc021a4fa9d8453eaf7595202613fdc1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 24 - train: 96.464 - valid: 23.989 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "407e2605345c46079c3d317483108306", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "efcc241618244641b3a4c04ec6e1ec2a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 25 - train: 95.522 - valid: 23.757 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8b3db8261d9c46479852ff77ab7f98a7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b509e2e8c3c5452fbfa4515cbe80f36d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 26 - train: 94.595 - valid: 23.528 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4d98cfb0104c4324a6b827d87052bed7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "46412175b83249f8ba7e146e27ba5ed1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 27 - train: 93.687 - valid: 23.305 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "634c09cce0a84566a84cd8f27b8d8f16", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "884f014e45f74e3186f8c5ada777610e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 28 - train: 92.8 - valid: 23.088 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2038fa092f7e47309c915d21cd28a9f5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b423b90ae0b944e8bba543cf2bb6e625", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 29 - train: 91.936 - valid: 22.876 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "aa89f8b6b93e4d75ba887c527b9442be", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4fb067e948844da292dd98ca74ae5f53", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 30 - train: 91.096 - valid: 22.671 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4ce993cd8e434705aefa389e2b31a6a9", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5d11d13802284642886b15156a9b0fb1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 31 - train: 90.281 - valid: 22.471 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "175a2d6852634115b30447a18ef91f99", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0834cc528b054364b084bad04279968b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 32 - train: 89.487 - valid: 22.278 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8dcc6eab06814d0992048e6c9437578b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "41a78fd5c70a4d409f5ce2d34f8a814e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 33 - train: 88.72 - valid: 22.091 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ac592a20e2c44951a3463185d39808d5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8ecb2e3886064a22a8c13b82e58ff15a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 34 - train: 87.986 - valid: 21.915 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "030ed7fbfe58439f938120ec86709eec", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9c8827c4c41d422595b325a05b42a6ac", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 35 - train: 87.292 - valid: 21.747 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "533c4c2d377f4844911ceb6e5c7ad3f4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "521ccb96e5884d0fb1dc64d22a19d300", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 36 - train: 86.634 - valid: 21.589 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "388875e7abba4623b5282879b2dc014c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "08b50d63e80940a491dc95fe1f41e739", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 37 - train: 86.01 - valid: 21.439 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9d555c8f3794465cb7645d5dbb92156e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3c2d9276307d4c098bf569f20ddfc22c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 38 - train: 85.42 - valid: 21.297 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d480cd128c4444a3af5833f939c2f428", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "238c9a55ab0d47739519e6a043ce5079", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 39 - train: 84.861 - valid: 21.163 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d1b0a3c670db40dba0fe22654e2c63ec", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "27d7b920538b4078a6acab88e73f3c71", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 40 - train: 84.335 - valid: 21.036 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "47eb3b06b45e42f186bf59d5adb1b647", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "337460ff470049d8a43f8f1eca630deb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 41 - train: 83.84 - valid: 20.917 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6820bc46445d400096cb1e38d7919b9c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "40894813a51e4864a43b545a556424f1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 42 - train: 83.376 - valid: 20.805 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a67585119c71475592a5470d2b51bfc7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3781c1380d844093aa3dcc62c0aade80", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 43 - train: 82.942 - valid: 20.701 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "eb30794df01f461fb4e8eb27886d0388", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "dd9cf26477a44b1e8ec00be57d5f6f7a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 44 - train: 82.537 - valid: 20.604 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2bd48c3045e747f8a914eb2f3f56dc42", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ea588a09c38b45a5bfe004f52d849feb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 45 - train: 82.165 - valid: 20.515 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2a93cc3a1f3b44b59a4d725003b70adc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c8340d64e9a748b4bc5b080f1a5225be", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 46 - train: 81.819 - valid: 20.43 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1753be20a5004ae7a57b37cdbbe6550f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "933f586d6bca42ea9a4a9d982f77b29a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 47 - train: 81.497 - valid: 20.352 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "51c0e78d5b5c459d9625b4b3038c49d1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "efb21bd337ef4b9b9de10844325bcac1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 48 - train: 81.196 - valid: 20.28 - stale=0\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "acf27b595f494c2db253a536fcf7604d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b56d1b85db354cf6b2a943c0cba4dda7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=20.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "epoch 49 - train: 80.915 - valid: 20.213 - stale=0\n", + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEICAYAAABWJCMKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7C0lEQVR4nO3dd1xV9f/A8dcHBAVUwAUqigNRcaHg3ntr7pWmlZhWamaOclemliMtLfx+NSu/Vu49yp04cW9U3HuLiIz7+f1xkZ/by7yXe9/Px+M+PJz5PnA973POZymtNUIIIWyPnbkDEEIIYR6SAIQQwkZJAhBCCBslCUAIIWxUBnMH8DpKKSmhFkKIJNJaq9ctlycAIYSwUekiAfTs2ZOePXtSrFixhOlly5ahtX7jJyAgwKT1nnx69uyZqPXTapukHCOx555WcaXFNmlx7vK3t8zfl6X+7VPrGMuWLUu4Lvbs2ZMcOXKYfG216FdATwQHBwMQFBSUMC2EEAKaN29O8+bNE37eu3cvN2/eNGnbdPEEIIQQIuUlKwEopQoqpRxSKhghhBBpx+QEoJTKp5RaoJRqHP/z78Bp4LJSqmxqBSiEECJ1JOYJYDpQDDivlPIB2gENgVXAuFSI7QVPv+eypGOkxTZpce5JOY6l/r6SwlLPRf721vG3t8RzV1qbVtVeKXUXaKu1/kcp9QVQWWvdTClVA1istc6e4sHFtwMwNcaXCQwMZM+ePSkWU3oi526b5w62ff62fO5gPP/Q0FDgze0AElMLKA4wxE83AJbGTyvAMZExJkpQUNAL854v+RZCCFu0fPlyli9fnvDzuXPnTN42MQlgG/C1UmopUAFop5TyAvoCRxOxn0RLStXPB1ExzNh0mjinbKkQkRBCWIa0qgbaH7AHvgAmaK2vA/2AusCQROwnTTyKjuOXkLM89Kln7lCEEMIimZwAtNZntNYVtNZZtNYj42ePAzy01htTJ7yky5U1E0E1ChHtUYLQc3fMHY4QQlicRLUDUEqVUkrli5+uDkwCeiulXlvQYC49qxfCxT6Or1ceTVZBcnr1srITW2HL5w62ff62fO6QuPNPTC2gD4AfgY7AEuAcxnYARYEftNZjEhuoCcdMdi2gP3adZ8iiQ8zoUo7GpXKnWGxCCGGpntyTv6kWUGKeAAYCY4AFwFvAI6AG8CnQLSlBpoV2gfko6pGFcWuOEx1rePMGQghhIxJTCygvsEJrrZVS9YHV8dOH45elmiRXA424gb1LDoY0KUaP2buZu/McPaoWTKUohRAi7T1fDTRRTO2WFDgBfAI4ATeAevHzuwOXEtvNqYnH1MYQk+BGmNbf5NN6z2xtMBh0l5k7dJnRa/XdyOik7U8Ikaq8vb11zZo1TVoP0D4+PtpgMLxyPYPBoAsWLKgBXbhw4YT54eHhGjDpWFprXbNmTf3kWvT0x8nJSfv7++vhw4frhw8fPrPNyJEjX7rN85+NGzeaFENiPXXtfO01NjFPAD8A3wPDgRva2CL4XWAq8EsSck/qylYIcpeBtcNQhesytEkxmk37l+kbTzG0SXFzRyeESKZTp05x4MAB/P39X7p83759hIeHp9jxunbtStasWQHjjfP169fZsGEDX375JWvWrCEkJIQMGZ69pNapU4fixV99vcmbN1VfnryRyQlAaz1NKXUaKA4sip/9CPgSmJLyoSWTnR20+AFmVIFlH1Oi62Jal/VidshZ3q7kTb5szuaOUAiRRJkzZyYiIoL58+e/MgEsWLAAABcXlxQ55pgxYyhQoMAz827dukW1atXYvXs3c+bM4b333ntmedeuXenevXuKHD81JKoaqNZ6ldZ6otY6PP7neVrr8Vrrx6kTXjK5e0P9MXBmI4T+wsCGvijgu3UnzB2ZECIZPDw8qFChAvPnz39pLUGtNQsXLqRKlSqJGiErsbJnz86AAQMA2L59e6odJ7Ukth1AM6XUZqXUdaXUbaXUv0qpFqkVXIoIfBcK1oR1w8itb/B+9YIs3X+ZgxfvmjsyIdK12NhYfvrpJ8qVK0eWLFnIlSsXDRs2fOFCGBkZybhx4yhZsiQuLi7kyZOH1q1bc+TIkWQdv23btoSFhXHw4MEXlh0+fJiTJ0/Stm3bZB3DFB4eHgBcvXo11Y+V0hIzHkB7YDEQhrHq58fAMWBB/DLLpBS0/ME4vexjPqhRiOwujoxYeoSYOKkWKkRSDR48mN69e3P+/HkaNmxItWrV2LRpE3Xr1uX48eMJ63Xt2pWhQ4dy584dWrRogb+/P0uXLqVOnTrcuHEjycd/cnGfP3/+C8sWLlwIQOvWrZO8f1Nt27YNgKJFi6b6sVJaYgqBhwDDtNbjn5o3Vyl1ChgK/JWikaUkt/zQ4EtY8QlZjvzOqBaN+HjePib9fZLBjYqZOzoh0p3o6GimT59OqVKl2L17NxkzZgSMF962bdvy119/MWLECC5evMiiRYto1KgRK1euxM7OeM85ceJEBg4cyOrVq+nWLWnNiAoWLEhgYCDz58/nyy+/5OkOCRYsWECFChXw9vZO/sm+hNaamzdvsmDBAqZMmYKdnR2dO3d+Yb3ffvvtlV1T9+zZkzJlyqRKfKZKTAIoAqx9yfy1wLCUCeflUqQ76IAecHQprBtO8951CamQnxmbTlOpUHZq+uZMwWiFsH4RERFERUXh4OCAo+P/9wbftGlTtm/fTq5cuQC4fv06AE5OTgkXf4D333+fqlWrki9fvmTF0a5dOwYPHsyhQ4coXbo0AMePH+fIkSNMmDAhWft+XsGCL29D5ODgwOTJkwkICHhh2YYNG9iwYcNLt6tXr16KJIDktANITAI4B5QH9j83vzxwPklHN1FSuoN+gVLQYhpMN9YKGtlpEXvP3WHAn/tZ1a86HlkzJf8YQtiIbNmyUb16dbZu3UpgYCDvvvsu9erVw9fXl0qVKiWs5+fnh4+PD4sXL6Z27dp069aN2rVr4+3t/cx6SdW2bVsGDx7MX3/9lZAAnrz+adOmTbL3/7Snq4EC2NnZ4ePjQ4MGDShW7OVvEmbPnp3qtYBedjM8c+ZM0zZ+U0MB/f+NsvoCDzB2B10j/vMFcB/oa+p+EvMhOQ3BXmX3LK1HZtV610x98up9XWzYat3x5+06Nu7VDUqEEC+6c+eO7t+/v86RI0dCwyMvLy89aNAgfe/evYT1Ll26pHv06KGzZs2asF6RIkX02LFjdVRUVMJ6iWkI9nTDroCAAO3r65vQKMzf31+XK1fulesntSFYeHi4Setr/f8NwWbPnm3yNikJExuCJaYW0DSMr3p6A5viP32AUUAK3KKnkYDuULgOrPmcIrFhjGlZgu1nbvHDhlPmjkyIdMXNzY3Jkydz9epVduzYwbhx43Bzc2PChAm0atUqYb08efIwa9Ysbty4waZNmxg5ciQxMTF8/vnn9OnTJ9lxtGvXjpMnT3Lo0CFOnz7N/v3706T2jzVIzHgAWmv9vdbaC3AD3LTWeTGOFPbQlH0opeyUUkuem+erlNqvlNoUP9pY6lIKWs+EzB7wRxfaFnWgddm8fL/+JNtP30r1wwthDU6fPs2oUaMICQnB3t6eihUrMnjwYA4ePEhAQAAbNmzg/v37hIaGMmrUKI4ePYqjoyM1a9Zk1KhRnDhxAk9PT5YsWZLsWNq1awcYawOl1usfa5WodgBPaK3va63vJ2YbpZQ3cAh4/mWZD/Cj1rqW1rplUuJJNJcc0HEuRN1F/dWNL5sVoUB2F/r9sY9bEZbZpk0ISxITE8Po0aMZNmwYcXFxCfMjIyOJiIgga9asuLi4cOPGDUaPHv1Cgezdu3eJjo4mT548yY6lUKFClCtXjvnz57NgwQJKly6Nr69vsvdrCxJTCJxc54EywMrn5hcEmiulugM/aa1/S5NocpeGlj/Cgh64rB/CD52+4q0ZIQz46wCzupfH3s4ix7gRwiL4+vpSo0YNNm7ciL+/PxUqVODq1ats27aNe/fuMXr0aOzt7alRowa+vr7MmTOH48ePU7JkScLDw9mxYweRkZFMmjQpReJp164dQ4cOBYxdNpjiyJEjr3xV5OzszK+//poisVmyNEsA8QUTsU8GeXnKaWAEcAr4Wym1XGt99+kVAgMDTT5OUFCQ6SPilGwN1w7D1on4eZZmZPMGfLH4MGNXHWN4Mz+TjymErbGzs2PhwoWMHTuWZcuW8b///Y/MmTNTsmRJevfunVAn3tnZmXXr1jFmzBg2bNjA/v37cXd3p0qVKnzyySc0adIkReJ5OgGY+vrn5s2bCa+Mnufq6poicaWk4ODglKkR+RSTRwR75Q6UqgiEaK3tTVx/jda60SuWTQP+o7U+EP9zskcEeyODAeZ1hNProdtSRh1055eQs4xs7idjBwgh0iVTRwR77ROAUmqWCcdKVisqpdSXwBpgJ1ASOJuc/SWanR20mQkz68Jf3Rj+/kYu3/VgzIqj5HFzomEJzzQNRwgh0sprnwCUUhtN3ZHWurZJB4x/AlBKjcA4vORdYBngCEzXWv/01Lqp/wTwxM0wmFkH3L151GUFnX49wvGr95nXsxJl87un/vGFECKFmPoEkOxXQKkpTRMAQNjf8L8OUKAqN1v+TuvgvTx8HMuiPlXwzp4yfYoLIURqS41B4a1fkfrw1nQI30KONX2Y/U5Z4rSmx+zd3HkYbe7ohBAiRUkCeF6ZjtBoPBxfQeHtnzOzazku3n1Ez1/3EBUT9+bthRAinUjLdgBJliK9gSZGpQ/g0R3YPI7ymdyY1O5DPpq3nw/n7mXG2wE4ZpC8KYSwDMnpDVTKAF5Fa1g9GHb9DHWG8Ztje4YvOUzDEh780LkcDvaSBIQQlknKAJJLKWg0Dkp3gA1f0dX+b0Y192PtkWv0/2M/sTKamBAinZME8Dp2dsbuInwbw8qBdHfZwbCmxVl56AoD/jpAnMFyn56EEOJNJAG8ib0DtJsNBavDkt68nzmEwY2KsezAZT6bL0lACJF+SQIwhYMTdPoTCtWCpR/SO8tWPq3vy6J9lxiy8CAGSQJCiHRIEoCpHJ2h0x/gUx+W9+PjrJvpW7cI80Mv8vniQ/IkIIQFGzVq1DODxr9JgQIFUn0oR0uQLmoB9ezZ84VlqVoN9HViH8Nf78DJ1ehG45l4rzY/bDxFS/88fNeujNQOEiIVBAYG0qxZM0aNGpWk7W/fvs3t27fx8fExaf2zZ8/i5OSEh4dHko6Xll5WDfTJmMDSFURqiI2GBT3g+Apo8DXToxsxYc0J6vt5MK1TWTI5mNQxqhDCRMlNALZGqoGmpgyO0O4X8GsJ676gj/0yxrQswd9Hr/H+nD1ERseaO0IhrEatWrUIDQ1l9OjRFChQgFq1ajF+/Hg+/vhjXF1duXnzJg8fPqRnz554enqSMWNGChUqxPjx4xNuHn/55Rfc3NwA4929UoojR47w1ltv4e7uTuHChZk3b94zx+zfvz9gfH3UqFEjVq5cSWBgIJkzZ6Zu3bqEh4cnrH/gwAFq1qyJq6srlStXZu/evSil2L9/f1r9mpIkXbQEtkj2DtBmFtj1gvWj6Vb1Hs5t32fQwoN0/e8uZnUvj6uTg7mjFCLRRi8/wtHLiRrxNVH88mRlZPMSJq8/d+5cGjZsSJ06dRgwYADdu3dn8uTJdOjQgdWrV+Pu7s4XX3zBokWLmD59OoUKFWL9+vUMGTKEypUrU6NGjZfu99133+XDDz9kwIABjBkzhm7dutGoUSPc3V/s/ffo0aNMnTqVKVOmcO7cOT766CMGDRrE/PnzuX79OtWrV6dt27Z8++23hIWFpZsxiSUBJId9BmgdDJmywrYptC13B+dOg+j350E6z9zBr+9WIHvmjOaOUoh0LW/evGTKlIls2bJRoEABALy9vZkyZUrCq46AgABq1qxJ48aNAfD392fMmDGcOXPmlQmgffv2dOvWDYCxY8dSsWJFzp0799IEcOXKFXbt2oWnpyfVqlVj27ZthISEADBjxgw8PT2ZOXMm9vb2VKhQgVu3btGvX7+U/lWkOEkAyWVnD00ngVM22PodTaLu4tzla3r97zDtf97Ob+9VJI+bk7mjFMJkibk7Nxd/f/9navW0a9eOw4cP89tvv3HgwAE2bNjAo0ePXruPypUrJ0znypXrtesWLFgQT8//Hxzq6fUPHTpElSpVsLf//7K/atWqmXwu5iRlAClBKag7HBp8DUeXUmvPx/zetQTX7z+mzYwQTl57YO4IhbAqTk7P3lQNGTKEmjVrsnnzZsqWLcvcuXNfeif/NGdnZ5OP97p1Y2NfLPN7OhlYMkkAKanKR9ByOoRvpvyWHsx/pxhxBk3bGSHsPnvb3NEJYZUiIiKYNGkSc+fO5T//+Q9dunQhT548PHiQNjdefn5+bN++HYPh//sHe/J6yNKli1dAad4ddHKU7QKZXGFBD4qt7sCSt3/j7fmXePs/O5naqayMMSxEEtjb2xMWFsbFixdfWObo6EiGDBmYM2cO2bJl48SJE/z0008YDAb27dtH69atUzW2Pn36MGXKFIKCgujVqxdnzpxh2rRpCXGntuR0B50uEkBwcLC5Q0ic4s3g7YUwrzN5FrRgcdt5dFvpQO/fQ/nqrVJ0rpjf3BEKka4EBQUxdOhQDh06RLZs2Z5Z5ujoyLx58xg0aBC1a9emRIkSDB8+nLCwMMaOHUuLFi1SNTYvLy82btxIv379qFu3LoGBgcydO5dy5cq9sWwhJbzsZvhJQ7A3kYZgqenqIZjbDqIjiWozh97bXNh44gb96xWhX90iiWqaLoSwTCdOnODs2bM0bNgwYd6///5L/fr1iYiIMEt5gDQEswSepeC9vyFrbjL92Z6Z5c7RNsCLKf+EMWThIWJkTAEh0r2bN2/StGlTfvnlF65evcq+ffvo168f3bt3t/jCYHkCSAuP7sAfXeDcNnT9L5kU0YBpG09TvUgOpncpR5ZM0mBMiPTs999/55tvvuH06dPkzJmTpk2bMmHCBLJmzWqWeEx9ApAEkFZiomDJB3BkMVT8gPnZezN0yVF8cmVmVvfy0lZACJFiJAFYIoMB1g2DHT9CsWaElPmGXn8cw8nRnlndy1Myr6u5IxRCWAGrSgAW1R10StjxE6wdCp6lOVXvP7wz/wJ3IqP5oXNZ6hSz/O5nhRCWQ7qDTo9OrIEF74KTG7db/sY7qx5x5PI9RrUoQbfKBcwdnRAiHbOqJwBLjjFZrhyE/3WAx/eJeus/fLQ7J/8cu0a3yt6MaOZHBhlcRgiRBJIA0ov7V2BeB7h6CEPDcYy/XYOft5yhepEc/NCpHK7OUkNICJE4kgDSk+iHsPB9OLEKKgQxP0cfPl96jHzZnPnvO+UpmMPF3BEKIdIRi2wIppSyU0otecWy9kqpIWkZj8VwdIEOv0Plj2BXMO2Of8IfXYtz52E0b/24jZBTN80doRDp0pPRv56MzPWmwd6fHjksKbp3754wZkF6kGYJQCnlDRwCir1kmQMwIq1isUh29tDwa2jxA5z9l4B1bVjZ2ZNcWTLSbdYu5u48Z+4IhUj3Nm3axPjx41NsfwMHDqRWrVoJP48fP55Nmzal2P5TW1p2BnceKAOsfMmyIGDtqzYMDAw0+SBBQUEv7T003SjXFbL7wJ9vk2d+M5a2nMmHO3PwxeLDnLj6gOHN/HCQwmEhkiS17849PFKvGndwcHDKd4yptU7TD7DmuZ+zAAuA2sCQ55ZpY4g26M45radX1XqUm44L+VGPXXFEew9eoTv8HKJvRTw2d3RCpJk2bdroBg0aPDNv+fLlWimlz549qydMmKALFSqkHR0dtYeHh+7Ro4eOiIjQWmsdHh6uAb1v3z6ttdY1a9bU/fr101prHRcXp8ePH699fHy0m5ubbtWqlf7222+1q6trwnG2b9+uq1evrrNkyaIzZ86sq1atqkNCQrTWWo8cOVI/uUYBOjw8XI8cOVKXKVMmYfu9e/fq2rVr66xZs2pvb2/94Ycf6gcPHiQs9/b21nPnztWffPKJzps3r86RI4ceNGiQNhgMyfqdPXXtfO312BK6gx4ATAakusvT3PLDu2tgcS/s1g5laNmuFG83gEGLT9Dih38J7hqIXx7z9DMirNzqIcaebFOLZyloPM7k1Tt27EinTp24c+dOwihff/zxB3Xr1uXYsWMMGjSICRMmULt2bY4fP07fvn0pUKAAI0a8/q3ypEmTGDFiBBMmTKBKlSosWrSIzz//PGH0L601rVq1oly5cixbtoyYmBgmTpxIly5dOHPmDH379uXChQscPnyYuXPnkjdv3mf2f+nSJapVq0br1q355ptvuHHjBv379+fy5cssWrQoYb0xY8bQsWNHFi5cyNy5c5kwYQI1atSgadOmJv+OksoSEkBRjHf/bkBWpdQurfUG84ZkITJmhva/waaxsOVb3rp5kiJdf+DdhedpMyOESe3L0LhUbnNHKUSqatKkCZkyZWL58uV069aNR48esXTpUn766Sfc3d356aef6NWrF2B8XTxnzhzOnDnzxv1OnDiRgQMH0rdv34RtDx48yL///gvAo0eP6NevH2+//TZeXl4AXL9+nbfffhuAbNmy4e7ujpOTEz4+Pi/sf/r06eTOnZtffvkloVdQV1dXatSowalTpxK2KVKkCKNGjQKMYx3/+uuvhIWFJeM3ZjqzJQCl1Ahggda6c/zPtYBKcvF/jp0d1BkGHiVgSR9KrGjJ6razee+fOHrP3UvfOj70r+eLnZ2MLSBSSCLuztOCs7MzLVu2ZOHChXTr1o1Vq1ZhZ2dHq1atcHZ2Jm/evCxatIiDBw+yY8cO1q9fT7du3V67zzt37nD16lXq1q37zPzatWsnJABnZ2f69evHzp07WbRoEaGhoaxatcrkuI8dO0a1atWe6RK6cuXKZMiQgePHjyckgKcHp8+YMWOyaiElVpqXJmqtG8X/O0ZrffSp+Zu01pb1zbMkJVrBe+vALgPZ/mzBX5XO0S7Ai6kbThH0WygPomLMHaEQqaZjx46sXbuWBw8e8Mcff9ChQwecnZ1ZvHgxJUuW5Ndff8XDw4MxY8aY9OrkVf3029n9/yXx+vXrVKxYkX79+nHr1i06derEhAkTTI5Zv6T9klIKpdQzA8knZnD6lCbVSdITz1IQtBHyVcBhWW8mZPmTUU192XjiOm/9uI3TNyLMHaEQqaJBgwY4OTkxf/58Vq5cSY8ePQD49ttv+eCDD1iyZAm9e/emfPnyXL9+/Y37y5o1K3nz5mX9+vXPzN+yZUvC9JIlS7h58yZ79+5l9OjRNGrUiKioKJNjLl68OCEhIcTFxSXM27FjBzExMfj5+Zm8n9QkCSC9cckBXRdDhV6oHT/SPfwz/ni7KHciY3jrh22sP3bN3BEKkeIcHR1p06YNgwYNIl++fFSqVAkAFxcXVq1axaZNm1i1ahXdunUjNDSUsLAwLl++/Np9fvrpp3z33XdMmzaNvXv3MmrUqGfq8Lu4uHDlyhXmzJnDzp07GTduXELB8pNEYW9vz5UrVzhz5gwxMc8+hffp04eLFy/y7rvvsmvXLlasWME777xDixYt8PX1TcHfTtJZQiHwG72sXn+67g46uewdoMkE8CwJKwZQ/k5rVnWcxXurI3n/1z0MqOfLh7V9pFxAWJWOHTvy3//+l4EDByZ0dTB16lR69uxJkyZN8Pb2pnv37qxdu5auXbsya9ashALbl+nXrx+xsbFMnTqV4cOHU6tWLWbPnp3QUrhDhw6EhITw2WefYWdnR506ddixYwc9evQgKCiI48eP0759exYvXkypUqU4fvz4M/v38vJi69atDBw4kPr16+Pq6krz5s0ZNy5l33S/rDtoU0lfQOndhd3w59vw+AHRzacx+FhhFu+7RKMSnnzXvgyZM6aLHC+ESEHSGZwteXAV/uwKF3ehq37CrIxdGLsmjEI5XAjuFiidyQlhYyQB2JrYx7B6EIT+Aj712FF2Ar0XnibWoPm+o7+MNCaEDZEEYKv2zIZVn4GrF1ebzOK91Q85euU+n9Tz5SMpFxDCJkgCsGXnd8JfXeFxBNHNf2DwsUIs3neJ+n4eTGpfhiyZpNcNIayZJABbd/+KMQlc3I2u2p85Tl35ctVJvLM7E9w1EJ9cmc0doRAilVhVAujZs+cLy2y6GqipYh/D6sEQOhsK12FPwLf0WhjO41gD37UrQ6OSnuaOUAiRTC+rBjpz5kzAShKAJceYLoTOgVUDIUturjedRc91jzlw4S69axVmYIOi2Eu5gBBWxaqeACw5xnTj4h5jVdFHd4hp9j0jw/34387zVPPJwfcd/cmeOaO5IxRCpBBJAOJFEdfhr3fgfAhU/IAF2Xvx+bIT5HBxZMbbAZTJ52buCIUQKUASgHi5uBhYNxx2zoB8FTlWbRrvL77EjQePGd2yBJ0q5Dd3hEKIZJIEIF7v0AJY1hccnXnQbCZ9QpzZGnaTdgFefPlWSTI5vLy7XCGE5ZMEIN7s+nFjP0K3z2CoO5LJDxsybeNpiufOyowu5SggXUgIkS5ZVQKQaqCpKOo+LPsIji6F4s3Z4jeajxedxmDQfCtVRYWweFINVCSP1rD9R/h7BLgX4Fqjnwla95gDF+8RVKMQnzUsioO9DB0hRHphVU8AlhyjVTkXAgveNVYVbTSBMRcD+G3HOcoXcOeHzuXwyJrJ3BEKIUwgCUAkTcQNWPgehG8G/y6s8BrAZ0tP4exoz5SO/lQvktPcEQoh3kASgEg6QxxsngCbx0Ou4pytM4Oeq+5x6kYEH9f2oV89X2k9LIQFkwQgku/UeljUE2If87jxZD4PK8rCvRepXCg733fyJ1cWeSUkhCWSBCBSxr1LxnKBCzsgoAcLc33IFytOkTmjA1M7+lPFJ4e5IxRCPMeqEoBUAzWzuBjY8BVsmwIeJTlT+0d6rrzLmZsP6Ve3CB/XKSKvhIQwE6kGKtJG2N+wuBfERBHVaCJDTxVn8b5LVCmcnSkd/MkltYSEsAhW9QRgyTHanPuXYcF7cD4EXbYrizz7MmzFGZwd7ZnUwZ+avlJLSAhzkwQgUk9cLGz6BrZOhJzFOFfnB4LWRHLi2gM+qFmYTxv4SsMxIcxIEoBIfafWG18JPY4gpsE3jLgQwLzdFyiX342pncri5e5s7giFsEmmJoA0vU1TStkppZY8Ny+/Umq3UuqQUuq9tIxHJJNPXfhgG+SviMOq/nxjmMSMtj6EXYugyfdbWXP4irkjFEK8Rpo9ASilvIFVgL3WuthT878CdgKrge1a6/JPLZMngPTAYDDWENrwFbh6cbX+dII2wsGL9+hSMT/Dm/lJ99JCpCFLfAI4D5QBzj03fxmwEXACHqVhPCKl2NlB9QHw7hrQGs+FLVlUeje9qhdg7s7ztPjhX05cfWDuKIUQz0nzMgCl1BqtdaPn5hUBdgMTtNZjn5qvAQICAkzef1BQEEFBQSkUrUi0R3dh2cdwbBkUqs32Ml/z8fLLPIiKZXgzP7pUzJ9wdyKEMF1wcDDBwcEmrRsaGgpYYCHw8wlAKZVNa31bKZUJ+AdooLWOjF8mr4DSI60hdDas+RwcXbjXcCofh+Zky8kbNCrhybg2pXBzdjR3lEJYLUt8BfQqM5VSxYBoQMV/RHqmFAS+C0GbILMHros7Myf3QoY3Ksz649do8v1Wdp65Ze4ohbB5ZksASqkRSik/YDKwGNgL/Km1fmiumEQKy1UMem6ACr1QO3/ivWPvs6JjThwz2NFp5g4mrTtBbJzB3FEKYbOkHYBIGyfWwNI+EB1JVP2xDDtXjgV7L1EuvxvfdyxLvmzSZkCIlCINwYTluX/F2HAsfDMUa8bqQp8zaOVFAL5qVZKW/nnNHKAQ1kESgLBMBgNsnwbrvwSXnFyvN5XeIS6EnrtD67J5Gd2yBFkyOZg7SiHSNatKANIdtBW6vN849OSt0xiq9GWa7sD3m87i5e7MlI7+lMvvbu4IhUgXpDtokT5FP4Q1Q2HvHMjtz8FKE+m9+j5X70fRr24RPqztI+MMCJEEVvUEYMkxihRwbLmx8VhsNJF1v2bImTIsO3iF8gXcmdzBXzqVEyKRJAGI9OX+5fgC4i3oYs1YVXAog1ddQin4ulUpWpTJY+4IhUg3JAGI9MdggB0/wj+jwSUH1+pM4YPtWdh3/i6ty+VldAspIBbCFJIARPp15SAsfB9unsBQ6UOmqc58v+kced2dmNLBnwDvbOaOUAiLJglApG/RkfD3CNg9EzxKcqTyRD5YF8mlO4/4qE4R+tbxIYOMOibES1lVApBqoDbs5FpY+iFE3Seq9ki+uFSZhfuu4J/Pje87+uOd3cXcEQphVlINVFi3iOuw9CMIWwuF67KuyAgGrrlGnEEzskUJ2gV4SRfTQjzFqp4ALDlGkUa0hj2zYO0X4ODE7brf0WdvHnacuU3jkp6MbVUKdxfpYloIkAQgrNXNMGMB8ZX9GPzf5pcsvfhmw0XcnR35rl0ZavjmNHeEQpidJABhveJiYNM4+HcSuOXnTI3JBG2059T1CLpXKcCQxsVkDGJh0yQBCOt3bjssDoJ7F4mt8gljI1sya/tFiuTKzJSO/pTI42ruCIUwC0kAwjZE3Yc1Q2D/XMjtz65y4/hw3UPuRkbzaYOi9KxeSPoTEjbHqhKAVAMVb3R0GSzvBzGPeFhzJAPCA1l79DoVCmZjUvsy0p+QsFpSDVQIgAdXjW0GTv2D9qnHigKfM/TvmyhgdMsStCqbV6qLCptgVU8AlhyjsDBaw+7/wLrh4JCJm7XG03tfPnafvUPTUrn5ulVJ3JyluqiwbpIAhG27GQaLguDyXgyl2jMrax/Gb7pCNhdHvm0r1UWFdZMEIERcDGydCJsnQBZPwqt9S9C/mQm7HkG3yt4MbVwcJ0epLiqsjyQAIZ64FAqLesGtMGLL9+LbuI78HHKFQjlcmNzBnzL53MwdoRApShKAEE+LjoR/RsKuYMjhy4Hy4+m9QXPtwWM+qu3DR3V8cJDeRYWVkAQgxMuc3gBLPoSIa0RVGcCwW41YsP8apb1cmdTeH59cmc0doRDJZlUJQNoBiBT16C6sHgwH/4Dc/mwp+TX91kcSGR3HkMbFeKdyAeyk8ZhIJ6QdgBBJcXQpLO8P0Q95UO1z+p+tzPoTN6lcKDvftS9DXjcnc0coRJJY1ROAJcco0rmI67CsL5xcjfauyoqCwxiy4T52SjGiuR9tZawBkQ5JAhDCVFob+xJaPQTQ3K42gg+OlmLX2TvU9/Pgm9alyJE5o7mjFMJkFpkAlFJ2wCKt9VtPzXMBlgC5gF1AkI4PShKASFN3L8Cyj+DMJnThuszz/IxRm++SOWMGvn6rJI1L5TZ3hEKYxNQEkGb13pRS3sAhoNhzi9oAGwF/QANV0yomIZ7hlg+6LoGmE1Hnt9N5Twe2NLhKXtdM9J67l77z9nHnYbS5oxQixaRlxefzQBng3HPzw4B58Xf9V9MwHiFepBSUfx96bwOPEnhu6M/SHNMZVjMbqw5docGULfxz9Jq5oxQiRaR5GYBSao3WutFz8xTQBegBNNJax8TP1wABAQEm7z8oKIigoKCUC1jYLkMc7JgB68eAozMXKo+hZ6g3x69F0KacFyOa++Hq5GDuKIWNCA4OJjg42KR1Q0NDAQsrA4AXE0D8xX884AF8pLV+8NQyKQMQ5nfjJCztAxd3YyjalJ8yf8jE7ffIlSUj49qUpqZ0LCcsjMWVAbxGq/h/uz998RfCYuT0hXfXQv0vsTv1D32OdmFDgxu4ONrzzqxdDF5wkPtRMeaOUohEM1sCUEqNUEr5ARWAusBGpdQmpVQ1c8UkxCvZ2UPVvvDBv5DdB+9NfVmbO5iBVdyYH3qBhpO3sOnEdXNHKUSiSDsAIRLLEAfbf4QNX4GDE+cqjOT9fYUIu/GQ9oFefNFUygaEeVlkO4DEkgQgLNrNMOMQlBd2Ele4PsFZP+bbHRHkypKJb1qXonaxXOaOUNgoSQBCpAVDHOyaCetHg7LnQvmhvHfQj5M3ImldNi/Dm/nh7iJDUIq0ZVUJQHoDFRbvzlljn0LhmzF4V2dW9k8Yt+Mxbs4OjGlZkibSilikEukNVAhLoDXs/RXWDYO4GK4FDCAorCIHLkfQqIQnY1qWIFfWTOaOUtgAq3oCsOQYhXjB/cuw8lM4sQrtWYYFXoP5YocdmTLYMayZH+2kh1GRyiQBCGFOWhvHG1j1GUTe4q5/Lz68XJ9t5x5RzScHX7cqiXd2F3NHKayUJAAhLMGjO7BuOOz7De1ekH98PueTXa7EGgz0r+fL+9UKkkHGIhYpTBKAEJYkfAss7we3z/DIrwNfPOzAohNR+OXOyrg2pSjt5WbuCIUVkQQghKWJeQSbJ0DIVMiYlUMlBvLefl9uPoymR9WCfNrAF2fHDOaOUlgBSQBCWKprR2FFf7iwk9h8VZjm/CHfH1DkdXNidIsS1PPzMHeEIp2zqgQg7QCE1TEYYN+v8PcIiHnEpZIfEBRegyPXo2lYwoORzUuQRwalFyaQdgBCpFcR12Ht53BoPtq9ECvzDWDgvhzYK8Un9X3pXqWAFBKLRLOqJwBLjlGIFHF6A6wcCLdPE+nTjGGPOrPoNJTIk5WvW5XCP5+buSMU6YgkACHSm9jHsG0qbP0Orew5UbQP754I5EpEHJ0q5OezBkWlXyFhEkkAQqRXd87C6sFwcg1xOYoxN/vHjD6UjayZMjC4UTHaB+bDzk5aEotXkwQgRHp3fJUxEdw7z32flgy635Y15+3xz+fGly1LUsrL1dwRCgslCUAIaxAdCdumwL9T0HYZOOrTk/dPVuJqpIEuFfMzsEFR3JzltZB4llUlAKkGKmze7XBjL6PHVxDnXpA/s/dh2JG8ZHVyYGCDonSqkB97eS1kk6QaqBC24tR642uhW2E8yF+XEVGdWHzemeK5szKquR8VC2U3d4TCAljVE4AlxyhEmouNhl0/w6bx6NhHnC3UhQ8u1OfEPTual8nD0MbFpBGZjZMEIIS1e3ANNnwJ+35HO2VjQ+6efHyyFFploHetwvSsXggnR3tzRynMQBKAELbiygFYMxTObSM6ezGmZ3yfKWfykMc1E4MbF6NFmTwyAI2NkQQghC3RGo4tMxYU3z3Pba+6DH3QlrXXXCmb343hzfwol9/d3FGKNCIJQAhbFBMFO6bD1knomEhO529Hn0sNOBmRiZb+eRjcSMoHbIFVJQCpBipEIkXcgM3jYc8stIMTWz3f4aMzFXmMI+9VK8gHtQqTNZODuaMUKUCqgQohXu7GSfhnJJxYRWzmPPyRpTvDw/1wd8lEv7pF6FwxPw7S26jVsaonAEuOUYh0IXyrsXzgyn4eZSvOVDoz43IhCubIzOBGRWlYwlMKiq2IJAAhxLMMBjiyCDZ8BXfCuZOzPCMftmPZbS8CvN0Z3KgYFQpmM3eUIgVIAhBCvFxsNOydYxyf+OF1LnrUZsCtluyKyEWtojkZ2KAoJfNKR3PpmUUmAKWUHbBIa/2WifMlAQiRWh5HwM4ZsG0qOjqCk7kaM/BaQw5F5aBZ6dwMqO9LoZyZzR2lSAKLSwBKKW9gFWCvtS72pvnxyyQBCJHaHt6CbZNh13/QcdEcztGET6/V53RsTtoFeNG3bhGpOprOWGICUIA9sFJr3fBN8+OXaYCAgACTjxMUFERQUFCKxCyETXlwzdj19J5ZaEMse90b8+nVBlwiBx3K56NPLR9JBGYUHBxMcHCwSeuGhoYCFpQAEg6o1BqtdSNT5ssTgBBmcP8K/DsZQmejtWa3W2OGXqvDeTxoH5iPPrV9yCuJwKJZ3BNAwgElAQiRPty7BFsnGjubM8RwwK0eQ6/X4xT5aBeYjz61CuPl7mzuKMVLSAIQQqSMB1chZBrsmQ0xDznqWoNhtxpy0FCIt8rm5YOahfHJJYXFlsTiE4BSagSwQGt99On5z60rCUAISxF5G3b+ZPxE3eN0lvJ8dbcBm2L9aOiXmz61C1Pay83cUQosOAEkhiQAISxQ1H3Y81/YMQMirnHdxZdJDxuyIKo8FX086FPLhyqFs0vLYjOSBCCESF2xj+HgX8bXQzdP8CCjJ/+Jach/I6tTIK8n71crRNPSuaWvITOQBCCESBsGA5z6G7ZNhXP/Ep0hC0tVHb6PqE1slvx0r1qATuXz4+osvY+mFatKANIdtBDpxKVQ2P4j+uhSMMQRmqkSE+/X4UCGUrQPzE+PqgXwzu5i7iitinQHLYSwLPcvw55ZxppDkTe5krEgPzysy6K4qlQumo9ulb2pUSQndnZSTpAarOoJwJJjFEK8RkwUHF5orDl09SBR9plZbKjBf6NqEZe9KF0redM20EsGp0lhkgCEEJZDazi/A/b8F310KSoumiMOJfn5YS02Z6hEs7IF6FLRG788Wc0dqVWQBCCEsEwPb8K+3yF0Ntw5S4S9K/+LqcG8mFq4ehWnc4X8NCuTG2fHDOaONN2SBCCEsGwGA5zZaOx87sRqlI7jkL0fc6KqsyVDVRqWLUznivkpnlueChJLEoAQIv14cBUOzEPv+x116xRRdk4si63MvJiaxOUJoG1gPlqUyYObs6O5I00XrCoBSDVQIWzEk7KCfb+hjyxGxURywc6LPx5XYRXV8PMrRdtAL2oUyYm91CACpBqoEMIaPX4ARxbDgT/h3L8AhFKcv2Kqsce5OvXK+tLSPy/Fc2eRbieeY1VPAJYcoxAiDdw5B4f+Qh/4E3UrjBgcWG8oy9LYylzIUY3GZQvR0j+PdE8dTxKAEML6aA2X98KBPzEcXohd5E0eKSfWxpZleVxlIvPVoEnZgjQu6UmOzBnNHa3ZSAIQQlg3Qxyc/RcOLyTu6DLso+4QgQurYwNYqysQ412T+qW9aWSDyUASgBDCdsTFwJlN6MMLMBxbiX30AyLJxPo4f9YaKvAwX23q+BemoZ8HubJmMne0qU4SgBDCNsVGQ/gW9LFlxB1bSYZHN4nGgS1xJfnbEMgNz5oElipOAz9Pqx3JzKoSgFQDFUIkiSEOLuxEH11G7JHlOERcBOCAoRAb4spy3LUqBUpWpr6fJ2Xzu6fLqqVSDVQIId5Ea7h+FE6sJvrYahyuhKLQXNXubIzzJ9ShLPaFa1O5RGFq+OYkm0v6bXRmVU8AlhyjECKdirgBp/4m5ugqOLMRh9gI4rBjv6EwWwyluZKzKvlKVKWqrwel87qSIR2NbCYJQAghTBUXAxf3oE/9w6Njf+N08yAKzV3twg6DH3vtShGVryo+fgFULZKTQjlcLLrxmSQAIYRIqoe34MxGHp/4h7gzW3COvATADe3KdoMfRzL6o72r4VO0NBULZyd/NmeLSgiSAIQQIqXcOQvhW4k4sQG7s1txfnwDgOvajV2GYhzPWIo4r0p4Fw+kQqEcFDTzE4IkACGESA1aw61T6LPbeHByM/bnQ3CJugrAXe1CqMGXYxmKEeUZQLYilShT2IuSebOSMYN9moUoCUAIIdLK3fPos9u4f2IznN+B68NwAOK04rjOz358ue1eBof85clfpBT++bOR2zVTqj0lWFUCkHYAQoh0JfI2XArl4ekQHp3ZTpabB8hoiATgvnbisKEgpxx8eZSzNM4FK1C4cDFKeLnh6pT4sZGlHcBrBAcHExQUlGIxpSdy7rZ57mDb52+R526IgxvHibmwh3undsGlvbg9OEkGYgG4rTNz2FCQi5mK8DhHSZy9y5HfpyQlvdzIkilxSSE4OJhevXoBkgAIDAxkz549KRZTeiLnbpvnDrZ9/unm3GOi4NoRIsN3ce9sKBmuHcQ94nRCUnignQh1LE+tL5a/YUfPCgwMJDQ0FHhzApBRl4UQwhwcMoFXAM5eAThXj58XGw03jhFxdi/3z+zB294lVUNI0wSglLIDFmmt33pqXibgdyAPMFNrPTstYxJCCIuRwRFylyFz7jJkrtwj1Q+XZm2blVLewCGg2HOLWgH/AtWB7kopeSoRQog0kJadW5wHygDnnpsfAOzRWscBF4D8aRiTEELYrDRLANooFni+RNcVuBw/fRlwf37bwMBAAgMDKVKkSML0qz7BwcHJivP56lSWsk1SjpEUaRFXWm2TFseQv33qHiOp26TFMdL69xUcHPzG619gYCDHjh0zef+W0L3dPcArfjofcPf5Ffbs2cOePXuoXbt2wvSrPsmt/mWpX2q5CFjHRSAp28jf3jr+9sk9RlBQ0Buvf3v27KF48eIm798SEsAeIEApZQ/kxfiqSAghRCozWwJQSo1QSvkBS4CqGAuCZ2mtY8wVkxBC2JI0r3GjtW4U/++Yp2a3Tes4hBDC1qWLlsBCCCES700tgS2hDEAIIYQZWPQTgBBCiNQjTwBCCGGjJAEIIYSNkgQghBA2ShKAEELYKEkAQghho6w2ASilMimlFiilQpRSqd+xtoVQStkppZbET1dQSm1XSm1WSll1L6tKKRel1N9KqQNKqZlKqcJKqS3xf/+y5o4vtSml8iuldiulDiml3lNKtYj/269WSrmZO760oJRqr5QaYmPfe1+l1H6l1Cal1NLEnrvVVgNVSnUCPIBpwAagbnxvpFYrfsyFVYC91rqYUmol8A7gA3TUWvc3Z3ypSSnVDWOngt8APwOFgU+Ba8B0rXUrM4aX6pRSXwE7gdXAdiASaAi8BXhqraeYLbg0oJRyAPZhHFyqOrbzvW8C5NVaz4z/OVH/5632CQDbHGfg+TEXsmutb2L8j+FvrqDSSBgwTxvvaK4CtYFDWusrGG8ErN0yYCPgBNgDMVrrKIydLZYzZ2BpJAhYGz9tS9/7gkAbpdQ2pVRXEnnu1pwA3jjOgLV5yZgLD+PnPwas+ulHa70dOKuUehtj54Ib45M/GO+GrZrWeheQG+PNzkJs6LuvlMqCMeGviJ9lM9974DQwAmgO9CeR527Nwy8+GWfgDK8YZ8AGZAFQSjkBjmaOJVUppRQwHuPd/lvAOqVUhviEmNWcsaUFpVQ2rXWYUsoTY8+69+MX2cJ3fwAwGXCI/9lmvvda6zVPppVSIcDb8dMmnbs1J4An4wxsw3bHGbihlMqJ8X3gXnMHk8qevOPvrrXWSqkDQCml1DXgkhnjSiszlVJfACeBx0AWpVQmjK9/Qs0aWeorivEJwA1jss9tK997pdSXwBqM5T8lgf2JOXdrTgBLMBYItQd+ttFxBkZhfDf8GOhm3lBSXQWgLrDR+DDAcOB7jN/xPmaMK61MBhZj/FvPAsIxlgncATqbMa5Up7XuDKCUqgVUAtZjO9/7GRjP1RGYjjHZm3zuVlsLSAghxOtZcyGwEEKI15AEIIQQNkoSgBBC2ChJAEIIYaMkAQghhI2SBCCEEC+hlPpFKaVf8bmbRjFsUkpNSa39W3M7ACGESK5dQJeXzDekdSCpQRKAEEK82iOt9SlzB5Fa5BWQEEIkgVKqQPzroMpKqfVKqXtKqb1KqWZPrWOnlOqrlDqilIpQSu1TSnV4bj8l4seyuKeUuqiUGquUsn92FTVGKXVeKXUnfryLDPELnJVSPyqlLiulIpVSe5RSNU09B0kAQgiRPAuA+UA9jB3xLVdKBcYv+xjjGBXTgBrAn8BcpVQLgPh+e0Iw9lVWH/gS6Iexg7snumPs0bUt8B3wPtApftmo+Ol+8cc/Dyx8LoG8krwCEkKIV6uhlIp6yfzVwCfx07O01j8BKKX2AGUxXpC7AoOA8U+WA3uVUkWAzzD22dMH46BFQfHdl+9SSrny7Pgl4UDf+LEudsV3eV44fpkfxo4vF8R3gtgbY8d4jsCjN52cJAAhhHi1Pby8U7WHGAfeAeOIg4BxTA6l1HqgWfyFPA+w5bltNwMt46dLASFPjV2B1nrCc+vv0s922vbwqemZwGzguFJqGbAc+FOb2MmbJAAhhHi1SK318ZctUEoViJ98vkZQHK+/thqeWu7wku1fiOFVC7TWS5VSXkA7oAHGp4pwpVQjrfW1N+xXygCEECKZqjz3cx3gmNb6HnAF4xjFT6sJHI2fPgpUUkolXIuVUv2UUktNObBSajhQQWs9R2vdBSiAcXyEZq/dMJ48AQghxKs5KaV8XrHsyfVzsFLqCnAI44hctTCOSwDwLfBl/MBEuzHepffAeMcOxj78+wE/KaVmAsUwDvE4zcT4qgKdlFKDMY6FXQfIBBw0ZWNJAEII8WoVgLBXLCsf/+9HQC+Mg7CfBlporXfGL/se4xjdn2As2D0FdNVaLwbQWl9QStUAJmEsS7gL/Ax8bWJ8vTEmizkYL/wngM5a692mbCwDwgghRBLElwGEA2W11vvNG03SSBmAEELYKEkAQghho+QVkBBC2Ch5AhBCCBslCUAIIWyUJAAhhLBRkgCEEMJGSQIQQggb9X83uDUa+ed1xwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "batch_size = 50\n", + "losses_train_ssl, losses_valid_ssl = train_mlpf(\n", + " data, batch_size, model_ssl, with_VICReg=True, epochs=50\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": {}, + "outputs": [], + "source": [ + "class MLPF_native(nn.Module):\n", + " def __init__(\n", + " self,\n", + " input_dim=8,\n", + " embedding_dim=34,\n", + " num_classes=6,\n", + " num_convs=2,\n", + " k=8,\n", + " ):\n", + " super(MLPF_native, self).__init__()\n", + "\n", + " self.act = nn.ELU\n", + "\n", + " # embedding\n", + " self.nn0 = nn.Sequential(\n", + " nn.Linear(input_dim, 126),\n", + " self.act(),\n", + " nn.Linear(126, 126),\n", + " self.act(),\n", + " nn.Linear(126, 126),\n", + " self.act(),\n", + " nn.Linear(126, embedding_dim),\n", + " )\n", + "\n", + " # GNN that uses the embeddings learnt by VICReg as the input features\n", + " self.conv = nn.ModuleList()\n", + " for i in range(num_convs):\n", + " self.conv.append(\n", + " GravNetConv(\n", + " embedding_dim,\n", + " embedding_dim,\n", + " space_dimensions=4,\n", + " propagate_dimensions=22,\n", + " k=k,\n", + " )\n", + " )\n", + "\n", + " # DNN that acts on the node level to predict the PID\n", + " self.nn = nn.Sequential(\n", + " nn.Linear(embedding_dim, 126),\n", + " self.act(),\n", + " nn.Linear(126, 126),\n", + " self.act(),\n", + " nn.Linear(126, num_classes),\n", + " )\n", + "\n", + " def forward(self, batch):\n", + "\n", + " # unfold the Batch object\n", + " input_ = batch.x.float()\n", + " batch = batch.batch\n", + "\n", + " # embedding\n", + " embedding = self.nn0(input_)\n", + "\n", + " # perform a series of graph convolutions\n", + " for num, conv in enumerate(self.conv):\n", + " embedding = conv(embedding, batch)\n", + "\n", + " # predict the PIDs\n", + " preds_id = self.nn(embedding)\n", + "\n", + " return preds_id" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Num of model paramaters: 66326\n" + ] + } + ], + "source": [ + "model_native = MLPF_native(input_dim=COMMON_X + 1)\n", + "print(\n", + " \"Num of model paramaters: \",\n", + " sum(p.numel() for p in model_native.parameters() if p.requires_grad),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e1ff10c3f14147b5ae4bffc8f5935d7d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=50.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "20aa14d866024f18bad25710a34d3954", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=80.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# train native MLPF\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m50\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mlosses_train_native\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlosses_valid_native\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_mlpf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_native\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwith_VICReg\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain_mlpf\u001b[0;34m(data, batch_size, model, with_VICReg, epochs)\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0;31m# make mlpf forward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m \u001b[0mpred_ids_one_hot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 65\u001b[0m \u001b[0mpred_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred_ids_one_hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mtarget_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mygen_id\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[0;31m# perform a series of graph convolutions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 52\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mnum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 53\u001b[0;31m \u001b[0membedding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membedding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[0;31m# predict the PIDs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch_geometric/nn/conv/gravnet_conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, batch)\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[0;31m# propagate_type: (x: OptPairTensor, edge_weight: OptTensor)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m out = self.propagate(edge_index, x=(h_l, None),\n\u001b[0m\u001b[1;32m 101\u001b[0m \u001b[0medge_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0medge_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 102\u001b[0m size=(s_l.size(0), s_r.size(0)))\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch_geometric/nn/conv/message_passing.py\u001b[0m in \u001b[0;36mpropagate\u001b[0;34m(self, edge_index, size, **kwargs)\u001b[0m\n\u001b[1;32m 292\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[0maggr_kwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 294\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maggregate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0maggr_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 295\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_aggregate_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 296\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0maggr_kwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch_geometric/nn/conv/gravnet_conv.py\u001b[0m in \u001b[0;36maggregate\u001b[0;34m(self, inputs, index, dim_size)\u001b[0m\n\u001b[1;32m 111\u001b[0m out_mean = scatter(inputs, index, dim=self.node_dim, dim_size=dim_size,\n\u001b[1;32m 112\u001b[0m reduce='mean')\n\u001b[0;32m--> 113\u001b[0;31m out_max = scatter(inputs, index, dim=self.node_dim, dim_size=dim_size,\n\u001b[0m\u001b[1;32m 114\u001b[0m reduce='max')\n\u001b[1;32m 115\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mout_mean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_max\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch_scatter/scatter.py\u001b[0m in \u001b[0;36mscatter\u001b[0;34m(src, index, dim, out, dim_size, reduce)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mscatter_min\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mreduce\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'max'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 161\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mscatter_max\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 162\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/pyg-coffea/lib/python3.8/site-packages/torch_scatter/scatter.py\u001b[0m in \u001b[0;36mscatter_max\u001b[0;34m(src, index, dim, out, dim_size)\u001b[0m\n\u001b[1;32m 71\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 72\u001b[0m dim_size: Optional[int] = None) -> Tuple[torch.Tensor, torch.Tensor]:\n\u001b[0;32m---> 73\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtorch_scatter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscatter_max\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 74\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "# train native MLPF\n", + "batch_size = 50\n", + "losses_train_native, losses_valid_native = train_mlpf(\n", + " data, batch_size, model_native, with_VICReg=False, epochs=50\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compare native vs SSL" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAF1CAYAAADBdGLoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAAsTAAALEwEAmpwYAABCA0lEQVR4nO3deXxU1f3/8dfJvpMEwpaFXZBFUAICbiCodYnYakWpVq2KS5Xa2m/1p63VWrVa695WwVa0blTrhpW6Am6syiourGEJBEIC2ffz++NOVoIkYTJ3ZvJ+Ph7zmDv3ztz5zBh8z7nn3nOMtRYREREJHCFuFyAiIiJto/AWEREJMApvERGRAKPwFhERCTAKbxERkQAT5nYB38cYo1PhRUSkU7LWmkNtU8tbREQkwAREeFtr629XX311k8ftuY0ePfqI9+GNOo50H/5QQzB9n/60D32n+j79fR/6Tr27j8bfZ2sERHiLiIhIgyMKb2NMP2NMuLeKERERkcNrdXgbY9KNMa8aY870PH4e2ATkGGOO7agCRUREpKm2tLz/BgwBthljBgI/Bs4A3gH+1AG1iYiISAvacqnYScAF1tqvjDG3A+9ba983xlQAr3dMeQfLysry1Vt9L2/UcaT78IcavMVfPou/7MMb/OWz+MPfuTf4w3fhT/vwBn/5LP6yj7YwrT2zzRizD/ixtfYjY8wi4E1r7UPGmFOAt6218V4vznOdd2trbK3MzExWrFjh1X12Zvo+vU/fqXfp+/Q+fafe1fj7NMa5vNt+z3XebWl5fwbcY4x5ExgL/NgYkwbMBNa3t2ARERFpm7b0ed8EhAK3Aw9Ya/cAvwAmA7d6vzQRERFpSatb3tbazTgt7sb+BPzWWlvh1apERETkkNp0nbcxZoQxJt2zfBLwEHCdqTtALyIiIh2uLdd5XwusAsZ5BmaZC/QHbgN+1yHVdZAZM2a4XUJQ0ffpffpOvUvfp/fpO/Wutn6fbTnbfCPwL+APwAU4h8wHApcAv7fWDmzTO7fuPb17tnltLWz7HBJSIbmfd/YpIiLiRd4+2zwV55Iwa4w5DZjvWV7n2dZhWvpFkpWV1fbr6sr3w3PnwdgZ8IN7vVKbiIhIe82bN4958+a1+XVtaXl/CzzpuW0DLrbWfmCMuRy4x1rr9QDvkOu8514C2Yvh5m8gVMOyi4h/uvzyy3n22Wdb9dw+ffqwdetWr733xIkTWbRoEc888wyXX355m167detW+vVzjmx6e4yO1qr77tx6/yPl7Zb3E8CjOP3bez3B/TPgMWBO+8v0sVGXwNfz4Lt34ehz3K5GRKRFkydPJi4ursm6Z599luLiYn7+8583WZ+cnOzV9/7hD3/I8OHDGTJkSJtfGx8ff1B94n2tbnkDGGPOAo4GXrPWbjHGXAxkAI90xOViHdLyrqmGh46GtDFw8Yve26+ISAfr27cv2dnZAdui9BW1vJux1r6DMxFJ3eOX2luca0LDYOQ0WPJ3KN4LcSluVyQiItImbb3O+xxjzCJjzB5jTL4x5lNjzLkdVVyHGfUTqK2Gtf92uxIREa+ZM2cOxhh27NjBY489xuDBg5v0WX/88cecddZZ9O7dm8jISPr06cP06dP5+uuvm+znzjvvxBhT34++detWjDHMmTOHL774glNPPZW4uDh69uzJxRdfzLZt25q8vm/fvkycOPGg/Vlrefzxxxk0aBBRUVEcddRRPPDAA9TU1DR5fWVlJbfffjujRo0iKSmJc845hyVLlvDggw/Wf762stby3HPPMXHiRJKSkujTpw/nnnsuX3zxxUHP/eabb5g+fTp9+vQhOjqawYMHc+edd1JaWtrkecuWLePcc8+ld+/exMbGMmLECB599FGqq6vbXF9bteU67wtxZg/bANwM3Ah8Dbzq2RY4uh8NvY+DlS9AgB5WERE5lMcee4xf//rXpKamcuKJJwJO0EyZMoUPPviA0aNHM23aNGJjY3nppZeYNGkS+fn5h93vV199xcknn0xhYSHnn38+SUlJvPzyy5xzzjkHBXBL7rrrLm699VaGDBnCeeedR05ODrfccgsPPfRQ/XOqqqqYMmUK9957L9XV1ZxzzjlkZ2dz2mmn8fnnn7f7O5k5cyaXXXYZX3/9NaeddhrDhg3j/fffZ9y4cbzyyiv1z9u6dSsnnHACL7/8Mt26dWPatGmEhYVx1113ccMNN9Q/b/ny5Zx00knMnz+f/v378+Mf/5ji4mJuuukm7r3XB1czWWtbdQO+BG5pYf0twMrW7qctN8A6JXaAZbOt/X2CtTtXdsz+RUS8rE+fPt/7/8RnnnnGAjY5OdmuWbOmybbrrrvOAnbhwoVN1t9zzz0WsG+++Wb9ut///vcWsFu2bLHWWrtlyxYL2JCQEHv//ffb2tpaa621VVVVduLEiRaw69ata1LnKaecctD+evTo0eR569evtxERETYzM7N+3axZsyxgb7jhBltTU2OttbampsZef/31ti4Ttm/f/r3f02WXXdbke1q+fLkF7JgxY2xeXl79+pUrV9rExESbmppqS0tLrbXW3n333RawL730Uv3zqqqq7MiRI21ERIStrKy01lp75ZVXWsAuXry4/nlFRUW2R48etl+/ft9b3+E0yr5D5mNbDpsPAt5tYf27OIO1BJbh50NoJKx6we1KRES86vrrr2fEiBFN1p199tk8/fTTnHzyyU3WDx8+HKBVLe+jjz6aX//61/UnVIWFhXHBBRcAsHfv3sO+/uabb2bYsGFN9nfMMcc0ee3jjz9OYmIi9913HyEhTkSFhIRw3333ER/fvpmnZ8+eDcBDDz1E165d69ePGjWKG2+8kZ07d/LOO87pXHv27AEgJiam/nlhYWG8+uqrLFq0qP4kuJaeFxcXx4cffshLL3X86WBtCe9sYEwL68fgXPcdWKKTYMjZsPYVqNa8KiISPEaOHHnQurPPPpsrr7wSYwwlJSUsWbKEhx56iFtvbf2kkCeeeGJ9oNaJjY1t9etPOumkg9Y1fn1tbS3fffcdmZmZB10ml5CQwHHHHdfq92psw4YNhIeHM2HChIO21fXNb9y4EYCpU6cCMH36dK655hree+89ioqKGDhwIOPGjSMiIgKA8847D4BTTz2Vm2++mY8//piysjKGDRvG8ccf364626It4T0LeMgYc7sx5mTP7XbgL8BTHVNeBzv2J1BWAN/Od7sSERGvaem6771793L99ddz9NFHEx8fzwknnMAzzzxDamrrx9fq0aPHEdV1uNfv3buXioqKQz6vZ8+e7XrfnJwcunfvftAPD4BevXoB1J8EN3nyZP73v/8xcuRIZs+ezRlnnEFSUhKnnXYaH330Uf3rrrjiCl544QXS0tJ46KGHOOWUU0hOTua8887jyy+/bFedbdGW8H4c+C1wHbDQc7seuBMn2A/LGBNijHmj2boMY8xyY8xaY8yVbajnyPWfBPG9dehcRIJKSyE1depUZs2axaRJk3j77bc5cOAAa9eu5fbbb2/1fo90AsnDvb5r166EhYXVH5Ju7lDrD6d3797s3buX2trag7bl5uYCDSEOcMYZZ/DZZ5+xe/du5s6dyyWXXMKnn37K5MmTWbRoUf1nmT59OqtWrSI7O5tnn32Ws88+m7fffpsJEybw3XfftavW1mp1eHv60R+11qYBiUCidYZE/QwoOdzrjTF9gLVA8yF7ZuBMdnIscG1r6/GKkFAYeRFs/ACKdvv0rUVEfGXHjh0sXryYGTNm8Le//Y2zzjqr/rB0dna2y9U1CAsLY8CAAaxYsYKSkqaxUlJSwqpVq9q130GDBlFZWcnixYsP2rZw4cL65wD84x//4N5778VaS/fu3bnwwguZM2cO//63c2nxW2+9BcBf/vIX/va3vwGQkZHBT3/6U1599VUefvhhKioqePfdlk4R8542Xeddx1pbaK0tbOPLtgEjcfrOG3sLWABEA2XtqeeIjPoJ2FpY/bLP31pExBeKi4sByMvLa7J+69at/PGPfwSgvLzc53W15JprrqGgoIDbbrutvqVsreWOO+6goKCgXfu86qqrAOeEucYn5q1cuZJHHnmE3r17c9ZZZwHw/vvvc/vtt9e3sOvUnVTXu3dvAF555RVmzpzJhg0bvvd5HaVNI6wdCc/p79V1Q542Wr/MGDMIWA480NJrMzMzW/0+M2bMaNu8qN0GQvrxzqHzE34BR3hYSETE3wwaNIiRI0fyyiuvMHz4cEaPHs2ePXv48MMPOf3009m8eTP33HMPoaGhXH311a7Wet111/HCCy/w2GOPsWDBAkaNGsXatWvJyclh+vTpvPjiiyQmJrZpn2PGjOGGG27giSeeYOjQoZxyyikUFxfz0UcfUV1dzezZs+tPnLvqqquYO3cuZ511FmeeeSbx8fF88cUXrFu3jt69e3PxxRcDcPXVV7N06VLGjBnDmWeeSVhYGEuXLmXDhg2MGDGC008/vUkNs2bNYtasVvUwt4rPwvtQjDHJ1toNxpiewAfGmEestU2GsVmxYkXHFjFqOsz7Bez8AtJa/0NBRCQQhIaG8vbbb3Prrbfy4Ycf8tZbbzF69GjmzJnDxRdfzH333ccjjzxS3//rpqioKD7++GNuv/123nnnHebPn8/EiRN55ZVXeOqpp4iNjT3oTPTWeOyxxxgzZgz//Oc/ef/994mJiWHKlCnceeedjB49uv55U6ZM4e233+aBBx7gk08+obCwkLS0NK699lpuvfXW+hb1z372M8LDw3niiSd49913KS8vJyMjg1tuuYWbb775oMva2tKwbM25BW2amOQQb3I88Lm1NrSVz/+ftfYHjR7/B7gd+A74BDjdWlvi2eb9iUlaUl4IDx7l9H9nPdKx7yUiIoe0Y8cOKioqGDBgwEHbzj77bL777ruDDlUHmyOemMQY889WvE+7ZvYwxtwBvAo8jDPsagXwz7rg9qmoBBh6Lqx7DX5wH4RH+7wEERGBP/zhD8yePZsVK1Y0aREvWbKE+fPn8//+3/9zsTr/8b0tb2PMgtbuyFo7ySsVNX1/37S8ATYvhOemwvn/gBEXdPz7iYjIQdauXcuECRMIDQ3lzDPPZNiwYWzevJkXXniByMhIvvnmmw4/GcxtrWl5H/Fh847k0/CurYVHj4Fug+DS1zv+/UREpEVr1qzhj3/8I4sXL2bv3r1kZGQwcuRIfve733HMMce4XV6HU3i31Uf3wMd/hl+ugy5pvnlPERGRRloT3u26zjtojboYsLC64weVFxERaS+Fd2PJ/aHPCbDqRc3zLSIifkvh3dyon0D+Zti2xO1KREREWqTwbm7oVAiPhVXPu12JiIhIixTezUXGwbDz4Ks3oNL3l5yLiIgcjsK7JaN+ApXFsP4ttysRERE5iMK7JX0mQFI/zfMtIiJ+yfWJSVqjpcHcs7KyyMrK6pg3NMZpfS/4IxRshaS+HfM+IiLSasYYXn/9dc477zy3S/GaefPmMW/evDa/ToO0HMr+7fDICDjlFpiksXRFRNwWjOHdEg3SciQS06H/Kc41354J4UVERPyBwvv7jLoEDmyDrZ+4XYmISND44IMPOP7444mNjaVnz578/Oc/p7KyktLSUn7+85/Tu3dvYmJiyMzMZNGiRW6X65cCos/bNUPOhsgEp/Xd/xS3qxERabO75n3F+pzCDtn30N4J/D5rWJtek5eXR1ZWFtOmTeORRx4hOzubyy67jMGDB7Njxw5eeuklnnrqKVJTU3nwwQc5//zzyc3NJTQ0tEM+Q6BSeH+fiBgY/iNYPRfO+rMz77eIiLTb1q1bKS8vZ8aMGYwfP57x48eTkpJCQkICd911F5mZmVxwwQUYY/j73//OggULqKysJDo62u3S/YpOWDuc7cvhH1Mg6zEYfZl7dYiIBIHKykqmTp3KggULOOuss5gyZQoXXXQRycnJvPnmm1xxxRWkpKRw7rnnkpWVxUknnVR/ApdOWGugPu/DScuEbkc5h85FROSIREREMH/+fFasWMHgwYOZM2cOaWlpPPLII0ydOpUdO3Zw2223kZOTw7nnnstxxx1Hbm6u22X7HYX34RgDo6bD9iWQt9HtakREAtrSpUu57bbbGDZsGPfddx/Lli3jpptu4i9/+Qt33303y5Yt47LLLuOFF15g69atfPvtt7z99ttul+131OfdGsdcBB/+wRlxbcrv3a5GRCRg1dbWct999xEaGkpWVhZ79uzho48+YtSoUXz22We89NJL3H///fTs2ZOPPvqI8vJyjjnmGLfL9jsK79ZI6AUDp8Dql+HU30KIznoUEWmP8ePH89hjj/Hoo4/ywAMPkJSUxBlnnMGf//xnSkpKuPHGG7nssssoLy9n8ODBvPjii4wZM8btsv2OTlhrra9eh1cuh0v+4wS5iIhIB9AJa940+CyIToKVmqxERETcpfBurbBIGPFj+Oa/UFbgdjUiItKJBUSft89nFTuUUT+BZbNg3X9gzFW+fW8REQk6mlXMF6yFv5/gtMJnLHC7GhERCULq8/Y2Y+DYn0DOl7Dna7erERGRTkrh3VYjLgQTCmtfcbsSERHppBTebRWXAn1PhK/ecA6ji4iI+JjCuz2GnQf5myD3K7crERGRTkjh3R5DssCEwPo33K5EREQ6IYV3e8SlQJ8TdOhcRMRlW7duxRjDnDlz3C7FpxTe7TXsPNi3QWedi4j4UHFxMcYYFi5cCEBqaiobNmzgRz/6kbuF+ZjCu72OPhcwOnQuIuKi8PBwBg4cSEJCgtul+JTCu73iujccOhcRkVYxxrBgwQIuv/xyUlJSSE1N5eGHH67fXlNTw29+8xvS09OJjIwkLS2NX//611RXVwMQHx8PwKRJk7j88svr9/nGG2/wxBNPkJCQQHl5ef3+8vPzCQ8P5/nnnwcgJyeHadOm0aNHD5KTk7nwwgvZsWOHjz6992iEtSOxbDa882u4fil0H+J2NSIiB5t/K+xe2zH77jkCzvxTm15ijGHEiBFcddVVjBkzhkcffZS5c+eydu1ahg8fzpNPPsmNN97Ik08+yciRI1mxYgUzZ87k6aef5qc//SmrV69m1KhRPP/880yZMoUePXpgjOH1119n3LhxpKam8uabb3LOOecA8PTTT/OrX/2K3bt3ExoayrBhwxg7diwzZ86kvLycu+66i507d7JmzRqioqI64ltqM42w1tGOzkKHzkVE2uaUU05h5syZ9XN7A2zcuBGA/v37869//Ysrr7ySzMxMrrnmGvr27cvmzZsBGDBgAOD0dffo0aPJfnv27MmkSZN47bXX6te9/PLLTJs2jZiYGF5++WUA/vWvfzFu3DgmTpzIm2++SXZ2Nl9++WWHf25vCoiJSfxWfE/IGA/r34SJt7pdjYjIwdrYMvaF8ePH1y937969ybbTTz+dTZs2MXfuXFavXs2nn37Khg0bWr3viy66iFtuuYWqqiry8/NZsGABf/jDHwBYu3YtmzdvJjY2tslrKisr2bBhAxMmTDiCT+VbARHefjOrWEuGnQfzfwN7v4OUo9yuRkTE78XExBxy21//+lduu+02zj77bKZMmcKFF15Y37fdGj/60Y+47rrrWLRoEd9++y0DBgyo/7FQXV1NZmYmzz333EGvS0lJafPn8Ib2zioWEOE9a9Yst0s4tKOznPBe/wac8hu3qxERCWj33HMP999/P9deey3gnMC2Z8+eVr8+OTmZM844g9dee41169ZxxRVX1PchDx06lOeff5709PT61ve2bdu44447uPvuu+natav3P9BhtNQQnT179mFfpz7vI5XQG9LHOYfORUTkiMTGxvLKK6/w+eef8+qrrzJ16lT27NnDunXr2LdvHyEhTmytW7eOvLy8Fvdx0UUX8fLLL/P5559z6aWX1q+fPn06kZGRTJs2jc8//5z58+dz0UUXsWbNGlJTU33y+bxF4e0NQ6dC7jrI2+h2JSIiAW3OnDns27ePyZMnc/fdd/PDH/6Q559/nkWLFvH2228TExPDtddey6233sqdd97Z4j7OPfdcysrKmDJlCmlpafXr4+Li+OSTTwgJCSErK4tLL72Ufv368d///rf+R0Gg0KVi3nBgJzw8FE79LZz8f25XIyIiAUyXivlKl1RIG6tD5yIi4hMKb28ZOtUZCGHfJrcrERGRIKfw9pahU517tb5FRKSDKby9JTEdUjM12pqIiHQ4hbc3DZ0Ku1ZD/ha3KxERkSCm8PYmHToXEREfUHh7U1If6H2sDp2LiEiHUnh729DzIGclFGx1uxIREQlSCm9vqz90/pa7dYiISNAKiIlJ/HpWseaS+0GvkU6/9wkz3a5GRET8WHtnFdPwqB1h4f2w8D74v00Q6/tZakREJHBpeFS3DDgVsLB5gduViIhIEFJ4d4TU4yAqETYpvEVExPsU3h0hJBT6T4RNH0KgHfIXERG/p/DuKANOhaJdsOdrtysREZEgo/DuKANOde43feRuHSIiEnQU3h0lMR26HeUcOhcREfEihXdHGjAZsj+HqjK3KxERkSCi8O5IAydDdbkT4CIiIl6i8O5IfSZAaIT6vUVExKt8Gt7GmBBjzBvN1sUaY943xqw2xsw2dUPLBIOIWMgYr/AWERGv8ll4G2P6AGuBIc02nQ8sAEYBFjjBVzX5xMDJsGc9FOa4XYmIiAQJX7a8twEjgexm6zcAL1lnAPPdPqzHN3TJmIiIeJnPZhXzhHN13WQjjdYvNo5LcFrddzd/bWZmZqvfZ8aMGS3OQuaaHsMhtrsT3sde4nY1IiLiglmzZjFr1iyv7c/ns4oZY/5nrf1Bo8cGuB/oAdxgrS1qtC0wZxVr7rVrYMN78H8bnaFTRUREDiFQZhX7oef+8sbBHVQGToayfNi12u1KREQkCLgW3saYO4wxQ4GxwGRggTFmoTHmRLdq6jD9Jzn3Gm1NRES8wOeHzdsiaA6bAzx5EkTEwc/mu12JiIj4sUA5bN45DJwMO5ZBeaHblYiISIBTePvKgMlQWw1bP3G7EhERCXAKb19JPx7CY2Gj+r1FROTIKLx9JSwC+p2kwVpEROSIKbx9acBkKNgC+ZvdrkRERAKYwtuXNFSqiIh4gcLbl7oOgMQM2KjwFhGR9lN4+5IxTut7y8dQU+V2NSIiEqAU3r42YDJUFsGO5W5XIiIiAcpns4odiZZmCcvKyiIrK8uFao5Qv5PBhDqXjPWZ4HY1IiLionnz5jFv3rw2v07Do7rhH6c7h81nLHC7EhER8TMaHtVfDZgMOSuhZJ/blYiISABSeLthwKmAhS2L3K5EREQCkMLbDb1HQXgMbFvsdiUiIhKAFN5uCA2HtDGQrfAWEZG2U3i7pc8EyF0H5QfcrkRERAKMwtstGeMBC9uXuV2JiIgEGIW3W9LGQEgYZH/udiUiIhJgFN5uiYiBXqN00pqIiLSZwttNGeNg5xdQVe52JSIiEkAU3m7qMwFqKiHnS7crERGRAKLwdlPGeOde/d4iItIGCm83xSRDyhD1e4uISJtoVjG3ZYyHdf+B2hoICXW7GhER8SHNKhao1vwbXrsarvkEeh3jdjUiIuIyzSoWCOr6vXXoXEREWknh7bbEdOiSrpPWRESk1RTe/iBjvNPyDubuARER8RqFtz/oMx6KcyF/s9uViIhIAFB4+4OMCc69+r1FRKQVFN7+oNtREJ2k+b1FRKRVFN7+ICTE0++tk9ZEROTwFN7+ImO80+ddlOt2JSIi4ucU3v6ij/q9RUSkdRTe/qLXSAiPUXiLiMhhKbz9RWg4pGVqsBYRETkshbc/yZgAueugvNDtSkRExI9pVjF/0mc82FrYvgwGTXG7GhER6WCaVSwYVJbAnzLghF/A5DvcrkZERFygWcUCTUSsc+KaBmsREZHvofD2NxnjYecXUF3hdiUiIuKnFN7+JmM81FRAzkq3KxERET+l8PY3GeOde10yJiIih6Dw9jexXaHbYA3WIiIih6Tw9kd9xsO2pVBb43YlIiLihxTe/ihjAlQcgD3r3a5ERET8kMLbH/Wp6/fWoXMRETmYwtsfJWZAQprm9xYRkRYpvP1Vn/GwbQl0ltHlRESk1RTe/ipjHBTtgv3b3K5ERET8jCYm8Vfp45z7bUsgqY+7tYiISIfQxCTBprYG7u8Hw38EWY+4XY2IiPiIJiYJZCGhkD7WaXmLiIg0ovD2ZxnjYO/XUJrvdiUiIuJHFN7+LMPT771jubt1iIiIX1F4+7Pex0FIuMY5FxGRJhTe/iwiBnqPUr+3iIg0ofD2dxnjYOeXUF3hdiUiIuInFN7+Ln0c1FRAziq3KxERET+h8PZ3dSetqd9bREQ8FN7+LrYbdB2kfm8REann0/A2xoQYY95o7XrxyBgH25dAba3blYiIiB/wWXgbY/oAa4EhrVkvjWSMh7ICyPvO7UpERMQP+LLlvQ0YCWS3cr3Uqev33q5D5yIi4sNZxawzu0h13WQjh1vfWGZmZqvfZ8aMGS3OQhbQkvtDbIrT7z36crerERGRNpo1axazZs3y2v58PquYMeZ/1toftGZ9p55VrLm5l8DutfCL1W5XIiIiHUizigWTjPFQsBUKd7ldiYiIuMy18DbG3GGMGerW+wecdPV7i4iIw+fhXXdo3Fr7B2vt+ubr5RB6HQNh0bBtqduViIiIy3TYPFCEhkNapkZaExERhXdAyRgPu9dARZHblYiIiIsU3oEk43iwtbBjhduViIiIixTegSRtLJgQ2K5+bxGRzkzhHUiiEqDHMPV7i4h0cgrvQJMxHrYvh5pqtysRERGXKLwDTfrxUFUCuWvdrkRERFyi8A40GeOde83vLSLSaSm8A02XVOiSofAWEenEfDar2JFoaZawrKwssrKyXKjGD2SMgy0fg7VgDjluvYiI+Ll58+Yxb968Nr/O57OKtYVmFTuE5U/Df2+GmasguZ/b1YiIiBdpVrFgpX5vEZFOTeEdiFKOhsguut5bRKSTUngHopAQZ6hUjbQmItIpKbwDVcY42PsNlOa7XYmIiPiYwjtQZUxw7rd+4m4dIiLicwrvQJU2xun33vC+25WIiIiPKbwDVWgY9D8FNn7oXO8tIiKdhsI7kA06DYpyYM96tysREREfUngHsoFTnPuNH7hbh4iI+JTCO5Al9Ibuw9TvLSLSySi8A92gKc5IaxVFblciIiI+ovAOdAOnQG2VM1GJiIh0CppVLNClj4OIOOfQ+ZCz3a5GRETaQLOKdWYv/wR2rYGb1miKUBGRAKdZxTqLgZPhwDbI+87tSkRExAcU3sFAl4yJiHQqCu9gkJgB3QbrkjERkU5C4R0sBp0G2Z9BZYnblYiISAdTeAeLgVOgphK2fup2JSIi0sE6XXiXV9VQVF7ldhne12cChMfo0LmISCfQqcL7QFkVI+58l+eXbHO7FO8Li4R+J+ukNRGRTqBThXeX6HDSk2NYvjXf7VI6xsApULAF9m1yuxIREelAnSq8AY7vl8zyrfnU1AbhwC+6ZExEpFPodOE9pm8yReXVfLs7CCfySO4HXQeq31tEJMh1yvAGgvvQ+dZPoarM7UpERKSDdLrwTkuKpneXKJZtCdbwPg2qy5xrvkVEJCh1ulnFjDGM6ZfM55v2Ya2tHwA+aPQ9AcKiYMMHDX3gIiLilzSrWBu8sDSb219fx4JfT6Rft1iv7tsvPH8+FGTDjSvcrkRERNpIs4odwti6fu9gPnS+bwMUbHW7EhER6QCdMrwHdo8jKSacZcF80hrokjERkSDVKcPbGMOYvsnBe9Ja1wGQ1Nfp9xYRkaDTKcMbYGy/ZLbll7L7QLnbpXifMc6h8y0fQ3WF29WIiIiXderwBoL70HlVCWxb7HYlIiLiZZ02vIf2SiA2IjR4T1rrdxKERmi0NRGRINRpwzssNITj+iQF70hrEbEwYDKsmatD5yIiQabThjc4l4x9s7uI/aWVbpfSMY6fASV7Yd1rblciIiJe1KnDe4yn33vF1gKXK+kg/SdByhBY+nfw48F4RESkbTp1eI9KTyQiNCR4T1ozBo6/Bnathm1L3K5GRES8pFOHd1R4KMekdQne670BjpkGUYlO61tERIJCpw5vcA6dr9t5gNLKardL6RgRsXDcT+Hrt2H/drerERERL+h0s4o1N7ZfMn9fuImV2/ZzwsBuR7w/vzT2alj8BCx/Gk67y+1qRETEQ7OKtVNheRUj73qPmacO4penHdVh7+O6uZc6I6796muIiHG7GhEROQTNKtYKCVHhDO2VENz93gDjroPy/c513yIiEtA6fXgDjOmbzMrtBVRW17pdSsfJGA89j4GlT+myMRGRAKfwxun3Lq+qZV3OAbdL6TjGwPHXwt6vYfNCt6sREZEjoPDGaXkDwX/ofPj5ENMNlj7pdiUiInIEFN5ASnwk/bvFBu8kJXXCoyDzZ/Ddu7Bvk9vViIhIOym8Pcb2S2b51nxqa4O8P3jMlRASBstmuV2JiIi0k8LbY0zfZArLq/k2t8jtUjpWfE8Y9kNY+QKUF7pdjYiItIPC22OsZ5KSoJ0itLHjr4XKIlj1otuViIhIOyi8PdKSounVJYqlwd7vDZA2GtLGwLKnoDaIL48TEQlSPg1vY0yIMeaNZuuijDGvGmM+N8Zc4ct6mtXBmL7JLN+S36EjuvmN46+F/M2w4T23KxERkTbyWXgbY/oAa4EhzTb9EPgUOAm43Bjj2njrY/sls6eogux9pW6V4DtDp0J8b802JiISgHzZ8t4GjASym60fDayw1tYA24EMH9bURF2/d9DO791YaLhz5vnmhbDxQ7erERGRNvBZK9c6x6Kr6yYbaaQLkONZzgGSmr82MzOz1e8zY8aMFmcha42BKXEkxoSzfEs+F2amt2sfAWXc9bD2VXhtBlz7KST0crsiEZGgNGvWLGbN8t4luj6fVcwY8z9r7Q8aPX4QeMta+7Ex5iXgt9baTZ5tHT6rWHNXP7eCNTv289r1J5CaGO2z93XN3m9h1iToNRIumwehATFLrIhI0AqUWcVWAKONMaFAKs7hdddcPqEvxeXV/OCRj3lrdc7hXxDoUgbDOQ/Dts9hwR/drkZERFrBtfA2xtxhjBkKvAGcgHPS2j+ttVVu1QRwwsBuvPOLkxjUPY6ZL63kppdXcqDM1ZI63shpcNxl8OnD8J3OPhcR8Xc+P2zeFm4cNq9TXVPL3xZu4tEPN9AzIYq/XDiScf27+rwOn6kqg6dPg8IdTv93lzS3KxIR6ZRac9hc4X0YK7cV8Mu5q8jOL+Wakwfwq9OOIiLMH3obOkDeRpg1EbofDVe845yRLiIiPqXw9pKSimr++N/1vLRsO8N6J/DoRaMY2D3e1Zo6zLrX4NUrYPwNcMY9blcjItLpKLy97L2vdnPra2spqajmH5eN4cRB3dwuqWP892ZY/jRc9CIMOdvtakREOhWFdwfYU1TO9NlLKa2o5t1fnkx8VBAeWq6ugH+cDgVb4JqPIamv2xWJiHQagXKpWEDpHh/FAxccw67Cch7437dul9MxwiLhx3PAAq9cAdWVblckIiKNKLzb4biMJH52Qj/+tSSbpZv3uV1Ox0juB+f9FXK+hP/8DCqK3a5IREQ8FN7tdPPpR5GRHMOtr62lvKrG7XI6xtFZcMa98M1/YfapsPc7tysSEREU3u0WExHGn340gi15JTz8QRCH2vifw6VvQOk+mD0J1r/pdkUiIp2ewvsITBjYjYvHpjP7482s2bHf7XI6Tv9T4JpFkDIE/v1TeO93UFPtdlUiIp1WQJxtfvXVVx+0LSsri6ysLJ/X1FxheRWnPbSIpJgI3rrhxOAdwAWcs9Dfvc25jKzvSXDBPyGuu9tViYgErHnz5jFv3rwm62bPng3oUrEO98H6XK56bgW/Ou0oZk4e5HY5HW/VS/D2TRCdDBc+B+lj3K5IRCRo6FIxH5kytAdZI3vz+Ecb+C63yO1yOt6oi+HK9yEsAp45E5Y8CbVBetKeiIgfUsvbS/YVVzDloUX06RrLf66bQGjIIX8wBY+yAnjtGtjwLqQcDaf+1hmRzXSCzy4i0kHU8vahrnGR3HnuMFZt388zn21xuxzfiE6C6XPhx89CbTXM/Qk8PQW2fOx2ZSIiQU0tby+y1nLVsyv4bFMe7910ChldY9wuyXdqqmH1i7DwT1C4E/pPgsl3QOpxblcmIhJQNLa5C3YdKOP0hz6mf0osT146ml5dot0uybeqyp2z0T/5C5Tlw9CpMOm3kHKU25WJiAQEhbdL/rduN7/69yrCQ0P4049GcOaIXm6X5HvlhbD4r7D4CagqdUZry7wS+p2sPnERke+h8HbRlrwSbnp5Jat3HODCzDR+nzWM2Mgwt8vyvZI8+OxRWPkv5wS3roMg82fOGevRSW5XJyLidxTeLquqqeXRDzbw14UbyUiO4ZFpozg2o5MGVlUZfPUGrPgH7FgOYVEw/HynNZ56nFrjIiIeCm8/sWxLPr+cu4rdheXcNHkQ108a2DkuJTuUXWtgxT9hzb+hqgR6jYTRl8PQ8yAm2e3qRERcpfD2IwfKqrjjzXW8uSqHzD5JPDxtFOnJnehs9JaUF8Laf8Pyf8KeryAkDPpPdFrkQ86GqC5uVygi4nMKbz/0xsqd/O6NdQBcP2kgl0/oS3REqMtVucxa2L0G1r0GX70G+7dBaAQMnALDfgSDz4TIOLerFBHxCYW3n9qeX8rv3lzHwm/3khIfyQ2TBnLR2HQiwzp5iIMT5Du/dEJ83WtQlOP0jw86HYac4wR6bFe3qxQR6TBBE97+PKvYkVi+NZ8H3/2WpVvySU2MZubkgZx/XBphoRr4DoDaWti+1Any9W9CcS5gIC0TBp0Bg05z+st1spuIBCjNKhagrLV8ujGPB9/9ltU7DtCvWyy/PO0ozhnRi5DOfFJbc7W1sGsVbHgPvnsXcr501sf1dEJ80OlOf3lUgptViogcsaBpeftzjd5ireX99bn85b3v+Da3iCE945k5eRCnD+2hlnhLivfAxg+cIN/0EVQUggl1WuJ9JkDfEyFjPEQnul2piEibKLwDUG2tZd6aHB75YANb8kromRDFxWMzuHhsOt0Totwuzz/VVMG2JbBlEWz9DHaugJpKwEDP4dDnROh7AmRMUH+5iPg9hXcAq6m1fPTNHv61JJuPv9tLWIjhjOE9+em4Poztl1z/H1daUFUGO1ZA9mew9VNnUJjqcmdblwzoMcwJ9R7DoMcISO4HITpZUET8g8I7SGzJK+GFJdn8e8V2CsurGdwjnkvG9+GHx6YS1xmHXG2r6grIWQnbFsPutZD7FeRtAFvjbA+Pge5HO2He7SiI7wUJqZDQy1kOi3S3fhHpVBTeQaassoZ5q3N4bslW1u0sJDYilDOG92TqqFROGNBVfeNtUVUOe7+B3HVOmOeug93rnJnQmovpBgm9G27xvZs+TugNkfG+/wwiEpQU3kHKWsuq7ft5adk25q/bTVF5NV1jIzj7mF5MHdWb4zKSdFi9Pax1TnwrzGm4Fe1qtOy5L9138Gsj4j1B3gsSMyB5AHQd4Nwn94PwTjY1rIi0m8K7E6iormHht3t5a1UOH3ydS0V1LWlJ0Zw7sjdTR6UyuKdahF5XVd4Q6kW7oHAnFNbd50DBVijNa/qahDTo2r9pqHcdCEl9ISzCjU8hIn5K4d3JFJVX8d5Xuby1OodPN+ZRU2sZkBLLlKE9OH1oD0alJ3XuCVF8qfwA7NsE+Zs995sa7ssKGp5nQqBLuhPkjUM9uZ/Tgg8Nd+8ziIgrFN6dWF5xBe+s3cV7X+WyZPM+qmstXWMjOHVId6YM7cFJg7oRE6GT3VxRmt8Q6vs2eoJ9I+zbDJVFDc8zodAlzQnypH4H32u8d5GgpPAWAArLq1j07V7eX5/Lgm/3UFReTWRYCCcO7Mbko50g7/QznPkDa53BZ/ZthIItkL+l6X3jFjs4J9Il9XEOvSf1hcRGywmpEKofZyKBSOEtB6mqqWX5lnze/zqX99fnsqOgDIC+XWM4cVA3ThyYwvgBXekSrcO1fqdsf9MwL8iG/dlOH/v+7Q2XvoEzvWpCqnPoPbGP577RLaG3rm0X8VMKb/le1lo27inmkw15fLoxjyWb91FaWUOIgZHpiZw0sBsnDkphVHoiEWG6DM2v1VQ7J8wVbG0U6NsabkW7mj6/Sbh7bl3Sm4a7+ttFXBE04R2ss4r5m8rqWlZuK+DTjXl8siGPNTv2U2shKjyEY9OTGNsvmbH9kjk2I1H95YGmqtwJ9/3ZDYFekA0Htjut9qJdQKP/F5gQJ9y7pDW6pXtunseaBEbkiGlWMfG6A6VVLN6cx9It+Szbks/XuwqptRAWYhie2sUJ877JZPZNIjFGlzsFtOoKOLDDE+bbnEDfv61hXeFOqK1u+prILtDFE/AJqZ7l9IblhFSNTifSDkHT8vbnGjuTwvIqvsguYLknzNfsOEBlTS0A/VNiGZWeyLEZSRybnsiQnvEa8S2Y1NY4J9PVhfmBHQ23Qs99S4PXxHZvCPLGIZ+Q5tzH9dSJdSLNKLylQ5VX1bBq+36+yC5g5bb9rNpeQF5xJeAcaj8mNZFjMxIZlZ7IMemJ9O4SpZHfgllVmTNIzYHtcGCn01pvsryz6aVw4Byej+vZMDpd46Fn43s1LGuEOulEFN7iU9ZadhSUsXL7flZucwJ9fU5hfes8OTaCEaldGJHaheGpXRiR1kWB3tmUH2gU5p4We+NR6op2OUPUNheV6Alzz2QxzZfje0FsilrxEhQU3uK6iuoa1ucUsnbnAdbuOMDanQfYsKeYmlrnv2lybIQT5KkJHN3LufXtGquR4DqziiJPkDceY353oyFpd0NxbtNL48Bpxcd2h/ienkBvfN8T4no497EpukxO/JrCW/xSeVUNX+8qZN1OJ8zX7Gga6FHhIQzuEV8f5kf3SmBIr3gSonTpknjU1kDJ3obx5Yt2NwR84+XmY8yDJ+RTGsK88X3dLd5zr8P14gKFtwSMiuoaNuQW8/WuQr7eVcQ3uwv5elchBaVV9c/p3SWKQT3iGdwznkHd4ziqRzyDesTpsjU5tOpKp5VetBuKdze02pvfl+wFW3vw6yMTIK670y8f173RrYfTyq9f7qbr4sVrFN4S0Ky15BZW8PWuQtbvKmRDbhHf5hazaW8xldUN/6NNT47mqO7xDOwRx4AU5zYwJY4uMfqfqbRSbQ2U5DlBXrzHCfriXCjKbbZu78En3dWJ6eoJ9JSGYI9N8dw3Wh+bopnk5HspvCUoVdfUsi2/lO9yiz2BXsSG3GI25xVTVdPwt9ItLpIBKbEM6O4Eev+UWPp3iyU1MVqXsUn7VZZCyR5PoNcF+56GFnzxHs/2vVBV0vI+ohIPDvXmoR/bzbmPiPXpxxP3KbylU6muqWVHQRmb9jqt8017Sti4t5iNe4o5UNZw+D081JCeHEO/rrH07RZLP8+tb7dYeiVEEaKT5cRbKks8Yb63UcDnNYR/yV7Ptr1QcaDlfYTHNAR541A/6HF3p/WvM+4DnsJbBOfvJ7+kkk17S9iaV8LmPOd+674StuSVUNHoEHxEWAjpSdH06RpLRnIMfbo6t4zkWNKTo4kM01nK0kGqyhvCvCSv0fLehsCvC/rSvINHvKsTndwo0Ls5s8/Vh3yz4I9OAl2q6XcU3iKHUVtr2V1YXh/q2ftK2JZfSva+Urbll1Ja2XA5kjHQKyGKtOQY0pNiSE+OJi0phvSkaNKTY+iREKVL3MQ3rIXy/YcI+br7RtvK8lveT0i4p1/+MP30cd2dHwUh6m7yBYW3yBGw1pJXXMm2/BKy9zmBvj2/lO0FpWzPLyO3qJzGf5rhoYbURCfQ05KiSU2MJtVzn5YcQ4/4SPW1iztqqp0Ar++Tb3bovnE/fcleqK06eB8mxGnFNwn4lIOX6246Ka/dgia8NauY+KOK6hp2FpSxvaCMHZ5A315Qyo6CMnYWlJFXXNHk+aEhhp4JUaQmRdO7SxS9E6Pplegs9+rihHxCdJhGnBN3WQtlBU1DvSSvWcA3OoRfXdbyfqISD9FP3+gHQIznUH5UYqdt1WtWMRE/U15Vw879TpA3v885UMbuA+VU1zb9246JCKWXJ8x7domiZ0JU0/suUSTHROikOvEP1kJlcUOQl+xtFvZ7W3f43oQ26p/vdog++0b99pEJQd1XHzQtb3+uUaS9amotecUV5OwvI2d/ObsOOPc5+8vYXVjO7gPl7Ckqp1m+ExEaQveESHokRNEjIZLu8VH1y/XrEqKIj1QrXvxMTRWU5jcK+X0NwV6a17SvvnRfy+PcA4RGHBz0Md0gtmuj5RTncUw3iIwPqLBXeIsEuOqaWvKKKz1h7rTWdxdWsPtAGXuKKsgtLGdPYQVFFQefeRwVHkKPhCi6xzsBnxIf6YR+fBTdPaHfLS6CJLXkxV9VlTshXteCL218gt6+RqHveXyo6+rrw75rQ+gf9Lir3xzGV3iLdBIlFdX1YV5321NYwZ6iCvYUlTv3hRUUtxDyoSGGrrERpMRHkhIfSbe4hvtucRF0jY2ka1wEXWMjSIqNIFwn3Ym/qixtaMGX7mvUis9zwr10X9Pth2rZm1CISW4U6C2EfuM+++hkr15fr/AWkSZKK6ubhHpeUQV7iyvYW1RBXnGl59553Lw/vk5iTDjJsRF0i40kOTaC5LgIkmOcYK8L+OSYhvXREbo2XvxUdUVDyNcHfF6jx3lNt5cVHHpf016Ao8/xSlkKbxFpl9pay4GyKvaVVLCvuJJ9JZ5bsfM4v6SSvOIK9pVUsr/UeXyIrCcyLITk2AgSYyJIigknKSaCpFjnvvG6LnXbYsKJjwrXNfPif+ouuWsS+J6AH34BdBvolbdReIuIT9TWWgrLq8gvqaSgtJL8kioKPIFfF+4FpVXOcmkl+z3Lhwp8Y6BLdDiJ0eEkxkSQGNOw3CU63HkcE05itBP6dc9NiA7XYX0JeApvEfFbdYFfF+r7S6soaBTs+8sath0oq6pfX1h+iGFBPeIiw+jiCfLEaCfYu0SH14d8QlQYCZ51CY22J0SFExGm4Bf3KbxFJOjU1FoKy6rYX9YQ+gfKquoDvm75QFllk/WF5VWUV7UwZ3cj0eGhJESHkRDlBHtd0DuPw4iPcpbjPevjozzP9TyODAvR5XlyxBTeIiKNlFfVUFheRWFZNQfKqigsc0K9Ybm6fl1hWfVB22oOdZzfIzzUeALeCfr4qLD6gG/+OM6zXL8+0lmOCtcPgM7Or8LbGBMFPA/0BmZba5/xrE8AXvGsn2WtfbzRaxTeIuIXrLWUVdVQWFZNUbkn4D1hX1TuBH1RuWeb5znO44ZtLV2q11xYiCE+KswJ98iGwK8L+bjIpj8M6rY56xuWdcJf4PK38L4Y6AE8DnwETLbWVhtjbgaKgDnAh8APrLUlntcovEUkaNTUWoornGB37htCvtCzXFzesL64om59w/OKKw5/BAAgNiLU07pvHOwN4d90XaMjAZHOD4e4yDBiI8I0gI8LWhPevpy1fTTwhrW2xhizHcgANgODgEettZXGmFXAQGC1D+sSEfGJ0BBTf4Jce9UdASj2BH79j4FGrfziiur6x84PAGfd7gPl9etacxQAnBMA4yLDiI0MJS4q3An3RgF/0PIhtul8AO/yZXh3AXI8yzlAkmf5W+BMY0wOMBF4sfkLMzMzW/0mM2bMYMaMGUdUqIiIvzLGEBMRRkxEGN0T2r+fmlpLSaUT8nU/AIrqlxvW1/0QKK6opqiimuLyKnILyympe1xRTWsOjoaFGGIjw+pb/LGNwz2iIeQP2tboR0HdtkC8HHDWrFnMmjXLa/vz5WHzB4G3rLUfG2NeAn5rrd1kjIkDngZ6ARa40lq7yfMaHTYXEfFjjY8EFFVUU+IJ+8JyZ7mk0vkxUNLsh0Cx57lFjdaVVta06j0jw0LquwAa/yBoCPxw4iJDPY+bnhBYfz5BVBiRYf45+p+/HTZfAYw2xnwGpALbPOsHAw8Aq4D/AFt8WJOIiByBJkcCjnBfjY8GlFQ0+gHQKOTrfwRUNPwoKKqoZteB8obugopqKqu//7JAgIiwEOIjw5peFtjo0sC6Swa7NBozoG5woPgod88HcONs81TgKZw+71dxDqG/CEQDf7HWvt3oNWp5i4hIm1VU11BSUXduQNMTBOuWCxtfEVB/iWDDVQQV3/MDIMTQMBBQTAT/78whjOvf1Su1+1XL21pbDlxwiM1n+aoOEREJfpFhoUSGhZIcG9HufTSMC1DVaJQ/Z4CgA55RAOseR/p4dD4N0iIiIuJHWtPyDrxT9kRERDq5Thne3jxdX/R9dgR9p96l79P79J16V1u/T4W3HDF9n96n79S79H16n75T71J4i4iIBDmFt4iISIBReIuIiAQYhbeIiEiACbjwnjdvntslAN6p40j34Q81eIu/fBZ/2Yc3+Mtn8Ye/c2/wh+/Cn/bhDf7yWfxlH20REOFdN1PYjBkz+M1vfsOMGTNc/+Pzh//Y/lCDt/jLZ/GXfXiDv3wWf/g79wZ/+C78aR/e4C+fxc19zJs3jxkzZpCdnd2mWTF9OTFJuzU+hX7GjBm6REFERIJCVlYWWVlZfPnll/XZNnv27MO+LiBa3iIiItJA4S0iIhJgAmJiEhERkc5GE5OIiIgEEb9ueYuIiMjB1PIWEREJMApvERGRAKPwFhERCTAKbxERkQDTqcLbGBNljHnVGPO5MeYKt+sJZMaYEGPMG57lscaYxcaYRcaYDJdLC0jGmFhjzPvGmNXGmNnGmAHGmI89f6vHul1foDHGZBhjlhtj1hpjrjTGnOv5G51vjEl0u75AZoy50Bhzq/7dHxljzFHGmFXGmIXGmDfb+n12qrPNjTEXAz2Ax4GPgMnW2mp3qwo8xpg+wDtAqLV2iDHmv8BlwEDgImvtTW7WF4iMMT8F0oD7gKeAAcDNQC7wN2vtD10sL+AYY/4ILAXmA4uBUuAM4Dygp7X2EdeKC2DGmHBgJfA8cBL6d99uxpizgFRr7WzP4zb9f7RTtbyB0cAKa20NsB3Qr8X22QaMBLI9j7taa/Nw/lGPcquoALcBeMk6v6Z3A5OAtdbaXTg/OKVt3gIWANFAKFBlrS0HVgDHuVlYgJsBvOtZ1r/7I9MPON8Y85kx5lLa+H12tvDuAuR4lnOAJBdrCVjWUQ3UHbYp8ayvAHQkox2stYuBrcaYS4ATgAWeH5ngtBqlDay1y4BeOD/S/4P+3R8xY0w8zo/Ktz2r9O/+yGwC7gCygJto4/cZELOKedEBnEOTm4F0YL+r1QSPeABjTDQQ4XItAckYY4D7cVrZ5wHvGWPCPD+SEtysLRAZY5KttRuMMT2BT4FCzyb9u2+/XwEPA+Gex/p3fwSstf+rWzbGfA5c4llu1ffZ2cJ7BTDaGPMZkIpz+FeO3F5jTApOX82XbhcToOr6tC+31lpjzGpghDEmF9jpYl2BarYx5nbgO6ACiDfGROEcMv/C1coC12Cclncizg/KXvp3337GmLuB/+GcmzEcWNWW77OzhfcbOCdaXAg8Za2tcrecoHEnTh9jBfBTd0sJWGOBycACpxHO74BHcf6NXu9iXYHqYeB1nL/JfwJbcPrAC4DpLtYVsKy10wGMMROBccCH6N/9kfg7zvcXAfwN50dlq7/PTnW2uYiISDDobCesiYiIBDyFt4iISIBReIuIiAQYhbeIiEiAUXiLiIgEGIW3iIgEFWPMHGOMPcRtv49qWGiMeaSj9t/ZrvMWEZHOYRnwkxbW1/q6kI6g8BYRkWBUZq3d6HYRHUWHzUVEpFMxxvT1HEIfb4z50BhzwBjzpTHmnEbPCTHGzDTGfGWMKTbGrDTGTGu2n2HGmPc9r99hjLnXGBPa9CnmD8aYbcaYAmPMbGNMmGdDjDHmr8aYHGNMqTFmhTHmlNZ+BoW3iIh0Vq8CrwBTcCawmWeMyfRsuxG4D3gcOBmYC7xgjDkXwDMO+ec4c2ScBtwN/AJnApc6l+PMYncB8CBwFXCxZ9udnuVfeN5/G/CfZuF/SDpsLiIiwehkY0x5C+vnA7/0LP/TWvskgDFmBXAsTpheCvwGuL9uO/ClMWYQ8H84Y5BfD+QCMzzT9y4zxnQBMhq91xZgpnXGIV/mmfJ3gGfbUJzJsl71TEZ0Hc7ELxFA2eE+nMJbRESC0QpanuCjBKhr3X5Ut9IToB8C53hCuDfwcbPXLgKmepZHAJ97grtuHw80e/4y23QCkZJGy7OBZ4BvjDFvAfOAubaVE44ovEVEJBiVWmu/aWmDMaavZ7H5mec1fH8u1jbaHt7C6w+q4VAbrLVvGmPSgB8Dp+O05rcYY35grc09zH7V5y0iIp3WhGaPTwW+ttYeAHYBJzXbfgqw3rO8HhhnjKnPUWPML4wxb7bmjY0xvwPGWmuftdb+BOiLM2f6Od/7Qg+1vEVEJBhFG2MGHmJbXfbdYozZBawFLgEm4sxVDvBn4G5jTC6wHKd1fAVOSxmcObh/ATxpjJkNDAHuwDnBrTVOAC42xtwC7Mb54RAFrGnNixXeIiISjMYCGw6xbYzn/gbgGmAUsAk411q71LPtUcDinNyWAWwELrXWvg5grd1ujDkZeAin73w/8BRwTyvruw4n6J/FCe1vgenW2uWtebFpZd+4iIhIUPD0eW8BjrXWrnK3mvZRn7eIiEiAUXiLiIgEGB02FxERCTBqeYuIiAQYhbeIiEiAUXiLiIgEGIW3iIhIgFF4i4iIBBiFt4iISID5/0XVn5lc3iwZAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(figsize=(8, 6))\n", + "ax.plot(range(len(losses_train_ssl[1:])), losses_train_ssl[1:], label=\"ssl\")\n", + "ax.plot(\n", + " range(len(losses_train_native[1:])),\n", + " losses_train_native[1:],\n", + " label=\"native\",\n", + ")\n", + "ax.set_xlabel(\"Epochs\", fontsize=15)\n", + "ax.set_ylabel(\"Loss\", fontsize=15)\n", + "ax.legend(title=\"Training loss\", loc=\"best\", title_fontsize=20, fontsize=15);" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfMAAAF1CAYAAAD85gOOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAAsTAAALEwEAmpwYAABJ60lEQVR4nO3deXxU1f3/8dfJvgdCQlgCYd/LDhVwA3HrFxV3VBT8tkStWq3fWvu1rWJt61KXuvRnDbWFr7gjoGjdBRShIqsgLuw7hCRs2bfz++NOJgkESGAyd2byfj4e85g7596595Op9HPPuWcx1lpEREQkeIW5HYCIiIicGiVzERGRIKdkLiIiEuSUzEVERIKckrmIiEiQi3A7gMYyxqj7vYiINDvWWnOsfaqZi4iIBLmgTeZTpkxhypQp9OrVy7v99ttvY61t9GvIkCEn9b3arylTpugc+k2b7Bz6PX1/Dv2mgfebBsrf4eY53n77bW9OS01NZcqUKQ3KiUHXzF4tOzsbgKysLO+2iIhIMLvooou46KKLAFixYgXZ2dlMmzbthN87pZq5MaazMSbyVM4hIiIip6bBydwY08EYM8sYc6Hn80xgI7DLGDOoqQIUERGR42tMzfz/Ab2AbcaYbsCVwPnAv4GHmyA2ERERaYDGPDM/A7jCWvuNMea3wEfW2o+MMaXAnKYJ78Sqny24zRdxhNI5fCFQ/pZAOUcgxBBK5/CFQPlbAuUcgRBDKJ2jMYy1DRu2bYzJA6601n5qjFkIvGWtfcIYcxbwjrU2sSkDrRWHBWho3A0xdOhQli1b5rPziX5TX9Pv6Xv6TX1Pv6lvVf+exjjDy+1xxpk3pmb+BfAnY8xbwHDgSmNMBvALYN0pxCsiIiKnoDHJ/E7gVeC3wKPW2hxjzF+Ac4BLmyC248rKyjqqrHaXfhERkWAzb9485s2bB8DWrVvrzXX1aXAyt9ZuwqmR1/Yw8DtrbWlDz+MrGlsuIiKhxi/jzI0xPzLGdPBsnwE8Adxiqhv0RURExO8aM878ZmAVcJpnopjXgC7AvcDvmyQ6P2loM4Y0nH5T39Lv6Xv6TX1Pv6lvNeb3bExv9g3Ai8AfgCtwmti7AROB+6213RpwjjBgtrV2fK2yjsCbQAzwV2vtC8aYi4H/BQ4A11hrD9Q63re92UsOwcZPoe/4Ex4qIiLibw3pzd6YZvb2OEPQLHAu8J5ne61n34mCyQTW4Ew8U1sWzg3CIOBmT5P9/wCjgRnA5EbE2HirX4U3JsGetU16GRERkabSmGS+DTjTGBOL03t9rqd8AJDfwO8PALYeUf42MB+IBYqBeKDcWlsCLAMGNyLGxvvRFRAWCatfadLLiIgsWrQIYwxnnHHGcY/7/PPPMcZwzjnnNPoaW7ZswRjD1KlTvWWdOnXCGMOCBQtO+P3p06c3+NiGmjx58lEx+dOCBQswxjB9+nRXru8PjRma9izwFM7z8X3W2o+NMf8NPA1MP9GXPbX4iupm8lrlS40x3YGvgEeBZGCXZ/cuoGV95xs6dGiDA8/Kyjr2s4e4FOh5AXz9GoydCuFaN0ZEmsbIkSPp2LEjixYtYufOnbRvX3+j5uzZswG4+uqrfXLdG264gfz8/GNez5cmT57MjBkz6jwKPeecc0hISGD48CMHRDVf2dnZPh2V1Zihac8YYzYCvYHZnuJi4EHgrycbgDEmxVq73hjTBvgY56Yhw7O7A85z86P4dJahgdfBt/NgwydOYhcRaQJhYWFMmDCBRx99lFmzZnHHHXccdYy1ljlz5hAeHs6ll/pmCo8//OEPPjnPybr++uu5/vrrXY0h0By3knmEhgwYa9TQNGvtv621j1trN3s+v2KtfeQUx5lPM8b0AsoAA1QC4caYGJwm9uWncO6G6TYW4lJh1UtNfikRad6uvfZaAF577bV6969cuZKtW7dyzjnnkJaW5s/QJIg1dpz5OGPMQmNMjjEm3xizyNPzvNGMMfcZY/oAT+Is1LICeM1aWwg8hvMc/Xoa0IR/ysIjof/V8P17UNSQx/8iIienf//+9O7dmyVLlrBt27aj9lc3sV911VV1yg8fPszUqVPp3bs38fHxtGzZksGDB/PMM89QVVV13GtWP7Ourbi4mF//+tcMGDCA5ORkzj77bF5++eV6v19eXs6TTz7pPTYxMZG+ffvy4IMPUlJSAtQ8q58xYwbg1CYnT54MHPs5/NKlSxk3bhwZGRm0atWKMWPG8NJLR1eqOnXqxOTJk9m3bx8TJ06kVatWJCUlcfbZZ7No0aLj/u3H88MPP3DllVfSpUsXkpOTOf3003n66aeP+j2Liop4+OGH6devH/Hx8bRr147LLruMb775ps5x+fn53HPPPfTq1YvY2FgyMzOZPHlyvf87+5y1tkEv4CqgHPgHTpK9DpiGU6O+qqHnOdUXYJ2wfWz3GmvvT7L2P8/7/twiIrU8+OCDFrCPP/74Ufv69OljIyIibG5ubp3y8ePHW8D26NHDTpo0yZ5//vk2KSnJAvZPf/qT97jNmzdbwN5///3eskmTJtX5/82ioiI7cOBAC9hOnTrZa6+91p522mkW8L7Pnz/fe/ydd95pAZuRkWEnTpxoL774YpuammoBO2XKFGuttbm5ufbWW2+1PXv2tIC99dZb7f/93/9Za63917/+ddQ5X3nlFRsREWFjY2PthRdeaK+88kqblpZmAXvnnXfW+dszMzPtFVdcYfv06WM7depkr7vuOnvmmWdawMbHx9tNmzYd9/eeP3++Bey//vUvb9mCBQtsfHy8DQ8Pt2PGjLHXXHON7dChgwXs+PHjbVVVlffYyy67zAK2Xbt2dsKECfbCCy+0YWFhtnXr1jYnJ8daa21paan98Y9/bAHbpUsXO3HiRHv22WdbwPbu3dsWFxcfN8bjqZX3jp0bj7fT1k2iK4B76im/B1jZ0POc6qvJkrm11j43ytrnz2qac4uIeGzYsMEC9sc//nGd8u+++84C9sILL6xTvnfvXgvYMWPG2IqKCm95fn6+TUtLs4MGDfKWNSSZP/LIIxawEyZMsCUlJd7yv//9797EUZ14KysrbXx8vO3du7ctLCz0HltSUmL79u1rW7ZsWSfWI69l7dHJ/PDhwzY9Pd22bNnSrlq1yntcbm6uHTJkiDXG2NWrV3vLMzMzbVhYmL300kvrJMXf//73FrDPPvusPZ4jk3lFRYXt37+/jYyMtJ988on3uMLCQvuTn/zEAnbevHnWWmu3b99uAXvBBRfYyspK77GPPfaYBeyMGTOstdZ+/vnnFrA33XRTnWvfdtttFrALFy48bozH05Bk3phm9u7AB/WUf4AzeUzwG3gd7FoJe7UInIg0na5duzJ8+HC+/PJLtmzZ4i2fM2cOcHQTu7WW5557jieeeILw8HBvecuWLcnIyCA/v3GPB6dNm0Z8fDzPPPMM0dHR3vKsrKyjepyXlJTw2GOPkZ2dTVxcnLc8OjqaHj16sH///kZdG5zFRPbu3ctdd93FgAEDvOWtWrXi0UcfxVp71HzkYWFhPPvss8TExHjLqnv779u3r1HXX7p0KV9//TU33HADY8aM8ZbHxcXx1FNPYYzh+eefByAnJweA2NhYwsJqUubPfvYzlixZ4h0+WH1c7d8I4Pe//z1LliyhT58+jYqxsRozNG0rMAxnStfahuGMIferJlk17UdXwoe/g9Uvw3l/PIXoRESO79prr2Xp0qW88cYb3H333YDzvDwyMpLx48fXOTY9PZ2bb74ZcBL7tm3bWL58OR9++CErV64kMzOzwdctLy9n8+bNnHnmmaSmptbZZ4zhvPPOY+nSpd6yuLi4Otfes2cPK1asYP78+bzzzjsn86ezfv16AEaPHn3UvjPOOIOwsDA2bNhQp7xr1660a9euTll8fLzPr9+tWzcyMjK81+/Tpw/dunVjzpw5jB49mhtuuIHRo0eTmZnJaaed5v3e6aefTkpKCk899RTr169n4sSJnHnmmbRt25bWrVs3OLbaq6Y1RmOSeTbwhDGmNfC5p+wMnGb23zX6yqeoSVZNi0+F7ufD16/DOVMhvDE/j4hIw1111VXcddddvP7669x9991s376dr776inHjxtGiRYujjp89ezbPPPMMq1at4sCBA6SkpDB06FDatm3bqOvm5eVRWVl5zO8dmTABFi5cyMMPP8zy5cvZt28fCQkJDB06lE6dOnkTY2Ps2uVMJZKenn7UvsjISFJTU9mxY0ed8vqOPVnHuz5A27ZtWbfOaaGNiYlh4cKF/O53v+PNN9/0duLr3r07N954I3fddRfR0dG0bt2axYsX89vf/pZ3333Xe6MzcOBAbrrpJqZMmVKnVeVY6quU+nrVtGdwkvYtwALP6+fAVJxEHxoGXgsFe5352kVEmkjbtm0ZPXo0y5YtY9OmTcydOxc4uokd4IMPPuDyyy9n//793H///Xz33Xfk5ubywQcf0KNHj0ZdNy0tjaioKHbv3l3v/iPL165dy9ixY/nhhx/45S9/6b2ZmD9/PiNHjmzUtatV3zDs3bv3qH0VFRXk5eUddbPhy8U5j3f96vLa12/Xrh3//Oc/2bdvHwsWLOD++++nvLyce++9l5///Ofe43r27MmsWbPIy8vjgw8+4O6772b37t3ccsst/PnPf/ZZ/PVpcDL3PId/ylqbAbQAWlhr2wNfAIVNFJ//dT8P4lppzLmINLnqMeevv/46s2fPJioqiksuueSo49544w0APvzwQ+6880569uyJMQZrLVu3HjlD9vGFh4fTrVs3vvrqK/Ly8urss9byySef1CmbO3cuFRUVvPLKK/zv//4vAwYM8NYwG3vtat27dweod8rYRYsWUVlZ6T2mKRzv+hs3bmTbtm3eY5YvX87UqVNZt24dUVFRnHXWWUydOpXvv/+eNm3aeG/CPvnkE6ZOncquXbuIi4vjvPPO49FHH2XNmjVERUV5j2sqjRpnXs1ae8hae8jXwQSEiCjn2fn3/9aYcxFpUpdddhlRUVG88MILfPbZZ1x44YUkJSUddVxBQQEAubm53rKqqioefvhhtmzZQklJSaNWkrz55pspKCjgtttuo7S0Zs6vF154gcWLF5/w2tZaXnzxRRYuXAjgHWteW1lZ2TGvf9FFF5Gens4TTzzBmjVrvOW5ubn86le/whjDz372swb/PY01fPhw+vfvz4wZM+ok9KKiIm6//Xastd5+Wfv27eOBBx7g0UcfrXOOAwcOUFZW5q3lr1+/ngceeOCoR8D79++noqKi3scXPnW8ru4NeQE/BipP9TyNuF7TDU2rtmuVM+Z86bSmvY6INHvV48cB+/LLL9d7zKxZsyxgExMT7YUXXmgnTpxou3XrZlNTU+2IESMsYC+++GK7Z8+eBg1NKykpscOGDbOA7dy5s73uuuvsqFGjLGDHjRtXZxjZl19+acPDw21MTIwdPXq0nTRpku3fv7+Ni4uzY8aMsYAdPXq0/eabb6y11t5yyy0WsBMnTrSvv/66tbb+ceYvv/yyd5z5f/3Xf9mrrrrquOPMzzrrrKN+l/r+1vrUN858/vz5Ni4uzoaHh9uxY8faa6+91nbs2NEC9tJLL/WOMy8sLLQ9evTwDiX86U9/aseMGWPj4uIsYKdPn26ttXb37t3esfdjxoyxP/3pT+2oUaNsVFSUDQ8Ptx9//PFxYzwefDw0rflo0x/S+8Gq+mdDEhHxleqm9piYGMaNG1fvMZdffjkzZsygc+fOLFy4kOXLlzN69GjWrFnDzJkzGTJkCOvWrTvhTHDVoqOj+eyzz7jnnntISkrirbfeIiwsjOeee4777ruvzrHDhw/n7bffZsCAAXz11Vd8/vnn9OvXj1WrVvHmm28yevRo1q9f762JZ2Vl0aNHD15//XW++OKLY8ZwzTXX8MUXXzBmzBhWrVrFRx99RJ8+fXjppZd48sknG/R3nIqzzz6blStXcumll7Jx40bmzZtH+/bteeqpp5g1a5b3GX1cXBwffvgh//3f/83evXuZOXMm69atY+TIkbz77rtMmjQJgDZt2jB//nyuuOIKfvjhB1588UW2bNnCBRdcwGeffXZSK+A1hrGNaJqp9wTG/BhYbK09cTc9H6hedW3KlClH7TvloWm1LX4WPvwt3LoU0nr65pwiIiLHUd/QtOre7NbaY/YCPG4yN8b8swHXTgN+4u9kfqo3ISdUkAOP94KRt8O5DzTttURERI6hupXgVJL5/IZezFp79Oj7JuC3ZA7w8gTYvQp++Q2E+eVeRUREpI6GJPPjzorirwQdsAZeAz+8BxvnQ/exbkcjIiJSL3WAO54eF0BsS2d6VxERkQClZH48EdHOmPNv34HiA25HIyIiUi8l8xMZcA1UlsI3s92OREREpF5K5ifSbhCk9oQ1s9yOREREpF5BuyxYkyyBWh9j4EdXwPw/w8GdkNzet+cXERHxONklUE950hh/8+vQtGq5G+DZIXD+n2HErf67roiINHsNGZqmZvaGSO3mTPG69k23IxERETmKknlD9bscdi6H/M1uRyIiIlKHknlD9b3Uef9mjrtxiIg0I8aYJl8LPBQomTdUy0zIGA5rNURNREQCi5J5Y/S7HPaugX3fux2JiIiIl4amNUbf8fD+b5za+ej/bbrriIiEmI8//pjf/va3rF27lsTERC6//HKefPJJKioquPvuu5kzZw4HDhygT58+PP7445x11lluh+wKDU3zl+nj4PAeuO0rZwy6iEiAeWDeN6zbdahJzt2nXRL3X9S3Ud/Jzc2lQ4cOXH311dx0001s3bqVSZMm8Ze//IUdO3bwj3/8g+eff5727dvz2GOP8dlnn7F3717Cw8MxxjBnzhzGjx/fJH9PMDjlVdOkHv0uh3fuhD1roG1/t6MREQl4W7ZsoaSkhKysLEaMGMGIESNIS0sjKSmJBx54gKFDh3LFFVdgjOG5555j/vz5lJWVERsb63boQUM188YqzIPHusPI2+HcB9yJQUQkiJSVlXHJJZcwf/58fvKTnzB27FgmTJhASkoKb731FjfeeCNpaWlcfPHFXHTRRZxxxhne2qhq5gE4aYwxJswYM/eIsnhjzEfGmNXGmGnGcYMx5j/GmAXGmMDKmPGtoOtoZ+GVILsREhFxQ1RUFO+99x7Lli2jZ8+eTJ8+nYyMDP76179yySWXsGPHDu6991527drFxRdfzODBg9m7d6/bYQcVvyVzY0wmsAbodcSuy4H5wEDAAqOAfsD11tqzrbX3+yvGBut3ORzY5kwiIyIix/Xll19y77330rdvXx566CGWLl3KnXfeyeOPP86DDz7I0qVLmTRpEi+99BJbtmzh+++/55133nE77KDiz2fm24ABwLtHlK8HPrfWWmPMHk9ZZ+B+zw3AHdbaFX6M88R6/ReERznTu2YMdTsaEZGAVlVVxUMPPUR4eDgXXXQROTk5fPrppwwcOJAvvviCV155hUceeYQ2bdrw6aefUlJSQv/+6pPUGH5L5tZ5yF1R/cy7VvkST9P6RJxa+YPAp8CbQCqQDZx+5PmGDm14Es3Kyqp3KNtJi0mG7uc5Q9TO+yOEhfvu3CIiIWbEiBE8/fTTPPXUUzz66KO0bNmS888/n7/85S8UFhZy++23M2nSJEpKSujZsycvv/wyw4YNczvsJpWdnU12drbPzuf3DnDGmPettRfU+myAR4B04DZr7eEjjv/MWntmrc/udoCrtvZNmPXfMPld6HTUvYaIiIhPBFwHuGPwTHrO5OpEbox5yxiTaIxpBVS6F9px9LgAIuO0kpqIiLjOtWRujLnPGNMHGA6cA8z39F4/HXgGWIzT3H6vWzEeV1S8k9DXvQWVFW5HIyIizZjGmZ+Kb9+B166DibOh2zluRyMiIiEoWJrZg1e3sRCdpJXURETEVUrmpyIyBnqNg2/nQUWp29GIiEgzFbRzs7uyalp9+l0Oq1+GDZ9Ar5/499oiIhJStGqaWyrL4bEe0HUMXPGC29GIiEiI0TNzfwiPhD6XwPfvQXmx29GIiEgzpGTuC73GQXkhbPnC7UhERKQZUjL3hU6jICIW1n/odiQiIiFvy5YtGGOYPn2626EEDCVzX4iMhS5nwfoPtCyqiIiPFRQUYIxhwYIFALRv357169dz2WWXuRtYAFEy95Xu58L+LZC3we1IRERCWmRkJN26dSMpKcntUAKGkrmvdD/Pef/hA3fjEBEJMMYY5s+fz+TJk0lLS6N9+/Y8+eST3v2VlZX8+te/pkOHDkRHR5ORkcGvfvUrKiqcqbITExMBGD16NJMnT/aec+7cuTz77LMkJSVRUlLiPV9+fj6RkZHMnDkTgF27dnH11VeTnp5OSkoKV111FTt27PDTX+8fGmfuKy06Qlpv57n5yNvciUFEBOC938CeNU1z7jY/ggsfbvTX7rjjDn72s59x00038dRTT3HXXXdx7rnn0q9fP6ZNm8aTTz7J3//+dwYMGMCyZcv4xS9+Qf/+/bnhhhtYtWoVAwcOZObMmYwdO7bOea+44gruuOMOPv74Y8aNGwfA7NmziY2N5bLLLqO0tJQzzzyT4cOH89Zbb1FSUsIDDzzAmDFj+Prrr4mJifHJz+IrJzvOPGiTuS/XgfWZ7ufCf56D0sMQneh2NCIiAeOss87iF7/4BQBdu3bltddeY8OGDfTr148uXbrw4osvMmHCBACGDBnCE088waZNm7zHg/OsPD09vc5527Rpw+jRo5k9e7Y3mb/66qtcffXVxMXFMWPGDABefPFFwsPDARg8eDBpaWmsWLGCkSNHNv0f3wj1VUqnTZt2wu8FbTIPSD3Oh8VPw6YF0NulFgIRkZOoOTe1ESNGeLdbt25dZ995553Hxo0bee2111i9ejWLFi1i/fr1DT73hAkTuOeeeygvLyc/P5/58+fzhz/8AYA1a9awadMm4uPj63ynrKyM9evXB1wyP1l6Zu5LHX4M0ckaoiYicoS4uLhj7vvb3/7G4MGDeeutt+jWrRtPP/00AwYMaPC5L7vsMg4dOsTChQuZNWsWXbt29d48VFRUMHToUFatWlXn9e2333pr8qFANXNfCo+ErqNh/UfOEDVzzJn3RETE409/+hOPPPIIN998M+B0iMvJyWnw91NSUjj//POZPXs2a9eu5cYbb/ROgdqnTx9mzpxJhw4dvLXzbdu2cd999/Hggw/SqlUr3/9BLlDN3Ne6nweHdzdd5xMRkRATHx/PG2+8weLFi5k1axaXXHIJOTk5rF27lry8PMLCnFS1du1acnNz6z3HhAkTePXVV1m8eDHXX3+9t/zaa68lOjqaq6++msWLF/Pee+8xYcIEvv76a9q3b++Xv88flMx9rZunp+V6DVETEWmI6dOnk5eXxznnnMODDz7IpZdeysyZM1m4cCHvvPMOcXFx3HzzzfzmN79h6tSp9Z7j4osvpri4mLFjx5KRkeEtT0hI4PPPPycsLIyLLrqI66+/ns6dO/Puu+96bxJCQdCumjZlypSj9rk6NK227LMhPAp+qmfnIiLScPUNTavuzX68VdOCNpkHdNzz/wyf/QXu3ghxKW5HIyIiQUxLoLql+/lgq2DDJ25HIiIizYCSeVNoNwjiUjVETURE/ELJvCmEhTkd4TZ8DFWVbkcjIiIhTsm8qfQ4D4rzYedytyMREZEQp2TeVLqOAROupnYREWlyQTsDXMCtmnak2JbO9K4/fABjfud2NCIiEgROdtU0DU1rSp8/AZ88AHd9B0lt3Y5GRESCkIamua37ec77ho/cjUNEREKaknlTSu8LSe313FxERJqUknlTMga6nwsbF0BFmdvRiIhIiPJrMjfGhBlj5h5RFm+M+cgYs9oYM804uhhjPjPGLDbGDPJnjD7X/XwoOwzblrgdiYiIhCi/JXNjTCawBuh1xK7LgfnAQMACo4B7gF949t3nrxibROcznUVX1NQuIiJNxJ81823AAGDrEeXrgVes0z19j6fsR8Aaa+1uIN1/ITaB6ATodLqSuYiINBm/jTP3JOuK6qFltcqXeJrWJ+LUyh8Eiq211fOgFtV3vqFDhzb42llZWfWOS/eb7ufB+7+B/M2Q0tm9OEREJCBkZ2eTnZ3ts/P5fZy5MeZ9a+0FtT4b4BGcGvht1trDxpglwBnW2gpjzFJr7fBaxwfPOPNqeRvhmcHwk8dg+NHrsIuIiBxLsIwzv9TzPtlae9izvRr4kTGmHbDTnbB8qFVXSO4IWz53OxIREQlBrk3naoy5D5gFDAfOAeZ77j5+h1NTn+GJ7+duxehTmSNh4ydgrTNkTURExEc0nau/LJ8B834Bt34FaT3cjkZERIJEsDSzNw+dTnfet37hbhwiIhJylMz9JaULJKTD1sVuRyIiIiFGS6D6izHOc/OtX+i5uYiI1EtLoAaDpdPg37+CO1ZDy05uRyMiIkFAz8wDTeYo511N7SIi4kNK5v6U1gtiW6oTnIiI+JSSuT+FhUHHkaqZi4iITymZ+1vmSMjfBId2ux2JiIiECCVzf+tU/dxcTe0iIuIbGprmb+k/gqhEp6n9R1e4HY2IiAQQDU0LJjMvh4M74db/uB2JiIgEOA1NC1SZI2Hft1CY53YkIiISApTM3ZDpmad9m3q1i4jIqVMyd0O7QRARoyFqIiLiE0rmboiIgoxhsGWR25GIiEgIUDJ3S6fTYc8aKDnodiQiIhLkNDTNLZkjAQvbvoQe57kdjYiIBAANTQs2ZUXwcEcYcSuc+4Db0YiISIDS0LRAFhUH7QdrJjgRETllSuZuyhwFu1ZCWaHbkYiISBBTMndT5iioqoAdX7kdiYiIBDElczd1GA4mTOPNRUTklCiZuykmCdr0hy16bi4iIidPydxtnU53mtkrSt2OREREgpTGmbstcyQseRZ2roDMEW5HIyIiLtI482BVlA+PdoYxv4czf+V2NCIiEmA0zjwYxKVA6z4aby4iIidNyTwQZI5ypnWtrHA7EhERCUJ+TebGmDBjzNwTlRtj7jXGLDLGLDDG3OLPGF2RORLKC2HParcjERGRIOS3DnDGmEzg30B4A8r7A2OttSX+is9VmaOc962Lof0Qd2MREZGg48+a+TZgALC1AeVtgBnGmI+MMZ39FJ97EtOhVTeNNxcRkZPit5q5dbqfV1T3Rj9B+bvA34AzgD8D1xx5vqFDhzb42llZWfUOZQsoHUfAt/OgqgrC1JVBRCSUZWdnk52d7bPz+X1omjHmfWvtBQ0pN05//AXW2rNqlYXW0LRqK16Et2+DW7+CtB5uRyMiIgEiKIemGWMijDHvGWPCgT7ALrdj8osOw513LboiIiKN5FoyN8bcZ4zpc2S5tbYCeBlYDUwH7vdzaO5o1R1ikmHHUrcjERGRIKMZ4ALJi5dBwV64RR3hRETEEZTN7M1axjDIWQelh92OREREgoiSeSDJGAa2yll0RUREpIG0alogyfBMGLPjK+hy1vGPFRGRkKNV00LFs8MgpQtc+5rbkYiISADQM/NglDHMqZmH6s2KiIj4nJJ5oMkYBkV5sH+z25GIiEiQUDIPNBnDnPftmjxGREQaRsk80LTuDVEJmglOREQaTMk80ISFQ/vBmglOREQaTEPTAlHGMFj0Vygrgqg4t6MRERE/0dC0UPL9+/DK1XDje5A50u1oRETERRqaFqwyPGu1b1dTu4iInJiSeSCKT4WWndUJTkREGkTJPFB1GK7JY0REpEGUzANVxjBnOdSD292OREREApySeaCqfm6upnYRETkBJfNAld4PImI1E5yIiJyQxpkHqvBIaDdINXMRkWZE48xD0Uf3wX+eg//dARHRbkcjIiIu0DjzYJcxDCrLYPdqtyMREZEApmQeyKpXUFNTu4iIHIeSeSBLbAPJHTUTnIiIHJeSeaDLGAo7lrkdhYiIBDAl80CXMQwO7YBDu9yOREREApSGpgW6DsOd9x1fQZ9L3I1FRESalIamhaqKUngoA358E5z3R7ejERERP9PQtFAQEQ1tB2omOBEROSYl82CQMQx2r4KKMrcjERGRAOTXZG6MCTPGzD1RuTFmuDFmiTFmoTGmoz9jDEgZQ6GiBPaudTsSEREJQH5L5saYTGAN0KsB5fcDFwH3AHf5K8aAVbsTnIiIyBH8WTPfBgwAtjagvJW1NhdYCQz0S3SBLKk9JLZVMhcRkXr5bWiadbqfV1T3Rj9BeaFnX6kxpqK+8w0dOrTB187Kyqp3KFvQMMZpatdMcCIiISE7O5vs7GyfnS9Qx5knAhhjYoGo+g5YtqyZzYqWMRy+nQcFOZDQ2u1oRETkFDSmklk9NO14ArU3+z5jTBpOE/sKl2MJDFp0RUREjsG1ZG6Muc8Y0+cYu6cCbwMPAU/4LahA1m4ghEXCtv+4HYmIiAQYzQAXTF44D6oqYconbkciIiJ+ohngQk3mSGfymLJCtyMREZEAomQeTDqOhKoKLYkqIiJ1BGpv9hNqNqum1dZhOGBg2xLocpbb0YiIiI9p1bTm4u+nQ2xLmNT4/7FFRCT46Jl5KOo40mlmryx3OxIREQkQSubBJnMElBfB7tVuRyIiIgFCyTzYdBzpvG/9wt04REQkYCiZB5vEdEjpCluXuB2JiIgECCXzYJQ5wunRXlXldiQiIhIAlMyDUceRUHIA9n3ndiQiIhIANM48GGWOcN63fgHpx5reXkREgo3GmTcn1sITvZ3pXa/4p9vRiIhIE9I481BlDHQc4XSCa843NSIiAiiZB6/MkXB4FxzY6nYkIiLiMiXzYJVZPd58sbtxiIiI65TMg1Vab4hpoWQuIiJK5kErLAw6nuaMNxcRkWZNQ9OCWccR8MP7UJADCa3djkZERE6RhqY1R9u/ghfGwpUzoO94t6MREZEmoKFpoa7tAIiIVVO7iEgzp2QezCKiIGOoOsGJiDRzSubBLnMU7FkDJQfdjkRERFyiZB7sMkcAFrYvdTsSERFxiZJ5sMsYBmERamoXEWnGNDQt2EXFOx3h1AlORCToaWhac/bh7+DL5+E32yEyxu1oRETEhzQ0rbnoOBIqy2DncrcjERERFyiZh4KOpznv2/TcXESkOVIyDwVxKc7CK1v13FxEpDnyazI3xoQZY+YeURZjjJlljFlsjLnRU3avMWaRMWaBMeYWf8YYtDJHwvYvobLC7UhERMTP/JbMjTGZwBqg1xG7LgUWAWcAk40xEUB/YKy19mxr7XP+ijGoZY6EsgLYu8btSERExM/8OTRtGzAAePeI8iHAXGttpTFmO9ARaAPMMMakAFnW2s1Hnmzo0KENvnBWVla9Q9lCSscRzvvWJdBukLuxiIjIcWVnZ5Odne2z8/l9aJox5n1r7QW1Pk8DHrLWbjLGPAq8BowB/oantm6tvabW8Rqadix/7Q9t+8PVM92OREREfCRYhqYdBDI82x2AA9bav1hri4APgXauRRZsOp0Bmz6DijK3IxERET8KhGS+DBhijAkH2gPbjDHveT73AXa5Gl0w6X0RlB6ETQvcjkRERPzItWRujLnPGNMHmAuMwukE909rbTnwMrAamA7c71aMQafraIhOgnVz3Y5ERET8SNO5hprZN8EP78GvNjjrnYuISFALlmfm4kt9xztrm29e6HYkIiLiJ0rmoabrGKep/Zu5bkciIiJ+oiVQQ01ENPS8EL57Byr/CuGRbkckIiINpCVQpcZ3/4ZXr4GJb0K3sW5HIyIip0DPzJurrmMgKlFN7SIizYSSeSiKjIGeF3ia2svdjkZERJqYknmo6jMeivfD5s/cjkRERJqYknmo6nYORCVoAhkRkWZAyTxURcZCjwvgWzW1i4iEOg1NC2V9x8PaWbDlc6dTnIiIBDQNTZOjlRfDX7pBv8vh4qfdjkZERE6ChqY1d5Gx0ON8T6/2CrejERGRJqJkHur6jIeiPKepXUREQpKSeajrfi5ExqtXu4hICFMyD3XVTe3fzlNTu4hIiFIybw76jnea2rcucjsSERFpAhqa1hx0Oxci45y52ruc7XY0IiJyDBqaJsf3xmTY/Dn8z/cQHrT3cCIizY6GpkmNPuOhKBe2fuF2JCIi4mNK5s1F9/Ocpnb1ahcRCTlK5s1FVJyT0L+dB1WVbkcjIiI+pGTenPQdD4X7YMMnbkciIiI+pGTenPS4EJI7wicPqHYuIhJCgrZbs4amnYTIGDh3Ksz6b1j1Egy+we2IRESkFg1Nk4axFl44Fw5sg9tXQHSC2xGJiMhxaGjaKbLW8tG6vVz3j/8wd+VOt8PxDWPg/IegYC988ZTb0YiIiA8EbTN7U6qqsrz/zR6e+XQD3+4+BMCegyVcMrCd9w4pqHUY5qxxvvgZGDIJkjPcjkhERE6Baua1VFZZ3lq1k/P/+hk/f2kFpRWVPH7lAP44vh8b9xXy9Y6DbofoO+fcD7YKPnnQ7UhEROQU+TWZG2PCjDFzjyiLMcbMMsYsNsbc6CkbboxZYoxZaIzp2NRxlVdW8cay7Yx9YiF3vLqKMGN45ppBfPTLs7h8SAYXD2xHVEQYs1fsaOpQ/KdlJoz4OXz9Kuxc4XY0IiJyCvzWAc4Ykwn8Gwi31vaqVX4NkA48A3wKnAO8BUwCugETrLV31jrepx3g5n+fw+/nrmXH/mL6tkvi9jHdOK9PG8LC6jan3/ryCpZszOM//3sOUREh0qBRcgieHgSpPeDGfzvP00VEJKAEWge4bcAAYOsR5UOAZdbaSmA70BFoZa3NBVYCA5syqPioCFolRPPCpKG8c/vpXNCv7VGJHODywe3JLyxj4Q/7mjIc/4pJgjG/hW2LnZnhREQkKPmtA5x1qtIV1TXrWpKBXZ7tXUBLoNDznVJjTEV95xs6dGiDr52VlVXvuHSA4Z1TmPvzkSfs2HZG9zRaxUcxZ+UOzu2T3uBrB7xBN8CX2fDRfdDjfIiIdjsiEZGQl52dTXZ2ts/OFwi92Q8CGcAmoANwAEgEMMbEAlH1fWnZsmU+C6AhPdQjw8O4eGA7XvrPNg4WlZMcF+mz67sqPALO/yPMvByWToORt7kdkYhIyDteJfNIDclRgfDwdxkwxBgTDrTHaY7fZ4xJw2liD5jeWZcPzqCssop31uw68cHBpNtY57XwUSjMczsaERFpJNeSuTHmPmNMH2AuMApYBPzTWlsOTAXeBh4CnnArxiP1bZdE99YJzFkRIhPI1HbeH6HsMCx8xO1IRESkkTSdayM9t2Ajj7z/HQvvPpvMVvGuxNBk3vklLJ8BP/8PpPVwOxoRESHwerOHhPGD2mEMzA7F2vnZ90JkHMy7A8qL3Y5GREQaSMm8kdomxzKyayvmrNwZeou9JKTBuCdh2xJ4ZYISuohIkAiE3uwnxc0lUC8blMH/vLGa5Vv3M7RTSpNfz6/6XwlVFTD3FiehX/MqRMa6HZWISLOgJVD9qLC0gqF//Jjxg9rz0GU/ci2OJrXqFSehdzkbrnlFCV1ExCV6Zt5E4qMjuKBfG979ehcl5ZVuh9M0Bl4D4/8fbFoAr1yjJncRkQCmZH6SLh3UnkMlFXz6XY7boTSdgdfCJX9zEvqr1yqhi4gEKCXzkzSqWyqtE6NDayW1+gy6zknoG+croYuIBCgl85MUHma4dFB7Fny/j7yCUrfDaVqDroNLnvUk9OugvMTtiEREpBYl81Nw6eD2VFRZ5q0Oseld6zNooiehfwqvXgPFB9yOSEREPDQ07RT0apNEn7ZJzFm5k8mjOvvtuq4ZNNF5f/t2+NtwuPAR6DNe66CLiPiIhqa55B+fb+KP737Lx3edSbfWiW6H4x+7VjqzxO1eDd3Ph/96DFp0dDsqEZGQpKFpfnDxwHaEher0rsfSbhD87FM4/8+w5XP422mw5G9QWe/S8yIi0sSUzE9R68QYzuyRxuwVOzlYVO52OP4THgEjboVbv4ROp8MH98I/xsCuVW5HJiLS7CiZ+8BNZ3Ylv7CMK59fzJ6Dzaynd4uOcO1rcOV0OLwHpo2GD34LpQVuRyYi0mzombmPLN6Qy5T/W0aLuCj+76fD6ZqW4HZI/ld8AD6eCsv/BXGtYPhNMHwKxIXY/PUiIn7UkGfmSuY+tGbHQSb/aykW+NfkYQzo0MLtkNyxYxl89hf44X1nSdXBk5wm+RYd3I5MRCToKJm7YHNuIde/8CX5hWU8f/0Qzuie5nZI7tm7DhY/DWvecD73uwJG3QHpfdyNS0QkiIR0Mp8yZcpR+/w9zvxYcg6VcMM/l7JxXwGPXzWQiwe0czskdx3Y7vR2XzEDyougxwVOUu84QmPURURqqW+c+bRp04AQTeaBHvfB4nKmzFjGV1vzuX9cn+YxqcyJFOXD0mmw9HkoyoPUns5ENAMmQEJrt6MTEQlIIV0zD4a4S8oruf2VlXy0bi+3j+nGXef28P6P0qyVFcHaWbByJmz/EsIinNr6oInQ7Vxn2JuIiABK5gGhorKKe+es4fVlOxjbuzUPXNKP9i1i3Q4rcOz7AVbNhFWvQGEOJKQ7NfWBEyGth9vRiYi4Tsk8QFhreWHRZh7/8AcAfnlud24c1ZnIcA3z96osh/UfObX1H94HWwntBkPf8c787y0z3Y5QRMQVSuYBZsf+Iqa+/Q0ff5tDrzaJ/PmyHzG4Y0u3wwo8h/fC16/BN7OdeeBBiV1Emi0l8wBkreWDb/Yy9e1v2Hu4hGuHd+TXF/QiOTbS7dAC0/4t8M1cWDf3iMR+KfS5RIldREJeSCfzQB6a1hAFpRU88eEPTF+8mZT4aH4/rjcXD2inDnLHk78Z1r0F38yB3aucstZ9oeeFzqvdYAjTowsRCV4amhak1u48yL1z1vD1joOM7NqKO8f2YHhnTX96Qvmb4bt34Pv3YdsS5xl7fGvocb6T2LucDVHxbkcpInLKQrpmHmxxH09lleWlL7fy9CfryS0oY3inFG4b040zuqeqpt4QRfmw4WP4/j3nvfQQRMRA57Og21joOhpaddMENSISlJTMg0xxWSWvfrWN7M82sftgCf0zkrl1dDfO7Z1OWJgSUYNUlMG2xU6N/Yf3nGfuAEkZ0PVs6DLaqbXHp7oYpIhIwwVUMjfGxAAzgXbANGvtvzzlScAbnvJsa+0zxph7gZ8AFcBr1trnap0nZJN5tbKKKmav2MFzCzeyNa+InumJ/Hx0V8b1b0e4knrj5G+CjfNh03zY/BmUHHTK2/R3auydz3SetWtlNxEJUIGWzK8B0oFngE+Bc6y1FcaY/wEOA9OBT4ALgBeAydbaoxYHbw7JvFpFZRXvfL2bv83fwPqcAjq1iuOGEZ24bHB7WsRFuR1e8KmqdHrEVyf37UuhqtzZ16IjtB0I7QZCu0HOthK8iASAQEvmjwFzrbWLjDEzgfustZuMMX8HnrLWfmuMeQb4B/AUsBdIAbKstZtrnafZJPNqVVWWD9ft4bmFm1i9/QBREWFc2K8NE4Z15LQuKXqufrJKC2Dncqdn/K6VsGsV7N9cs79FppPcU7pAcgcn4Sd3cJZyVec6EfGTQEvm04CHPAn8UZzm8+XGmF8CBqc2vgjIAk4H/gacgVNDv6bWeSzAkCFDGnztrKwssrKyfPa3uGndrkO8+tU25qzcyeGSCjq1iuPqYR25YkgGaYnRbocX/Ir3w+7VTmLfvcrZPrC9pgZfLTbFSerJHaBlJ0jtDq26O+/xaepsJyLHlZ2dTXZ2doOOXb58ORA4yfwx4G1r7WfGmFeA31lrNxpjEnBq420BC/zUWrvR8x0DLLDWnlXrPM2uZl6fkvJK/r1mN68u3c7SLflEhBnG9k7nqmEZnN4tjagIjbf2mapKOLwHDm53EvvBbZ737XBgG+zfCpWlNcdHJ0NqN09y97y37g0pXbWIjIg0WqDVzCfgJOyngfk4z8zLjTFDcJL4KuBN4GrgLWAc0Asn6R9VM2/uyby2DTkFvPbVNt5csZP8wjKSYiI4v28bftK/LaO6piqxN7WqSiex526AvA2Qtx5y1zvbh3bWHBce5Sz7mt4HWnte6X0gqb1q8iJyTIGWzKt7s7cHngc6ArOAXcDLQCzwuLX2HWPM9cA9QDFwnbX2h1rnUTI/htKKShatz+XdNbv56Ju9HC6tIDk2kvP7pvNf/dsxsmsrLe7ib2WFTmLP+RZy1jmvvevg8K6aY6KToXUvSO0BaT2dhJ/WA5I7akY7EQmsZO4rSuYNU1pRyec/5PLvNbv5aJ2T2FvERXJen3TG9k5nVLdU4qPV5Oua4v01CX7vOtj3Pez7Dopya46JiHWewVcn+NRuzuQ3KV3UAU+kGVEyF8B5vv75+prEXlBaQVR4GD/uksKYXq0Z3bM1nVKVHAJCUb6T2HO/d9Z6r34/uK3ucUntoVVXJ7lXv1K6Oj3uIzRsUSSUKJnLUcoqqli2JZ9Pv8vh0+9z2LSvEIAuqfGM7tWaMb1aM6xTip6zB5qyQmcCnLzq5/IbPc/l19dMhANgwiA5w6m9p3SBlp1rbXeCqDjX/gQROTkhncyDfdW0QLE1r5D53+Xw6ff7+M+mPMoqqoiNDGdY5xRGdm3FqK6p9GmXpJnnApW1Tm0+b72z+Ez+Jue137NdvL/u8QltIKWzk+RbdqrZTukMca3UEU/EZVo1TU5ZUVkFX2zI44sNuXyxIZf1OQUAJMdGclqXFEZ2TWVUt1Z0TUvQRDXBonh/TZLfvxnytzjv+7fU7WkPEJXorA/fIrPmvUXHmu3oBDf+ApFmL6Rr5sEWdzDKOVzCko15LN6Qxxcbc9mxvxiAtMRohndKYWinlgzrlEKvNolEqJd88CkvgQNbnWS/f4snyW91yvZvhfLCusfHtXKSu/eVWTMzXosOEJ3oyp8hEuqUzMWntuUVsXhjLks25bFsy352HnCSe3xUOIMzWzIk00nuAzu0UE/5YGctFOV5kvuWukm+evKc2hPlQN1Z8aqnvk3O8JR1dOa6V4uOSKMpmUuT2nmgmGVb8lm+dT9fbdnPd3sOYS2Ehxl6tUlkYIcW3lfXtAQt4xpKqqqgcJ8zA96Brc77we2eZL/D2S4vqvudyDgnuVcnee97BiS3d3roR2hKYpEjKZmLXx0qKWfF1v0s27Kfldv38/X2gxwurQAgMTqC/h2SGZBRk+BbJ8W4HLE0GWud5/XVSf7gjrpT4R7a6dwMHCkh3UnuSe2c5O5992wnttXQO2l2lMzFVVVVlk25BazcdoDVOw6wavsBvtt9mIoq53+71onR9G2XRL/2yfRtl0y/9km0bxGrznXNRXmJk9QP7qj18iT+Q7ucfWUFR38vvjUktYXEdp53z6t2WUwLNelLyFAyl4BTUl7JN7sOsmr7Qb7ZdZBvdh5iw74CKj0JvkVcJP3aJdO3fRJ92ibRs00iXVITNO69uSo5VJPYD+2Cw7trtg/tdqbFLco7+nsRsZDYxlObb3NEwq/1ilTrkAS+kE7mGmceOkrKK/luz2HW7nQS/Nqdh/h+z2HKKqsAiAw3dE1LoGebRHq1SaJXm0R6tkmkbXKMavECFaWeJO9J7od2O5+9ZZ5XRcnR341p4Uns6Z73Ns5Y/ETPKyHdeY+M9fufJc2TxplLSCmvrGLTvkK+23OI7/Yc5nvPq7oHPUBiTATdWyfQvXUi3dMT6NY6ge7pibRTkpcjWQslB+om98N7PK/dULC35vORa9eDsxhOQuuaBJ+Q7twAJLSpeU9oDbEt1bwvPhfSNfNgi1t842BxOT/sPexJ8IfYkFPAhpwCcgvKvMfERYXTrbWT3LumJdAlNZ4uaQlktoojJjLcxegl4FVVOR33qpN9gSfBF+R4tvfWvFcUH/398OhaiT69JvnHp9XcBCS0dl7quS8NpGQuzUZ+YRkbcgpYn3OY9XsLvNt7D9WMhTYG2reIpYs3wcfTJdVJ8u1axGrKWmk4a6H0kJPkD+9xavbVtXvvtifxHzmlbrWY5CMSfK33+Na1tlMhTDehzZmSuTR7BaUVbN5XyKbcAjbtK2RTbiGbPdtFZZXe46LCw+iQEkvn1HgyW8XTKTWeTq3i6NQqnrbJMZrhTk5eRZkzDK9gr6eGX/t9b9199fXeN2HO7HvxaZ4afmtPsvd89m63dj5r6F7IUTIXOQZrLXsPlbIpt4CteUVsyStkS26hd7ukvMp7bESYIaNlLB1bxdMxJZbMlHg6pMSR2SqOjilxmu1OfKe0AApzoGBfTbIvyKkpK8zxJP99R0+3Wy0muaZmH59aa7v2jYDnpY59QUHJXOQkVFVZcg6Xsjm3kC15hWzLL2JbXhHb8ovYmlfIoZKKOse3io+iQ0qc82oZS4cUJ8l3aBlH2xYxRKpWL02hrNCT6PfVvFe/an8uyHE6/9UnKrGmVu9t6q/1XL92s79q/K4J6WSuoWniloNF5WzNL/Qk9yK25xexfX8R2/OL2XWg2DspDjhT27ZJiiGjZSwZLeNo3zKWjBaxZLSMpX3LWNomx2oMvTS96qb+2rX+2rX9gpyaFoBjPuNvUVOzj089orm/uubvafqPivfrnxdKNDRNJABUVFax+2AJ2/cXsSO/mO37nRr9zv3F7DxQzJ5DJdT+T9cYSE+MoV2LGNq2iKVdcgxtk2Np1yLWKUuOpVV8lOa1F/+pKD3iGX9OzTP+whwozK1J/CUH6z9HZJynib/6uX7t7dqfWzv9AcL1qOp4QrpmHmxxiwCUVVSx52AJOw44CX6HJ8nvOlDM7oMl7DpQTGlFVZ3vRIWH0SY5hjZJMaQnx9AmKZr0pJiaMs9LNXzxO2/ir27Wz63V3F/Pdn1j+MFZca/exJ9a87y/+nN0UrMby69kLhJkrLXkF5Z5E/vugyXsOljM7gMl7DlUwt5DJew5WHJUwgdIiY+idWI0rZNiSE90En7rpGhaJzrv6UkxpCZEER2hYU7iAmudmnxhbk1HvupEX5BzdOI/1nP+8CiIS4V4Tw//uFTPDUBqzbb3vZXTITDIk7+SuUgIstZysLicPZ7E7iT4UvYeLiHnUCk5nvd9BaXeOe9raxEX6ST9xBjSEqNpnRhNWq1X68RoUhOiSY6N1Ex64p6KMmfe/TpJ3tPMX5Tr+Vxru75hfQBhkZ6hfam13qu3WznvcbX2xaYEXLO/krlIM1ZZ5dTy9x4q8Sb4nMOl7DvsSfiHS71Jv6yemn5kuCE1IdrziiLNk+RT4qNITYimVUKUdzslPkq99sVd5cW1knteTZL3vufV1PqL8qH0GM/7wensV7uGX2c7re7NQVyrJp/NT8lcRE7IWsuh4gpyDpewr8BJ9rkFZZ5351WzXVZvbR8gKSbCm9hbxkfRqvZ7XBQpCVGkxEV598dHhavmL+6pKIPi/JokX5RX86qv9l+UB/bom17AGeLnreV7avoJreHcB3wSakgncw1NE/G/qirLoZJy8grLyCsoI6+g1LudX1hKbmEZ+QVl7C8qI6+wjP2FZXWG6tUWGW5oEeck+BZxkaTER9EiLoqWcZG09JRVf24RF0lyrFOmFgBxRfW8/UVHJPjCWjcBtctMGPxyTaMvo6FpIhJwrLUcKqlgf2FNcs8vKuNAURn5heWe9zIOFJWzv6jM8yo/Zu0fICE6guTY6gRfk+hrlx35SoqJJDEmQkP8JCiFdM082OIWkYax1nK4tIKDngR/oKicA8VO4q9O+geLyjlY7JQfLC7nQFE5B4vLKK889v8vGAOJ0REkHZHkk2MjSYqNcLbjji5L8hwXExmmxwLiCiVzEWk2rLUUl1fWSu7lHPIk+4PF5Rwqqaj7ubicQyXV2xUUl1ce9/wRYcaT2CNIjKlJ9okx1e9OWWJMzTGJMU4rQmJMBAnREVqwR05KQCVzY0wMMBNoB0yz1v7LU54EvOEpz7bWPmOMGQ48BZQB11trt9U6j5K5iPhcaUUlh4orvAn+YHE5h0sqOFxS7i2v3j5cUnu/s6/2KnzHEhcVTmKtRF99M+Ati44godb+xOia7QTPcZonoPkJtGR+DZAOPAN8Cpxjra0wxvwPcBiYDnwCXAC8DkwCugETrLV31jqPkrmIBJyKyqo6yf1QrcRfXe7dLj1iX6mzXXu1vmOJigjz1vwToiPq3gzE1LQMJETXLqtVHhNBbKRGEgSThiRzf46MHwLMtdZWGmO2Ax2BTUB34ClrbZkxZhVOAm9lrc01xhwGHvZjjCIiJyUiPIyWnmF3J6usooqC0vpvAGqXH/KUO2UV5OYWeo8vKK044XXCw4y36b92K0B17T8huuYmICG67nEJnuMSoiOIjlA/gkDhz2SeDOzybO8CWnq2vwcuNMbsAs4GXgYKAay1pcaYev/LHDp0aIMvnJWVRVZW1slFLSLiJ1ERYaREOGPxT1Zlla2T+I++OaigoNSzr9aNwZ5DJRTsc8oOl1RQVnniVoKIMONN7NWtBPHRNZ9rbyd49iV6yuOjw0mMjiQ+OpyEZvj4IDs7m+zsbJ+dz5/N7I8Bb1trPzPGvAL8zlq70RiTAPwDaAtY4KfAK9ba4caYWOADa+2Ztc6jZnYRkSZWWlFJgfdmwHl0UFhaSUFpuZPwSyu8+6uPKSyt+Vzg+VzYgL4E4Mw7UFPrr+k/UF2WGF23VaC6BaHu5wjiQnAyokBrZl8GDDHGfAG0B6o7tfUEHgVWAW8Cm4F9xpg0nCb3FX6MUUREgOiIcKITwmmVcGpTlVZVWQrLnATvJPvKOjcBtW8ADntuGKpbD3IOl7BpX83NQn0LDB0pzOBtAai5GYis92ag9iOF6huGxBjnBiHYZih0ozd7e+B5nGfms3Ca3F8GYoHHrbXvGGOGAU8DpcAN6s0uIiJlFVXe5H/Ye0NQXusGoebRwpGtBrUfNzRk5EGYgaTYmlEH3mGInnkHWiVEkZoQ5V2/IC3RWa+gKR4XBFRvdl9RMhcRkVNR6WktqO43UFDqzENQfQNwqPawRM/8BIeqOx96hi0e6/FBUkwEqYnRtEuOZebPfuyTeAOtmV1ERMR14WHGW9s+WSXlld7Fh7wLETVgQaKmopq5iIhIAGtIzVxzC4qIiAS5oG1mr2/c+MkugZqdna1x6D6m39S39Hv6nn5T39NveupqL4H67bff0rt37wZ9T83sOBPQLFu2zGfnE/2mvqbf0/f0m/qeflPfqv491cwuIiLSDCiZi4iIBDklcxERkSCnZC4iIhLkgj6ZV/f6c5sv4gilc/hCoPwtgXKOQIghlM7hC4HytwTKOQIhhlA6R2MEbTKvXtb017/+tXfbzf8YA+V//EA5hy8Eyt8SKOcIhBhC6Ry+ECh/S6CcIxBiCPZzzJs3z5vTtm7d2uChfkE7zrx6HdisrCyfrgkrIiLiltrzpaxYsYLs7GymTZt2wu8Fbc1cREREHErmIiIiQS5oZ4ATERFpTjQDnIiISAgLupq5iIiI1KWauYiISJBTMhcREQlySuYiIiJBTslcREQkyDXrZG6MiTHGzDLGLDbG3Oh2PMHOGBNmjJnr2R5ujFlijFlojOnocmhBxxgTb4z5yBiz2hgzzRjT1Rjzmee/1UFuxxeMjDEdjTFfGWPWGGN+aoy52PPf6HvGmBZuxxesjDFXGWN+o3/zp84Y08MYs8oYs8AY81ZjftNm3ZvdGHMNkA48A3wKnGOtrXA3quBkjMkE/g2EW2t7GWPeBSYB3YAJ1to73Ywv2BhjbgAygIeA54GuwP8Ae4H/Z6291MXwgpIx5o/Al8B7wBKgCDgfGA+0sdb+1bXggpQxJhJYCcwEzkD/5k+JMeYnQHtr7TTP5wb//2izrpkDQ4Bl1tpKYDugu8mTtw0YAGz1fG5lrc3F+Yc+0K2ggth64BXr3G3vAUYDa6y1u3FuQKXx3gbmA7FAOFBurS0BlgGD3QwsiGUBH3i29W/+1HUGLjfGfGGMuZ5G/KbNPZknA7s827uAli7GEtSsowKobuop9JSXAmrtaCRr7RJgizFmIjAKmO+56QSnRimNZK1dCrTFuXF/E/3bPyXGmEScm8x3PEX6N3/qNgL3ARcBd9KI3zRoV03zkYM4TZmbgA7AAVejCS2JAMaYWCDK5ViCjjHGAI/g1MLHAx8aYyI8N0xJbsYWrIwxKdba9caYNsAi4JBnl/7tn5y7gCeBSM9n/Zs/Rdba96u3jTGLgYme7RP+ps09mS8DhhhjvgDa4zQVi2/sM8ak4TzrWeF2MEGo+pn4ZGutNcasBn5kjNkL7HQxrmA2zRjzW+AHoBRINMbE4DSxL3c1suDUE6dm3gLnBrOt/s2fGmPMg8D7OH07+gGrGvqbNvdkPhen48ZVwPPW2nJ3wwkpU3GeUZYCN7gbSlAaDpwDzHcq6fweeArn3+zPXYwrmD0JzMH5b/KfwGacZ+j7gWtdjCsoWWuvBTDGnA2cBnyC/s2fqudwfsMo4P/h3GQ26Ddt1r3ZRUREQkFz7wAnIiIS9JTMRUREgpySuYiISJBTMhcREQlySuYiIiJBTslcRERCnjFmujHGHuN1wE8xLDDG/LUpzt3cx5mLiEjzsRS4rp7yKn8H4mtK5iIi0lwUW2s3uB1EU1Azu4iINHvGmE6eJvcRxphPjDEHjTErjDHjah0TZoz5hTHmG2NMgTFmpTHm6iPO09cY85Hn+zuMMX82xoTXPcT8wRizzRiz3xgzzRgT4dkRZ4z5mzFmlzGmyBizzBhzVkPiVzIXERGpMQt4AxiLsyDPPGPMUM++24GHgGeAM4HXgJeMMRcDeOZRX4yzzse5wIPAHTiL0lSbjLNK3xXAY8DPgGs8+6Z6tu/wXH8b8OYRNwP1UjO7iIg0F2caY0rqKX8P+KVn+5/W2r8DGGOWAYNwkuv1wK+BR6r3AyuMMd2Bu3HmUP85sBfI8ixZvNQYkwx0rHWtzcAvrDOX+lLPMsddPfv64CwANsuzwNItOIvZRAHFx/vDlMxFRKS5WEb9C5YUAtW130+rCz0J9RNgnCcptwM+O+K7C4FLPNs/AhZ7Enn1OR494viltu6iKIW1tqcB/wK+M8a8DcwDXrMNWERFyVxERJqLImvtd/XtMMZ08mwe2bO9kuPnyqpa+yPr+f5RMRxrh7X2LWNMBnAlcB5ObX+zMeYCa+3e451Uz8xFRERqjDzi8xjgW2vtQWA3cMYR+88C1nm21wGnGWO8udUYc4cx5q2GXNgY83tguLV2hrX2OqATzrrx4477RVQzFxGR5iPWGNPtGPuq8+E9xpjdwBpgInA2znrtAH8BHjTG7AW+wqk934hTkwZnDfI7gL8bY6YBvYD7cDrMNcQo4BpjzD3AHpwbiRjg6xN9UclcRESai+HA+mPsG+Z5vw24CRgIbAQuttZ+6dn3FGBxOst1BDYA11tr5wBYa7cbY84EnsB59n4AeB74UwPjuwUn8c/ASeLfA9daa7860RdNA56ri4iIhDTPM/PNwCBr7Sp3o2k8PTMXEREJckrmIiIiQU7N7CIiIkFONXMREZEgp2QuIiIS5JTMRUREgpySuYiISJBTMhcREQlySuYiIiJB7v8DvbulHebq70gAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(figsize=(8, 6))\n", + "ax.plot(range(len(losses_valid_ssl[1:])), losses_valid_ssl[1:], label=\"ssl\")\n", + "ax.plot(\n", + " range(len(losses_valid_native[1:])),\n", + " losses_valid_native[1:],\n", + " label=\"native\",\n", + ")\n", + "ax.set_xlabel(\"Epochs\", fontsize=15)\n", + "ax.set_ylabel(\"Loss\", fontsize=15)\n", + "ax.legend(title=\"Validation loss\", loc=\"best\", title_fontsize=20, fontsize=15);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluate the SSL against native MLPF" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [], + "source": [ + "test_loader = torch_geometric.loader.DataLoader(data[5000:], batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_mlpf(model, with_VICReg):\n", + " num_classes = 6\n", + " conf_matrix = np.zeros((num_classes, num_classes))\n", + "\n", + " model.eval()\n", + " encoder.eval()\n", + " decoder.eval()\n", + " with torch.no_grad():\n", + " t = time.time()\n", + " for i, batch in tqdm(enumerate(test_loader)):\n", + " if with_VICReg:\n", + " # make transformation\n", + " tracks, clusters = distinguish_PFelements(batch)\n", + "\n", + " ### ENCODE\n", + " embedding_tracks, embedding_clusters = encoder(\n", + " tracks, clusters\n", + " )\n", + " ### POOLING\n", + " pooled_tracks = global_mean_pool(\n", + " embedding_tracks, tracks.batch\n", + " )\n", + " pooled_clusters = global_mean_pool(\n", + " embedding_clusters, clusters.batch\n", + " )\n", + " ### DECODE\n", + " out_tracks, out_clusters = decoder(\n", + " pooled_tracks, pooled_clusters\n", + " )\n", + "\n", + " # use the learnt representation as your input as well as the global feature vector\n", + " tracks.x = embedding_tracks\n", + " clusters.x = embedding_clusters\n", + "\n", + " event = combine_PFelements(tracks, clusters)\n", + "\n", + " else:\n", + " event = batch\n", + "\n", + " # make mlpf forward pass\n", + " pred_ids_one_hot = model(event)\n", + " pred_ids = torch.argmax(pred_ids_one_hot, axis=1)\n", + " target_ids = event.ygen_id\n", + "\n", + " conf_matrix += sklearn.metrics.confusion_matrix(\n", + " target_ids.detach().cpu(),\n", + " pred_ids.detach().cpu(),\n", + " labels=range(num_classes),\n", + " )\n", + " print(f\"Time taken is {round(time.time() - t,2)}s\")\n", + " return conf_matrix\n", + "\n", + "\n", + "CLASS_NAMES_CLIC_LATEX = [\n", + " \"none\",\n", + " \"chhad\",\n", + " \"nhad\",\n", + " \"$\\gamma$\",\n", + " \"$e^\\pm$\",\n", + " \"$\\mu^\\pm$\",\n", + "]\n", + "\n", + "\n", + "def plot_conf_matrix(cm, title):\n", + " import itertools\n", + "\n", + " cmap = plt.get_cmap(\"Blues\")\n", + " cm = cm.astype(\"float\") / cm.sum(axis=1)[:, np.newaxis]\n", + " cm[np.isnan(cm)] = 0.0\n", + "\n", + " fig = plt.figure(figsize=(8, 6))\n", + "\n", + " ax = plt.axes()\n", + " plt.imshow(cm, interpolation=\"nearest\", cmap=cmap)\n", + " plt.colorbar()\n", + "\n", + " thresh = cm.max() / 1.5\n", + " for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n", + " plt.text(\n", + " j,\n", + " i,\n", + " \"{:0.2f}\".format(cm[i, j]),\n", + " horizontalalignment=\"center\",\n", + " color=\"white\" if cm[i, j] > thresh else \"black\",\n", + " fontsize=15,\n", + " )\n", + " plt.title(title, fontsize=25)\n", + " plt.xlabel(\"Predicted label\", fontsize=15)\n", + " plt.ylabel(\"True label\", fontsize=15)\n", + "\n", + " plt.xticks(\n", + " range(len(CLASS_NAMES_CLIC_LATEX)),\n", + " CLASS_NAMES_CLIC_LATEX,\n", + " rotation=45,\n", + " fontsize=15,\n", + " )\n", + " plt.yticks(\n", + " range(len(CLASS_NAMES_CLIC_LATEX)), CLASS_NAMES_CLIC_LATEX, fontsize=15\n", + " )\n", + "\n", + " plt.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "454682ec20174c3c8bef066fc1fa9ba2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcgAAAGoCAYAAADCVXWwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAAsTAAALEwEAmpwYAAB5E0lEQVR4nO3dd3xUxRbA8d9JCCQhhQQISegdBBu9SO/tUUQBK2JDBQs2sCDSBUQQEEXF3hFREKQXRaVbAAsKNgKhpEISCGTeH3cTsuwG0rYknO/77Odl5869e8Zd9uzMnTtXjDEopZRSyp6PpwNQSimlvJEmSKWUUsoJTZBKKaWUE5oglVJKKSc0QSqllFJOaIJUSimlnNAEqZSLicgGETEXePwjIqtE5B4R8T1v32oX2dfZo5qHmqpUsaIJUinPqwx0AV4CvhORYA/Ho5RCE6RS7vSWMUayPwB/4HJgvq1OU2B8Dvt3OH//HB5/uaEtShV7miCV8iBjzCljzG5jzL1YPUiA+7UXqZTnaYJUyntkJkgfoK4nA1FKaYJUypv8le3vGp4KQill0QSplPeolu3vfzwVhFLKoglSKe9xj+3/M4C9ngxEKQUlPB2AUpcyESkF1ALutT0A5hhjkpxUXy8iFzrc38aYaoUboVKXLk2QSrnPrSJy60XqbAeecUcwSqkL0wSplOelAD8Cy4Dpxpj0HOp1MMZscFtUSl3iNEEq5T5vGWOGejoIpVTu6CQdpZRSyglNkEoppZQTmiCVUkopJzRBKqWUUk5oglRKKaWc0ASplFJKOaEJUimllHJCjDGejkEppZTyOtqDVEoppZzQBKmUUko5oQlSKaVUsSQiPiKy5LwyfxFZJCLfishtF9xfz0HaExH9D6KUUjbGmAveY62gCus79/w4RaQqsBzwNcbUy1Y+BKgAzAHWAZ2MMWecHVN7kEoppYqjf4Argb/PK28MbDfGnAX+BarkdAC9m0cO/K+6z6Es/d/1+FXukKv9c1v31G8fU6ru9YV6zJzqxm+b67TufcPvYt7LCy56zNzWA2jdvAmbt2zPVd28HNcVsXq6Xa6K1RXt8oZYc1vX05/BvNT19GfQWd0AP5d2HB04+87NjbQf5jktN9bw6BknPdRQIMb2dwwQltOxNUEqpZTyPHHbgGYiUAnYD1QGEnKqmGOCFJFdQK7Gho0xjfIWn1JKKZWNuK3Huh1oLCKbgYpYQ7FOXagHuaSQg1JKKaWcc3EPUkTGAouwctu7wPXAK8aY9Jz2yTFBGmOeLewAlVJKKadc1IM0xnS3/f/4bMUDc7Nvrs9BikgHYBBQC7gDaA78YYzZkftQi470f9c7FpYIyPX+PiHVCi+YfBwzL3V79u5TqPXyKi/HdUWsnm6Xq2J1Rbu8Idbi2C5Pfwa/XLaUf/75m/uG3+WSOIqqXF0HKSL/Az4FPgGuAxoANwGPA72MMWtcGaQ7Zc54yu+MqrzKyyzWgsppFqsr5GWmXVGi7So6imObwH3typzF6q7rIP2bPpyv/dO2PQ+4Js7cDvqOBcYbY24AztiCGQvMBiYWdlBKKaUuMSL5e7hQbhNkfeALJ+WfAA0LLxyllFKXJPHJ38OFcnv0v4GqTsqrce6CS6WUUip/vLAHmdtJOm8Az4tInO15tIhcCcwCXnVFYEoppZQn5TZBPg8EA6uAUlgLvJ4B5qHnIAvEt2wDT4fgEsPuKJ6z4bRdRUdxbBMU33a5cSWdXMvT3TxEpBTWZR4lsC7xOOmqwDzF3bNY3cmds1iVUkWb22extnoiX/unfTsZcE2cebkOsjwwDKgHnAJ+EpG3imOSVEop5WZe2IPMVUQi0ghrYddHgHCs9evGA/tEpIbrwlNKKaU8I7c9yBeAzcC1mT1GEQkGFgNzgZ6uCU8ppdQlwX2Lledabvu0V2EtFJA1nGqMSQaeBVq6IC6PS/93vcPjbOKBXO//w+KnuKJOxQvWad2oJuveeIjDm6az5vUHaXSZ/X07o8uHsmjW3Rzc8BxbPhrNjX2a56stBWWMYeL4cTSoV4vqlaMYfuftpKSk5Fh/8aeLaNWsMZHlytCnZzcO7N9vt/2XvXvp3qUjFcqGck3LZqz8aoWLW+Cctsui7fJcu7ylTV8uW8p9w++ye7hdLq55PBu3j/Q/vrR7uFJuE+R+rJtMni8E+K/wwvEefpU7ODx8Q6tffL8Svjw8tDN1q0desF7tqhF8Mfc+Nu/6kz73zmX3HzEsf3kkFSPKAODr68OKBffj4yNc+8DLLPz0W+Y/fQO92l1eGM3Lk6mTJzJ/3hwmT53OwrfeZevW7xk29GandTdt3MCtNw1h0JAbWbJ0BUHBwXTu2JbU1FQA4uLi6NqpHVWqVGXp8lX06NmL6wb0ZecO9y/pq+3Sdnm6Xd7Spl69+zDv5QV2D7fLRYL0LVcfvzp97R4uDSmXa7F2xxpmHQ58g3WfyGuA14CnjDEfuzJIdyrILNa7rmvDcw8PwL+UHwDNB03hp98POq075aH+NL+iGh1veyHzddmx6Ane/3IbMxauokebhrw//XaqdBpD8sk0AGY/MYjqFcvxv/uc30H7YvIzizU9PZ3a1SszdtwEht1xJwBbt2yhfZuW/L7/HypVqmRXf8iggZQqWYo333kPgLS0NKpVimT23PkMGjyE+fPmMvP5afy67wC+vr4A9O3dg8qVqzB3/iv5ald+aLu0XZ5ul7e3ye2zWNuPv1hVp9I2jAXcvBariMSLSJxtcYD3sS7vWAek2R7rsSbrjC7soIqqz9bsovWN0+hz78UTWMcWdfnqm71Zz40xrNq8l84t6gHQqUVdNu/6Mys5Aqz8Zg8dmtXBx8d9Y/V7du8mNjaWbj3OnWZu0rQp4eHhbFi31qH+ujWr7er6+/vTvmMn1q5eZW1fu5ouXbpl/QMG6Na9J2vWrHJhKxxpuyzaLs+1qzi2qbi50CSdB90VRHFxNP4ER+NPcCLl1EXrVigbwqGjiXZlMUcS6dr6sgtsT6BECV/KhpbmaPyJwgv8AmJjDwMQGXluyNjHx4eoqGhij8Ta1U1JSSEpKYno6Gi78ujoiuz/8w8ADh8+zBVXXmW/vWJFjsTGYoxB3HSiXtul7fJ0u4pjmwrECy/zuNANk9+62M4iEgXcVqgRXSLCQ0pz8rxEmnwyjbJlggAICy3Nvr+P2G+31S8bFuS2BBkXF0dgYKDdr1KAoOBgjh87ZlcWHx8PQOnSQXblwcHBHLPVjY+Po3SQ4/bU1FRSU1MJDAws7CY4pe3Sdnm6XcWxTQXihQk8V5d5iEgA8DBw/jWPtYHqwORCjqvYi0s6SVDpUnZloUEBJCRZM9jiE08S7GQ7kFXHHcLDw0lJSeHs2bN2/5CTkhIJCwuzq5v5/MSJZLvyxMRzdcPCwjmR7Li9ZMmSBATk/obUBaXt0nZ5ul3FsU0F4oU9yNxGNBd4CKgE3AqUAy4DmgPOp1ypC4o9nkS0bcZqpuiIUA4fSzq3vbzj9rNnMzgSZ/+PwJUqVLCGf2Jizt20JSMjg0MxMURGRtnVDQwMJCQkhJiD9hOTYmIOZtWNjIx03H7wIJFRUW4dAtJ2abs83a7i2KYC8cK7eeQ2QfYERhpjumItWD7eGNMC+BBo4qrgirP1W36jm+18I1izWLu0uoy13/8KwLotv9HyqhqEBPln1enaugEbtv1ORkbu188tqAYNGxIREcGqbNdTbdu6lYSEBNp16OhQv0Onzqxcea5uWloaG9evo2PnLtb2jp1Zs2YVZ86cyaqzauUKOnXq4sJWONJ2WbRdnmtXcWxTcZPblXTCgF9tf38PNAa2Ax8AM4DpBQnCNs23I1bvtBdwGphhjHnBtt0HGAHcjXVfyn3AVGPMR7bt7YGvsHq1C7B6tn8DDxpj1tjq+AKjsNaTrQr8gpXoPy9I7LkVXNqf8uFB/HMojjNnMnj9083cObANE+7/H5+v/ZGb+7Ygqnwo73zxPQArN+/ln0NxvDl5KM+9tpKr6lVmaN+WDHrYvXcX8/PzY/i9Ixj71BjKR0QQHBzMqAdH0v/agVSpUoW4uDji4+KoWasWAPfeN5Ke3TrTqFETWrRsxQszpxMSGkq//gMAGHzDjUyZNJ57776TO+4azqqVK9iwfh2bNm/Rdmm7Lql2Fcc2FYgXDrFijLnoA/gDGGX7uz+w2PZ3O+BEbo5xkeMb4CfgfqyVeT60lTW0bX8AOIl1HWYjrEtLzgD/s21vb3u+EegHdAV2Akc4d63neOBPW/xXA88AZ4FOTmIxElA+148Sldob/6vuy3rU6fG0McaYZtdPziq74+m3jTHG1OnxdFZZ59tfMN//uN8kJqeYb3bsMy0GT7E7To0uT5gvN/5s4hJPmp9++8/c9uSbdtvz+khNN/l6pJzOMGOefNpUq17dVKhQwQy97XYTl5RiUtONefLpZwxgV/+9Dz8xV155lQkNDTWdu3Q1v/y+3277jh92m7bt2puQkBDTuElT88WXX+U7toI8tF3aLk+3y91tmvvSK6ZRo8a5emR+Fxb0+z2X3//Gv+v0fD1cGWduFwp4FHgOqwe3HNgNzMcaek0yxrS96EEufHwDzDXGjLQ9jwBigf7GmCUichB4xRgzPts+rwN1jDFtbD3I9cBAY8yntu3XAx9h9X7TgHigmzFmU7ZjfAAkG2Puylamt7tSSl3y3L5QQLfn87V/2sqHAQ/e7soYM11EfgLijTEHRWQY1p09jmD1+grDd9le70jmSWURCQWigU3n1d8InL/O0HfZ/s5+jUQtwB9Ynflm2Pg5Oa5SSil388KJRLm+H6QxZmW2vz8DPivkWPJ67UIGjvHndIzMej2B89d+O53H11VKKXUJyDFBikiue4bGmBcLJxynx04UkUNAG2BDtk3tgL1Od3L0B9Y5ynLGmLUAYnVRn8WarLP/AvsqpZRyNS+cpHOhHuRDuTyGAVyWIG2mAxNEJBbYhjUJ5zbgutzsbIw5ISLzgFm2Ida/gIFYbWzlkoiVUkrlXlEaYjXGXPzeTu4zGysRPwRUweoR3mwb6s2tx4AkrFV/orEmGvU2xmwr5FiVUkrlVS56kGdjfyYjdrcbgrHkahbrpURnsSqllAdmsfbO32380pZZ39WuiNP7Bn2VUkopL5DrWaxKKaWUyxSlc5BKKaWU2xSxWawORCQS6xZXu4CzxphUl0TlBdL/Xe9Q5hNSDd9Qb5q7pJRSBfflsqUsX7bUs0F4YQ8yt0vNhQJLsK49zMBaFHw0EAIMNca45+69bqCTdJRSygOTdPrl70YMaUvuBDw7SWc6UAaoj3XBPVj3iGwITCnsoJRSSilPy22C7AuMNsb8hnU9IsaYncAYYICLYlNKKXWp8MIbJuf2HGQJ7Bf/zvQvEFh44SillLoUiReeg8xtD/J74MZszzNPXN4D7CjUiJRSSl1yRCRfD1fKbQ9yFPCdiFyFdYuoqSJSB6gGXOOa0JRSSinPyVUP0nbusQ7wNdZtrnyAL4C6xpgfXReeUkqpS4Lk8+FCebkf5DGs20MppZRShcobz0HmKkGKyMwLbTfGjCqccJRSSl2KimyCBK4+73kgUA9IAVYUakRKKaUuOUU2QRpjOpxfJiLBwOvA5sIOyhvoUnNKqUuFVyw1lwtnDu7iTMwut71ege4HKSL1gEXGmIaFF5Jn6VJzSinl/qXmQga/na/9kz68BXBNnAW9m0dtoGphBKKUUuoS5n0jrAWapBMO9AO+KcyAlFJKXXqK7DlIHCfpZFoEjC2kWJRSSl2iinKCvBZIMMZkuDIYb/LDimmeDqHQ1Rix2NMhuMT+ucVvvfyMjPzPDfBmp88Uz68Q/5K+ng5BuUBu12L9E2jqykCUUkpdurxxLdbcJsh3gMGuDEQppdSlyxsTZG6HWP8DHhSRFsAPQGr2jbqSjlJKqQLxvlOQuU6QPYDfbH/XO29b8TxZopRSym2K7CQdZyvpKKWUUsVZjglSRM4CtY0x+90Yj9d4+pERDmUduvakY9eeHohGKaVcxxuWmitqPUjvi9aNJszQZdmUUpeGXr370Kt3H7uyha+/6tYYvDFB5nYWq1JKKeU6hXjDZBHxF5FFIvKtiNyWrTxMRNaLyI8iMuxiIV3sHOQAETlysYMYY/K3yqxSSilFofcg+2MtgzoHWCci7xhjzgA3Am8BbwPfAgsvdJCLJcjcLCdjbC+mlFJKeYPGwBJjzFkR+ReoAuzHuodxOFCaXFyBcbEEWetSnaSjlFLKfbL3IFN+XU3qr2sKcrhQIMb2dwwQZvt7MfAH8CTg7CYcdgp6uyullFKqwLInyNL1u1K6ftdc7Xf41YHOihOBSli9xspAgq18CjAA2AosFZFXjDHHcjq2TtJRSinlcYW81Nx2oLGI+AIVgX9s5eHAEeAUcBIIuVBMF0qQbwHJeWmgUkoplS+FOIsVWAK0xpqosxAYIyKXAZOw1hbfBey52CnEHIdYjTG35bRNKaWU8lbGmDTA6dgrebgzlZ6DzIGupKOUulToSjrO6TnIHEyYMdfhkZvkaIxhzoxJdG15OW2urMGTo+4hNSXlovs989j9vLXAcfWe06dOMeHJh+nU7DJaXFaFcY8/kKvjFbaHe9dn8/iu7HquJ8/f3IgAv/zfIPayiqF8+EBrfpnZm6/HdeHRPvXxxL8NYwwTx4+jQb1aVK8cxfA7byflAv9tF3+6iFbNGhNZrgx9enbjwH770Zlf9u6le5eOVCgbyjUtm7HyqxUuboFzxhgmTRjH5fVrU6NqNPfcfeF2fbZ4Ea1bNCE6Ioz/9eru0K5Mn3z8IYMG9ndV2BdljGHqpGdpdHld6tWoxMh77rxguz7/7FPat25G1eiyXPu/Hvx1IOfRtLuG3cKNg9x/421v+Qz26t2HeS8vsHu4mzfe7koTZCGb/8JzvLvwZR59ehLT5r7Ojzu28djIO3Ksb4zhmw1r+PyT951uHzX8Vnb/sINJM1/iqUkzWPvVUp6fPNZV4Tv1YM963Na+BhMW72bkwm00qh7Oi7c1cVp3/5y+OT4qhgcQHRbAkkfbcjAulcGzN/PcF3sZ1KoaT/Rr4NY2AUydPJH58+Yweep0Fr71Llu3fs+woTc7rbtp4wZuvWkIg4bcyJKlKwgKDqZzx7akplp3fouLi6Nrp3ZUqVKVpctX0aNnL64b0JedO3a4s0kAPDdlIi+/NJdJU6fx+hvvsG3LFu647RandTdt3MDQm29g0OAb+OyL5QQHB9O1c7usdmXa9/vvTJk43h3h52jGc5NY8PI8xk96jldef4vt27Yw/I5bndb9ZtMG7hh6I9cNGsInny0jKDiYXl07OLQL4JOPPuCTj5z/+3O14voZzA9vTJBijN6tKjsRMQC/HjqZ533T09Pp0LgO9z82lutvsk7h/rhzG4N7d2D99t+IjK5oV3/PT7sYel0vkpMSARjz7HPcete5od19v+2lf+eWrN/xG+UjIgFYuvgjvlzyCfPf+iTPH44O477Kc5tK+Ajbp/Rg2tK9vP/NXwBcXS2MpY+1p+kTX3Eowf4Lp1aFIIdj3Nu1DlFhAdw4ZzNP9G9Iu8si6DppHZkfvXaXRfDKnc25+vHlpJ4+m+cY98/N+y//9PR0alevzNhxExh2x50AbN2yhfZtWvL7/n+oVKmSXf0hgwZSqmQp3nznPQDS0tKoVimS2XPnM2jwEObPm8vM56fx674D+Ppaveu+vXtQuXIV5s5/Jc/xZWTk799leno6dWpWYewz47ntdqtd27ZuoUPbVvz2x99UPK9dNw6+jpKlSvHGW+9mtatGlShmzXmJ6wcNAaBurar89++/APTu05ePFn2Wr9gATp/JyNd+6enpNKxTjSfGPsutt1k/OLdv20LXDtfw828HqFjRvl233ng9JUuW4tU33gGsdtWtUZHnZ81j4PXn7v3+z99/0a51UypUiKRmrdq899HifMXnXzLvIyre/hkM8LO+X4wxLs1Cmd+5le5dkq/9/3upH+CaOLUHWYj2/bqHY0eP0K5Tt6yyy69qTGiZcL77Zr1D/eq16vDe56v5Yv1WyleIdNi+4ovFNGnROis5AvQZMIiX317ktvH6ehVDiAj1Z93uw1llP/wdT0LKaa6pV96h/h+xJ+we4UGluKZeBCPe2E6GgdqRwfzwVzzZf5dt2XeMkAA/6kZfcMZ1odqzezexsbF063Fu2LxJ06aEh4ezYd1ah/rr1qy2q+vv70/7jp1Yu3qVtX3tarp06Zb1xQTQrXtP1qxZ5cJWONqzZzdHYmPp1v1crI2bWO1av95Ju9auplv3HlnP/f39ad+hE2vXrM4qW/z5l2zd+RO9ev/PtcFfwC97dnPkSCxdup2LtVHjpoSFh7Np/TqH+uvXrbGr6+/vT7v2HVm/9ly7zp49y9133Mptt99Fo8a5nrdRaIrrZzDfCncWa6HQBFmIjh2JBaBcRIWsMh8fHyIiIzl+1HFJ28DA0tSp14A69Rrg51fSYfuhg/9SrnwFpjzzOJ2bN6Bj0/pMePJhTiQnua4R5ykf4g/AkcS0rDJj4HBCGuVDSl1w31IlfJg1tDFjP/6R48mnADiWfIrosEC7epXKlgYgMtS/MEO/oNhYK+FHRp778eHj40NUVDSxtvcxU0pKCklJSURHR9uVR0dX5Iit7uHDh4k6f3vFihyJjcWdozSxh612VTivXZFR0RyJdd6uqKjz22Vft0GDhjRo0JAyYWF4Sub7VaHCee2KjM56DzKlpKSQnJREVFSUXXlUtvcL4IUZU0lNSWX0k8+4MPKcFdfPYH554xCrVyRIERknIj9cYHt7ETEiUsadr5tXiQnxBAQE2v2CAygdFEx83PE8H+/YkVhWfPEp8cePMW3u6zw9eSabN67liYfuKayQL6pM6ZKknDrD+SN+J0+dITzowgnyjo61+O94Cl/9eCir7LOt/9KufgTXt6xCyRI+VI8ozYybrgbAr4T7Po5xcXEEBjq+V0HBwRw/Zr+wRnx8PAClS9sPHwcHB3PMVjc+Po7SQY7bU1NTnZ73cpX4eOftCg4O5vhx+3Yl2NoVdF7cQU7qelp8fHwO71eQY7sSbO/X+e0KCiLOVnf7ti28OOt5Fix8m5IlHX+cukNx/QzmlzcmSL3MoxCFlgkjNTWFs2fP2n3oTyQlEhJaJs/H8ytZkmo1ajNl9oKs4wUEBjJ0YE+OHY2lXPkKFzlCwSWcPE1gqRL4CHZJMti/BAknT+e4X7B/Ce7vUZcb52y2K//mt6OM+eAHxl9/JbNubULq6bO88OUvXFUtnNiEtByOVvjCw8NJSXF8r5KSEgk7r6eU+fzECft1MxITz9UNCwvnRLLj9pIlSxIQEOCKJjgVFua8XYmJiQ49wMznyefFnZSYSJkynustOhMWFub8/UpMcog18/n570dSklU3NTWVu4bdwrgJU6hTt57rg89Bcf0M5pde5lHMZQ6tHjl8rseUkZHBkdjDTs8xXvR45StQq259u388tepY/6APHfyvgNHmztEkK2lFljn3D0wEKpQJ4EhSzgltUKuq/Hs8he374xy2vfP1AeqPWkrj0Suo8+AXfPjt35Qs4cN/ce67fCVzqC4mJiarLCMjg0MxMURG2g/NBQYGEhISQszBg3blMTEHs+pGRkY6bj94kMioKLf+w88cWj10XrsOH7pAu2LOb1cMkecNT3pa5vt16NB57TrsvF3BISF27y3AoZiDVIiM4uiRWA7s/5Mxjz5EZHhpIsNL89EH77Liy6VEhpdm8zebXN8giu9nsDhxa4K03azyTRE5JCJxIvKhiFTItv1aEdktIidEZLWIVDrvEPVF5Bvb9p9FpF22feuKyAoRiReRFBHZKSK9s20vJSLTReRvETkiIq8DgRSi2vUaULZceTatO3dS/Kdd20lKTKBF6/Z5Pl6jpi3Y+/MPnDlzJqvsl59/BKBq9ZoFjjc3fj2YxNGkNDo0ONdbvbpaGKEBfmz+9WiO+93argYffvuXQ3mHBhWYcVMjfEU4lJDK2QzD/xpXZNufx4mJd98wUIOGDYmIiGBVtuvEtm3dSkJCAu06dHSMu1NnVq48VzctLY2N69fRsXMXa3vHzqxZs8ruvVq1cgWdOnVxYSscNWjQkPIREazKFuv2bVa72rd30q6OnVm18tzs5rS0NDZuWEfHTp3dEm9u1W/QkPLlI1iz6lysO7ZvJTEhgTbtOzjUb9+hk13dtLQ0Nm1cT/uOnYmKrsiWnbv5+vudbPpuB5u+20GPXn1o07Y9m77bwdWNnF/CVNiK62cwvy7pIVaxWrIM8AUGYy0aOxlrnbxtQC1gOHA3UA14EXgWuD3bYRZg3abkFNaaeguBzEzxFtacpiFY9/y6E/hERMoYY04Bz2PdLPMB4Bfb6zwE7CmsNvr5+XHjbcOZOeUZypYrT+mgICY++QjdevcnulJlEuLjSEyIz3Vy695nADOnPMNjI29n6F0jORp7mElPPcKgm2/P15BtfpzJMLyxYT9j+jXgePIpTqSdYeLgK1m26yAH41MpE+hHmdIl+evoucti6kaHULNCMJt+cZyY9NfREwxoXhmD4cPNf1M3OoTH+jZgxMJtbmlPJj8/P4bfO4KxT42hfEQEwcHBjHpwJP2vHUiVKlWIi4sjPi6OmrVqAXDvfSPp2a0zjRo1oUXLVrwwczohoaH0629dYjL4hhuZMmk89959J3fcNZxVK1ewYf06Nm3e4v523XMfzzz9BOXLRxAUHMwjD91P/wEDqeykXcPvHUHvHl1o1KgxLVq2YtbMGYSEhtK3n/svmr8QPz8/7hx+L+OfedJqV1AQjz/yIH37X0vlylWIj4sjPj6OGjWtdt01/D769e7G1Y0a06xFS+bMep6QkFD69O2Pn5+fw9BqSEgoxhi3DrkW189gvnlhJ9ed5yDbAi2AmsaYvwBE5ARWwvMDSgI3GWNigc0icg1Q/7xjjDXGfGHbtwzwYbZtnwBfGWP22LafBW4CokTkGFbyHW6Medu2fQfQPKdgr+12Ta4bdv1Nwxh08zAA7nnocc6cSWfquNGkpabQvksPnpr4PADvvD6fec9PzvU1lv4BAbz/+RqeHf0gdwzpS3BIKH2uHcx9o8bkOrbCMGv5r/j5Cs8MvIKAkr6s+fkwT330AwC3d6zFw73rEz383PVjnRtGcjz5FL8fclzr/sCRkwyb/z1PDmhAvyaV2XswkQfe3M6anw871HW10U88RXp6Oo8/OorUlBR69OzNzNlzAHhp7otMmvAsqenWide27drz9nsfMm3KJKZMGk/TZs1ZvXZj1rmd8PBwVq7ZwEMPjKBPz67UrlOXT5cs5aqrr3Z7ux4fY7Vr9GMPk5KaQo+evXj+Batd8+e9yOSJ4zl5KiOrXW+9+wHTpk5m6uQJNGnanJWrN3jlOatHHn+S9PR0nhz9CKkpKXTr0Yvnnp8NwCvz5/Dc5AnEn7R6T9e0bc/rb73P89OmMG3qRJo0acaXK9d5Xbvc/Rl8/dUFLHzN/avk5EZueoMpf24hZb/7Er7bFgoQkRHAI8aYak62jQNuMMbUyVY2A2hijGkvIu2B9UBFY0yMbXtvYGnmxaEiUgIr4V0BXA10x7oPWHWsm2XuBGoYYw5ke405QBtjzFXZyvK9UIC3y89CAUVBfhYK8Hb5XSjA2+V3oQBvl5+FAryduxcKqDFqeb723z/Tuja0qC8U4Adc6F9HbmZoOK0jIv7AGqxh1urAl8Bd2aqccbbfReJRSil1CXNngtwLVMk+8UZEWonIbqzzkQXRFmgDXGmMecwY8zn2I9r7gXSgU7bXFtt+SimlPEwkfw9Xcuc5yDXAbuAjEXkCKA2MB44BjtcC5M1JrGR/v4isAVphTcYB66aZHwHzgekicgprks4wrN7mXwV8baWUUgXkjZeiuK0HaYw5C3QD/sWaUPMO8AfWrNOC+hZ4BngQWG17nd6215kBBAGPYs2CnQh8hXWJx2OF8NpKKaUKyBt7kHo3j/PoJJ2iRyfpFB06SafocPcknTqP5e/76fdp3YGiP0lHKaWUKjJ0LVallFIe54WnIDVBKqWU8jwfH+/LkJoglVJKeZz2IIuQpx8Z4VDWoWtPOnbt6aS2UkoVXV8uW8ryZUs9GoM3XuahCTIHE2bM9XQISinlFr1696FX7z52ZQtff9VD0XgPTZBKKaU8zgs7kJoglVJKeZ4OsSqllFJOaIJUSimlnMhNfkz87TuSfv/O9cHYaIJUSilVJITWbUlo3ZZ2ZXE783cfydzQBKmUUsrjdIhVKaWUcsIL86MmSKWUUp6nPUillFLKCS/Mj5ogc6JLzSmlLhXesNScN9IEmQNdak4pdanwhqXmdIhVKaWUcsIL86MmSKWUUp7njT1IH08HoJRSSnkj7UHm4I9jyZ4OodDtnzvA0yG4xND3dnk6hEI3qUc9T4fgEhXDAzwdgvJSXtiB1ASplFLK87xxiFUTpFJKKY/zwvyoCVIppZTneWMPUifpKKWUUk5oDzIHc8Y94lDWrH1Xmrfv6oFolFLKdbxhJZ3cdCDj9m4mbu+3rg/GRowxbnuxokBEDMCynw97OpRC16leBU+H4BI6i7Xo0FmsRUeAn5WxjDEuHfvM/M69ZsbX+dr/m0faAK6JU3uQSimlPM4bz0FqglRKKeVxXpgfdZKOUkop5Yz2IJVSSnmcDrEqpZRSTnhhftQEqZRSyvO0B6mUUko54YX5USfpKKWUUs5oD1IppZTH+XhhF1ITZA50qTml1KWiqCw15246xJqDkeNmODxykxyNMbz30nTu7NmCmztcweyxD5GWmuK07ulTabw+Yxy3dW3MdS1qMWbYAP7Y86PTuhtXLGHi/UML0qR8M8Ywcfw4GtSrRfXKUQy/83ZSUpy3CWDxp4to1awxkeXK0KdnNw7s32+3/Ze9e+nepSMVyoZyTctmrPxqhYtb4Jwxht2fvcyXj/2Pzx/oytbXn+XMqdR8H+9s+ml2vPMcyx7pzWcjOrL9rckFOl5+GWOYPX0iHZs3pOXl1Rn94HBSL/B+ZXr60ZG88cpch/J1q5bTp1MLLq9Wjs4tr+CtV1/CE0tUFsfPobe0qVfvPsx7eYHdw91EJF8PV9IEWcg+fOUFlr2/kNseHsvDU+bx2087mPnESKd1P3plFhu+XMxdj09kyuufEl6uAmPvGUJyYrxdvYN//ckH8593R/hOTZ08kfnz5jB56nQWvvUuW7d+z7ChNzutu2njBm69aQiDhtzIkqUrCAoOpnPHtqSmWokiLi6Orp3aUaVKVZYuX0WPnr24bkBfdu7Y4c4mAbD3i9fYt+Yjrhz0IC3unsDxP39my4KnndY9ceRfPrmjhcNj/9efZ9X5bv4Y4g7soemwsTS66TEO7tzAT5/McVdzssx7YSrvvP4yo5+ZxPPzFvLDzm08MuL2HOsbY/h6wxo+++R9h21/7vuN4bdeT6euPflw6RruvO8hpj47hs8//dCVTXCqOH4Oi2Ob8stH8vdwJR1iLURn0tP58sM3uPXBJ2nVqScAD4yfxSM39eLY4RjKRUbb1V/z+UcMHDaClp16ADDy2ecZ3KouP2/7lladewFwW5fGHD18EIBK1Wq6sTWW9PR0Xpk/jwmTptK3X38AXl6wkPZtWvLff/9RqVIlu/rzX5rLtQOv54GHRgFw1dXvUq1SJF98voRBg4fw0Qfv4x8QwPwFr+Hr60uz5s3ZuuV7Fr62gEaNX3FbuzLOpPPH2o+54rqRVGrcEYBmtz/DmolDSYmLJTDcfmH3pJgD+PqVpPPTb9mV+4eWAyDx4J/E/Pg1fZ5fTkAZqwxj+Of7rzDGuG0Ke3p6Ou8ufIVHnhxP1559AZj6wnyu69WBQzH/ERVt/37t/mkXN1/bk+SkRKfH+3zRB9St34AHHnsaEaHB5Vex9btvWP75p/QbOMTl7clUHD+HxbFN3kJE/IF3gWjgVWPMG7ZyX2Ae0Bx4wxjz4oWOoz3IQvT3H7+ScPwoTdp0yiqr3fAqgkPD+HGL/Ur1Z8+epUrNOtS7qklWmX9AIAGlgziVlpZV9sxL7zJ38Xqat+/m+gY4sWf3bmJjY+nWo2dWWZOmTQkPD2fDurUO9detWW1X19/fn/YdO7F29Spr+9rVdOnSDV9f36w63br3ZM2aVS5shaPEg3+SlnScqCuvySoLr96AkqVDid271aF+0qG/CKlYg5Do6naPkqWDAfh36yrK17n6XHIEqrbsQZuHZrv1+q7ff93DsaNHaN/53OfliqubUCYsnO++3uBQv0bNOnz0xRqWb9hGRIVIh+1BwSF0693Prg1ly5W3+4y6Q3H8HBbHNhVEIQ+x9ge+AdoAQ0UkszPYGUgCGgNdbAkzR9qDLETxx44AEFYuIqvMx8eH8PIViD9+1K6ur68vExZ8BFhDXKkpJ1m75EPOpJ/m8qatsupVq10fgKCQUE4mJ7m6CQ5iY63bfkVGnvvy9PHxISoqmtgjsXZ1U1JSSEpKIjravqccHV2R/X/+AcDhw4e54sqr7LdXrMiR2Fi39rTSEo8D4B9aNqtMfHwIKFOOtKTjDvWTDv3F2fTTrJk4lORDf1G6fEXqdL2Bqi17IiKcPH4Y/9By7PrgeQ7u3IDJyCD6qrZcMfA+/AKC3NImgGO296R8hP37FVEhimNHjzjUDyxdmjr1GwDgV7Kkw/a7RozK+vv06dP8uucnVixdzJ33PlTYoV9QcfwcFsc2FUT28P75+jP++XpJQQ7XGFhijDkrIv8CVYD9QDvga2NMhojcdLGDFJkEKSJvAmWMMf0K+bh/AbOMMbMKeqzkxARKBQTY/YIDCCgdRFJCXI77ffzqbN6ZMxWA0c+/SrkKUQUNpdDExcURGBjo0Kag4GCOHztmVxYfb507LV3aPiEEBwdzzFY3Pj6O0kGO21NTU0lNTSUwMLCwm+DUqROJ+Jb0x8fHvl0l/EtzOjnBoX7yob84lRzP1YNHEVguisM/f8vWV59BfHyp2qI7aYnHid2zhSrNu9HiromcPpnIDx++wNbEY7QeMd0tbQJIiI8nIMDx/SodFER8nGPiz61DMf/R5uo6AHTo3J2bbx9eoDjzqjh+DotjmwpCOJchq7YZQNU2A3K135fDmzkrDgVibH/HAGG2v8sB/UXkCWCZMWbShY5dZBJkURAcWoZTqamcPXvW7kN/8kQSQSFlctyv+3U30/iajuzZ8T2zn34Q/4BAu2FaTwoPDyclJcWhTUlJiYSFhdnVzXx+4kSyXXli4rm6YWHhnEh23F6yZEkCAtx3M91SQaGcPZ1GRsZZuySZnnoCv9IhDvVb3TeNEv6B+PlbXzJlazTk5LEY9q35kKotuuNTwo/gyCo0u/PZrOP5lgpgw3N3k5Z43K6n6kplwsJITXV8v5KTkggJLZPv45YrX4HlG7fz798HeH7yOB4deScz5r5WCBHnTnH8HBbHNhVEIU+4SQQqYfUaKwMJtvJUYA9wN7BYROoYY37PMaZCDekSlzm0Gnf0cFZZRkYGcUdiCc827AqQciKZX3/cwZn0dELDylLrsivoe/NdtOjUgw1fLnZr3BdSwXZeKiYmJqssIyODQzExREba93QDAwMJCQkh5uBBu/KYmINZdSMjIx23HzxIZFSUW4eAMhNWWvy5oW+TkUFawlECQss51A8oUy4rOWYKq1qPlOOHs44XEl3TLtmGRtcA4OTxw7hLuQhrclHs4UNZZRkZGRyJPeT0HOPF7Pn5B2IPx+Dn50edepfRqVsvnp44nS8/X0TKyZOFFvfFFMfPYXFskxfZDjS2nWOsCPxjK/8BSDDGpAPJXCQHek2CFBEjIh1E5E0ROSoiB0XE4USHiNwrIvtEJFlEPhWRkGzbbhWRvSKSJiLHbNsjs22vIyIrRSRRRPaIiPP51PlUtVY9yoSXY8fX67LKfv95FyeTE7mi+TV2deOPH+WRm3rxz377Hy+pJ086DLl4UoOGDYmIiGBVtuuptm3dSkJCAu06dHSo36FTZ1auPFc3LS2NjevX0bFzF2t7x86sWbOKM2fOZNVZtXIFnTp1cWErHIVWrEmpkHAO/bw5q+z4/t2cTkkm4rKmdnXTU0+wfPQA/t22xq48MeYAIdHVAShX+yri//6VjLPn2hX/968ABFeo4qpmOKhTrwFly0Wwae3KrLIfd24nKTGBlte0z/Pxnh0zilfnzbIrO3EiGYzBt4T7BqCK4+ewOLapIAp5ks4SoDXWRJ2FwBgRuQz4GLhFRHYCh4wxv14oJm8bYp0NvAa8AjwAzBSR1caY3bbtHYBTwE1AI2CWrd4EW+PfAJ4HPsHqVk8HJgPDRCQY2AT8CvQGStv2t59LXQAl/PzoNWQYb82eTGh4OQJKB/HKlCdo3aU3EVGVSE6MJzkxgegq1YmsVJUqNevw0oTHueWBMQQFh7L967Vs3biKia9+XFghFZifnx/D7x3B2KfGUD4iguDgYEY9OJL+1w6kSpUqxMXFER8XR81atQC4976R9OzWmUaNmtCiZStemDmdkNBQ+vW3zicMvuFGpkwaz71338kddw1n1coVbFi/jk2bt7i1XT4l/KjdaRA/fTKXUsHh+AUEsvPdaVRu0pnSZaM4dSKR0ycTCa5QBb+AIMKq1Wfne9NITz1BWNV6xO7dxoGNn9H2YevC+spNO/PzJ3PYsuBp6nS7kbSEY+x8bxo121+bNdPVHfz8/Lj59ruZMfkZypYrT+mgYJ598mG69+lPdKXKJMTHkZAQT7XqubtkqFO3XsybOZX6DS6n4RVX889f+5n8zGj6DhxCqVKlXNyac4rj57A4tqkgCrOTa4xJAwbmsLl3bo/jbQlyY+Z1KSLyJzAIqAVkJsg04FZb47eIyP+AzH/pGcBjwAvGmLPAVhHpDtS2bb8BKykOMMbE2V7jJmCbs0AeHJT7JeW6D7yZ7tdZndHBdz/E2TPpvDb9GU6lpdKsXRfuHmOdB/7ivdf4YP7zLPv5ML6+vjw5+01enzGOqaPuJD39NFVr1WPsnLe5omnrXL+2O4x+4inS09N5/NFRpKak0KNnb2bOti6Af2nui0ya8Cyp6dbKKm3bteft9z5k2pRJTJk0nqbNmrN67cascyDh4eGsXLOBhx4YQZ+eXaldpy6fLlnKVVdf7fZ2Xfa/O8g4e4YfPpzJ2VNpRF3VhkY3PQbAvtUfsOfzBQx6cycATYY+yU8fz+bnT1/iTNpJQirW5JoHZ1HhMmuCQImS/nR8ciE73p7Cxhn3UTIwmKote9Kg711ub9d9D43mTHo6k58ZTWpqCh269OCZyTMBeOu1l5gzYzJ/xF58ZR2AYcPvJ+XkCV6cMYmjR2KJiq5E3+uGMHyk41KMrlYcP4fubtPrry5g4WvuXyUnN7xxLVbxxJJRzoiIAW40xrx/Xll/Y8wS2yzWaGNM12zbFwEnjDFDbc/LY10AegXQDOgOfG+MaS8is4GrjDHtsu0vWNfEPJ05i9X2miz72X3njdylU70KF69UBA19b5enQyh0k3rU83QILlEx3PsniyhLgJ+VsIwxLs1cmd+5/V/bnq/9P7vDupbcFXF6Ww/yYj9rc9wuIq2AL4CdwApgBnAIqG+rciaHXc/mMUallFKFzAs7kF6XIAtiBLDGGDM4s0BEsi+C+gtwl4iEZw6xYvU0Q90Yo1JKKSe8caat18xiLQQngbYi0k1EOtqGVAcAlUWkBvAB1rTeT0WkjYh0wVqrLznnQyqllHIHkfw9XKk4JcixwI/AYqyZsKewzkP6Ao8bY05iLTN0GlgKvIg1DOu48KZSSim38hHJ18OVvGaI1dkJ1uxlmRNxzts+MNvfh4AeTg5dLVudfcD5q36/hVJKKXUer0mQSimlLl3edwZSE6RSSikv4I2TdDRBKqWU8rhCXqy8UGiCVEop5XHagyxC5oxzXEqrWfuuNG+f+yXolFKqKPhy2VKWL1vq6TAu6r+dGzm4a6PbXs9rlprzFrrUXNGjS80VHbrUXNHh7qXmbnr3h3zt/+5NVwGXxlJzSimlLkE6xKqUUko5oZN0lFJKKSe8sQdZnJaaU0oppQqN9iCVUkp5nPf1HzVBKqWU8gKuXng8PzRBKqWU8jgvzI+aIJVSSnmeN07S0QSZA11JRyl1qSgqK+m4mybIHIwcN8PTISillFv06t2HXr372JUtfP1Vt8bghR1ITZBKKaU8TyfpKKWUUk54YX7UBKmUUsrzdJJOEVIpJNDTIahcmtWvgadDKHQdpq73dAgusWtCN0+HoFSuaYJUSinlcblZ9/TAtvUc2L7B1aFk0QSplFLK43IzxFqjWUdqNOtoV7Z39SeuCkkTpFJKKc/zxttd6d08lFJKKSe0B6mUUsrjvLEHqQkyB+NH3+9Q1q5zD9p17uGBaJRSynW8Yak5vcyjCBk79UVPh6CUUm7hDUvNaQ9SKaWUcsILO5A6SUcppZRyRnuQSimlPE4XK1dKKaWc8MbhTE2QSimlPM4LO5CaIJVSSnmeNw6xemOvVimllPI47UEqpZTyOC/sQGqCVEop5XneuFCADrHmYPzo+x0eG9esuOh+xhjmvzCZPm2vpHOT2ox79D5SU1NyrPvpB28yoFNTWtSLZGDXFnz+yXsYY7LqxB6O4f5h19P2iipc370VXyx6r9DamFvGGCaOH0eDerWoXjmK4XfeTkqK8zYBLP50Ea2aNSayXBn69OzGgf377bb/sncv3bt0pELZUK5p2YyVX138v6srGGOYMWUCLa+uz5V1qzJqxN0XbNeyzxfTtV0L6laJYPCAXvz9l327Nq5fQ4+OralZMZwOrRrx2aKPXN2EHN3XqSZfPdKGTWPaM3FAA/z98v9PvXaFIF4b1pjvx3Zk/eh2PNPvMkL83f/bujh+Dr2lTV8uW8p9w++ye7ibj8hFH/u+X8eyWU/bPVxJsn8ZKxARA/DD30n52n/B7Od4b+FLPPPcXEoHBfPcuMeoXrMOz7/yrkPd9au+5JHhN/Hw05Np1LQVO7d9ywuTn2bcc3PpNWAwZ86c4douzahctQZ3jHiUX/f8yIzxo5nx8ru079Izz7HVjQ7OV5umTJrA3Bdn8dIrrxESEsLDD91Pnbr1+PDjTx3qbtq4gV7duzBx8nM0b9GS2bOeZ+uW79n9yz4CAgKIi4vjygZ16dGzN3fcNZzVq77iuSmT2PD1dzRq3Dhf8SWcPJ2v/WZOm8xr8+cw48WXCQ4J5qnHR1Grdl1ef8cxsX379UYG9e/Jk+Mm0aRZC16eO4ud27eyecceAgIC+O2XvXRp24wRDz5Kj9592b71O54e/TDvLfqCdh065zm2DlPX56tNAPd0rMEtravy9Kd7OHHqDE/2qc/+oyd54L0fHOr+MD7n2HrO/Ib4k+msePgafv4vkVc3HiAkwI9He9ThWPIpbl+4I8+x7ZrQLc/7ZPL2z2Fxa1OAn9WlM8a4tG+X+Z07fvW+fO0/tkttwDVxaoI8T0ESZHp6Ot1b1ufeh5/i2iFDAfh51zZu6d+Zr77bS4Woinb17x92PQGBgTw3982ssunjR/PTzq28s2Qdm9au4JF7bmHdjj8JCg4BYNKTD/HfPweY/86SPMeXnwSZnp5O7eqVGTtuAsPuuBOArVu20L5NS37f/w+VKlWyqz9k0EBKlSzFm+9YPd20tDSqVYpk9tz5DBo8hPnz5jLz+Wn8uu8Avr6+APTt3YPKlaswd/4reY4P8pcg09PTadygJo89+Qw33Xo7ADu3b6V3l7Zs3/0H0RXt23XHLYMpWaoUL736Vla7rqxThakz59B/4CCmTXqW1V99yeqvt2btc/vNgyhdOogXX349z/HlN0GW8BHWjW7HnNV/8Mm2/wC4onIoHwxvTsfnNhKbdMqufvXypR2OcXvbakSG+nPXGzvodFkFpg+6nDaTN5CcdgaAupHBLHmgFT1nfsOBoyfzFF9+E2RR+BwWtzZpgtQh1kL15297OX70CNd06JpV1uDKxoSWCWPL5g0O9UWExs2vsSsLCy/H4ZiDAHz39XqubtoqKzkCXNOhK1s3b+Ts2bOuacR59uzeTWxsLN16nOuxNmnalPDwcDasW+tQf92a1XZ1/f39ad+xE2tXr7K2r11Nly7dsv4BA3Tr3pM1a1a5sBWOft27h6NHYunUpXtW2VWNmlAmLJxvNjomp00b1trV9ff355p2Hdi4bg0AaWmplA6y/wESHByS4/C6q9SuEET54FJs/O1oVtnP/yWSmJpOi1plHeofOHrS7hEW6EfLmmV57KOfyTAQ5O/L9r/is5IjQJztB0lUqL/rG2RTHD+HxbFNBeEj+Xu4NCbXHv7ScuxoLADlylfIKvPx8aF8hSiOHzvqUH/26x9x/c13ZD1PSoxn+Wcf0fAqazjk+NEjlK8QabdPRIUozp49S2J8nCua4CA29jAAkZHn4vDx8SEqKprYI7F2dVNSUkhKSiI6OtquPDq6IkdsdQ8fPkzU+dsrVuRIbCzuHM04csRqV0QF+3ZFRkZx9Khju5KTkoiMjLIrj4qKzqrbtUdvdu3YyrLPF5Oens53m79m2ReL6fW//i5uib1ywaUAOJZ8rqdoDBxJOkW5oJIX3LdkCR+mXHc5k5f9mpUEP9sRwx3nDaXe1KoKp89k8Ouh5EKOPmfF8XNYHNtUEJLP/7mSzmItRIkJ8fgHBNr9ggMILB1EQtzxC+77086tjHtsBEdiDzF9/jtZx6tavab9sYKCAIiPO054ufKFGL1zcXFxBAY6tikoOJjjx47ZlcXHxwNQunSQXXlwcDDHbHXj4+MoHeS4PTU1ldTUVAIDAwu7CU4lxMcTkEO74o7bv1eJCbZ2nRd36Wx1W7S6hrtHPMidtw7J2j5w0I30u/Z6V4Sfo9BAP1JOnyHjvO/Dk6fOEBZ44QR5S6uqxCSksnbvEafbywT6MapbHa5rVokXVv6elUTdoTh+DotjmwpCZ7EWc6FlwkhLTXEY/jyRnERIaBmn+5xITuLZx0ZwS//O+AcE8NbiNdSsUy/reCknTzjUB3I8XmELDw8nJcWxTUlJiYSFhdmVZT4/ccK+Z5GYeK5uWFg4J5Idt5csWZKAgIDCDj9HZcLCSM2hXaFl7NuV+fz8uJOTEgktUwaA995eyHtvLWTGi/NZueF75r/+Dt9/+w0zpkxwXSOcSExJJ7BkCYcvm2D/EiSmpue4X1CpEtzdoQazV/3hdHvvq6JY8XAbul9Rgac/3c2CDQcKM+yLKo6fw+LYpoLwxiFW7UEWosyh1aOxh4iMtk6wZ2RkcDT2EOUiIh3qJ8QfZ+i1XYk9FMPYqXPoN+hmfHzO/WYpWz6CP3/7xW6fI4cP4ePj45beI0AF2xBkTEwMlStXBqw2HYqJcRhyDAwMJCQkhJiDB+3KY2IOZtWNjIx03H7wIJFRUW69o3iE7f04fCiGipXOtSv20KGsNmcKDAwkOCSEQ4di7MoPxcRQoYLVrpdmP88DDz/OjbcMA+CKq64mMDCQ++4ayshRj1GqVClXNwk4N7QaEeLP4cQ0wLoAu3yIP0eTT+W434AmFTkYn8oP/yQ4bHukRx1ub1udpT/EMHXZb27tOWYqjp/D4tim4uaS6EGKyFci4jCNS0S2isjUwnqdmnUvI7xceb7ZsDqrbPcP20lOSqRZq7YO9WeMH0NSYgJvf7aGAUNutUuOAC2u6cAPO74nOSkxq2zzhtU0bdXOYVjGVRo0bEhERASrsl1PtW3rVhISEmjXoaND/Q6dOrNy5bm6aWlpbFy/jo6du1jbO3ZmzZpVnDlzbtLHqpUr6NSpiwtb4ajeZQ0oVz6CdWtWZpXt2rGNxMQEWrdt71C/TbuOrM9WNy0tjc1fb6Bth06AdY7o/PdPREhLTXXbhCqAfbEnOJZ8ijZ1ymWVXVEplBD/Emz5M+fz1kNaVGbxjoMO5c1rhHN72+pMX/4bj330s0eSIxTPz2FxbFNBiEi+Hq50qfQgXwbeE5HHjTEJACLSFGgCDCqsF/Hz82PwLXcx57lxhJctT+nSQTz3zKN07tmPqIqVSUyIIzEhnirVanL27FlWf7mEAUNupZS/P//89WfWcXx9S1CxclVat+9CVMXKjLn/du4c+Si/7P6RJR+9zcwF7lsswM/Pj+H3jmDsU2MoHxFBcHAwox4cSf9rB1KlShXi4uKIj4ujZq1aANx730h6dutMo0ZNaNGyFS/MnE5IaCj9+g8AYPANNzJl0njuvftO7rhrOKtWrmDD+nVs2rzFbW3KbNdtd97DlGefply58gQFB/PkYw/Ru+8AKlWuQnx8HAnxcVSvYbXr9rvu5fp+PbjiqkY0bd6Sl16cSXBIKD379ANg0I23MPO5SYSEluHyK67izz/2MWHsaPoOuN6t537OZBje//4fRnWvTdzJ05y0XQe5cncshxLTCA3wIzTQj3+On5tdW7tCENXKlebbfcccjtfzykgOJ6ax7pcjVClr347YxDROnclweZugeH4Oi2ObCsIbz0FeEtdBikgJ4C9gmjHmRVvZG0AFY0zP8+oWaKEAYwzzZ07iy88+Ji0thbYdu/P4+On4+wcw/4XJvDJrKj/8nUTcsaN0bFzT6TGiKlVhxebdgLWSzsQxD/DD9i1ERlfk1rvvp/eAIU73u5j8LhRgjGHCs8/wwfvvkpqSQo+evZk5ew4BAQFMHD+OSROeJTX93Odo8aeLmDZlEn/9dYCmzZozZ97LVKtePWv73j17eOiBEfywaye169TlmWcn0KVr/i8gz+9CAcYYpk8ez6cfv09qaipduvVk4rQXCAgIYMaUCTz/3EQOJZwbllz2+WJmz5jKP3//xdVNmjJt5lyqVLPadebMGV6bP4f33nmD//75mwqRUfQbOIj7Rz2erwRZkIUCAEZ2rkWfq6Pw9/Nl469HmfjFL5w6k8F9nWoyonMt6o851xu+o211bmtTjdaTHF/zlaGNaFvX+XD+LQu2su1AfJ7iKshCAd7+OSxubXL3dZDPb/zzYlWderid9T2qCwUUgIiMBW4A6gPhwH/A9caYpefVMwCXXX51ro894IahDLzhtsIL1kXymyC9XX4TpDcraIL0VgVJkKrgXn91AQtfW5Crujt3Wpf3uCtBvrBp/8WqOvVQ2xqAa+K8VIZYAV4DxgIdgauBo8DynCq/v2yjm8JSSin3uP3Ou7j9ztyts5rZg7yUXRKTdACMMTHA58BI4B5ggTHGfbMnlFJK5agwL/MQEX8RWSQi34qIw/CeiFwvIqMvGlPBm1WkzAf6AlWAvC+QqZRSyiVE8vfIQX/gG6ANMNQ2D8X2OuKHNZp4UZfSECvAOmA/8KMx5pCng1FKKWXxKdxl4xoDS4wxZ0XkX6xOUeZJzruAlTnumc0llSCNMRm2/1j/eDoWpZRSzn3z+fts/uLDghwiFMhc2SMGCAMQkWCgAzAPaH6xg1xSCVIppZR3yj5c2qbfDbTpd0Ou9ruvdXVnxYlAJaxeY2UgwVY+CngB8MvNsTVBKqWU8rhCXihgO9BYRDYDFTk3algXqwdZBggRka3GmHU5HeSSS5DGmPaejkEppZQ9n8JdNm4J8C5wPfAKMEZEFhljbgAQkfZAiwslR7gEE6RSSinvU5j50RiTBgy8wPYNwIaLHUcTZA7Gj77foaxd5x6069zDA9EopZTrfLlsKcuXLb14xUvMJbPUXG4VdC1Wb6ZLzRUdutSc8jR3r8X62pa/87X/Hc2rArrUnFJKqWLKG29ZqQlSKaWUx3njsm6aIJVSSnmcq29+nB/emLSVUkopj9MepFJKKY/zvv6jJkillFJeoJAXCigUmiCVUkp5nPelR02QSimlvIAXdiB1ko5SSinljPYgc6BLzSmlLhXesNRcbi7z2LFxNTs3rXZDNBZdau48utRc0aNLzRUdutRc0eHupeY+3PlfvvYf3KgSoEvNKaWUKqa8caEATZBKKaU8zvvSo07SUUoppZzSHmQOSvh64++ZgjmckObpEFwisoy/p0ModMX1XF2z8Ws8HYJLbB3b2dMhFHk6xKqUUko54Y3DmZoglVJKeZz2IJVSSiknvC89emevVimllPI47UEqpZTyOC8cYdUEmZNxj410KGvfpQftu/T0QDRKKeU63rDUnI8XDrLqUnPnyVz2aPd/yZ4OpdCVLlU8fw8Vx8s8iiu9zKPocPdSc0t/Ppyv/ftcHgm4Jk49B6mUUko5UTy7FEoppYoU8cIhVk2QSimlPE4n6SillFJO5GaSzvcbVrJlwyo3RGPRBKmUUsrjctODbNmhGy072K9T/NWid1wUkU7SUUoppZzSHqRSSimP03OQSimllBM6i1UppZRywsf78qOeg8zJuMdGOjw2rF5+wX2MMcx7fjI9Wl9J+0a1ePrhe0lNTbnoaz07+gHeeW1eYYVe6IwxzJo2kfZNG9CsQXUef2A4qSkXb9eTj4xk4ctzHMp/+2UPN1/XmytrRdHyipo8+chIEhPiXRH6BRljmDh+HA3q1aJ65SiG33k7KRdo1+JPF9GqWWMiy5WhT89uHNi/3277L3v30r1LRyqUDeWals1Y+dUKF7fAueLaLoB7OtRg2QOtWPtoG8b1rY+/X/6/wiKCSzHnxiv5enQ7Prm3Of+7KqoQI80db3mvvly2lPuG32X3cDfJ5/9cSRNkDsZNm+PwuNg6rK/Mnsb7b7zMw09NYOqLr/HTrm2Muf/OHOsbY9i8cS1LF33gsO2ZR0fQqGY5p49XZk8rcPvyYs7Mqbz12nzGjJvMC/MXsmvHVkbdOyzH+sYYNq1fw+KP33PYlnLyJLde34eAgADe/Ohzpr4wnx1bv+O+O27C3cseTp08kfnz5jB56nQWvvUuW7d+z7ChNzutu2njBm69aQiDhtzIkqUrCAoOpnPHtqSmpgIQFxdH107tqFKlKkuXr6JHz15cN6AvO3fscGeTgOLbrrvaVWdI80rMXLWPJz7dzRWVQ5k8oKHTutue7pDjIyrUH18f4bXbGiEijHjvBxZtP8gzfevTvm45t7bJW96rXr37MO/lBXYPpWuxOsjvWqzp6el0blaPkY88xcAbbwPgp53buLFvJ1Zv+YXI6Ip29ff+/AO3D+pNclIiAI+Pm8rNd9yXtf1o7GGSkxPt9jnwx+88+dBw3l+6jhq16ua5bflZizU9PZ1WV9Zm1JixDLnZSoq7dmzl2h7t2fzD70RFV7Krv/vHXdwwoEdWu56eMI1hw88t/L5i6Wc8eM9tbNvzFyGhZQDYu/snenVozppvf6Bm7by3Kz9rsaanp1O7emXGjpvAsDusHzFbt2yhfZuW/L7/HypVsm/XkEEDKVWyFG++YyX9tLQ0qlWKZPbc+QwaPIT58+Yy8/lp/LrvAL6+vgD07d2DypWrMHf+K3mOL7+8vV35XYu1hI+w6uFrmLfuTz7dEQPA5ZVCeOeOpnSb+Q2xSafs6lcrF+hwjNuuqUaFkFLc+84urqldjucHXU77aZs4eeosAE/1rkfF8ADueXtXnuPLz1qs3v5euXst1nW/HsvX/h3rWT9qdC1WL/bHb3s5fvQIbTqeu0an4VWNCS0TxpbNGxzqV6tZm7cXr+KzNVsoXyHSYXv5CpHUqFU361GxUlVemjmFpybNzFdyzK/ff9nDsaOxdOh8rl1XXt2EMmHhfLtpg0P9GrXq8MmytXy1aTsRTtp1IjmZZi2vyUqOAGXLWR/wmIP/FXr8OdmzezexsbF063FuVKBJ06aEh4ezYd1ah/rr1qy2q+vv70/7jp1Yu9q6aHnd2tV06dIt64sJoFv3nqxZ476LmqH4tqtWRBDlgkvx9e/Hs8p2H0wiMTWd5jXCHer/dSzF7hEW6EfzGmE88eluMgy0rBnOzn8SspIjwNf7jtG8epjbzoUV1/cqv3SItRg7diQWgHIRFbLKfHx8KF8hiuNHjzjUDwwsTe16l1G73mX4+ZW86PFfnv0cFStXofeAQYUXdC4ctbWrfMS5ZOfj40NEZBRHj8Y61A8sXZq69RtQt34D/Eo6tuu6G27hnU+WZT03xvDWq/MpWbIk9Rtc7oIWOBcba905IDLSvl1RUdHEHrFvV0pKCklJSURHR9uVR0dX5Iit7uHDh4k6f3vFihyJjXXr0HFxbVe5YOuzdOzEuZ6iMXA0+RRlgy7876dkCR8mDmjAc8t/J+5kOgBlg0py9Lxe55GkU5Tw9aFMoF8hR+9ccX2v8stH8vdwJZ3FWkgSE+IJCAi0+/UGUDooiPi44znslTsx//3D+2+8wsfLNxXoOPmRkBBHQKBju4JKF7xd8XHHeW7C03z07hs88sSzlCsfUaDj5UVcXByBztoVHMzxY/ZDPfHx1gSi0qWD7MqDg4M5ZqsbHx9H6SDH7ampqaSmphIY6Djk5wrFtV0hAX6knj5Lxnnf8ydPn71oQruxRWUOJaSx/tejWWWhgX78fcx+MszJU2cAKBNYMiuRulJxfa/yyxsv87gke5Ai0l5ExhXmMUPLhJGamsLZs2ftyk8kJxFSpkyBjj3v+Un0+N+1VK1Rq0DHyY8yZcJJTXFsV3JyEqGhYfk6pjGGJYs+oEPzy1n++adMmfkS9z74aGGEm2vh4eGkOGlXUlIiYWH27cp8fuKE/XnpxMRzdcPCwjmR7Li9ZMmSBAQEFHb4OSqu7UpKTSegpK9DjyG4VAmSUs/kuF9QKV/ubFudOWv/tCtPTEl3OCcf5F8i67Xcobi+V8XJJZkgXSFzaPVI7KGssoyMDI4cPmw3PJlXx44eYfnni7juppxnjbpSeVu7Yg/HZJVlZGQQe/iQ03OMuTHl2Sd46J5htO/UlXVbfmbwzbchbl5Go4It9pgY+3YdiokhMtJ+un9gYCAhISHEHDxoVx4TczCrbmRkpOP2gweJjIpya9uKa7uOJZ8GoHxwqawyEev50eRTOe1Gv0bRxCSk8uO/9hPejp84TURIKbuyiBB/zmYY4k6eLsTIc1Zc36v8Esnfw5UuqQQpIs1E5FfgbWCEiPwqIiMK49i16l5GeLnyfLPu3Anxn3dtJzkpgeat2+X7uIs/fIsaterS4IqrCyPMPKtTvwFly0ewYe25dv24cxtJiQm0bJP3dn379QZenTeL0c9M4oX5b7h1WDW7Bg0bEhERwaps14lt27qVhIQE2nXo6FC/Q6fOrFx5rm5aWhob16+jY+cu1vaOnVmzZhVnzpzrzaxauYJOnbq4sBWOimu7/jhyguMnTnFN7XOXYVxeMYRg/xJsPRCX437XN63Ekl0xDuXf74/jqsqhBJU6N7x5Te2ybD0Q5zCM6yrF9b3KL8nnw5WKzTlIESkFjAcGApHADuAJY8w3mXWMMVuBeiLSHmhvjBlXWK/v5+fHDUPvZtbUcYSXK0/poGAmP/0IXXv1I6piZRLj40hMiKdK9Zp5Ou6aFUtp2aZDYYWZZ35+ftxy+3CmTxxLOVu7xo0ZRY8+/alYqQoJ8XEkxMdTrUbu2rX0s0+IjIqmS48+/H3A/iLnyKho/N00FOTn58fwe0cw9qkxlI+IIDg4mFEPjqT/tQOpUqUKcXFxxMfFUbOWNax9730j6dmtM40aNaFFy1a8MHM6IaGh9Os/AIDBN9zIlEnjuffuO7njruGsWrmCDevXsWnzFre0p7i360yG4YMt//FAl5rEnTxNyukzjO5Zl9V7j3A48RQhASUIDfDj37jUrH1qRZSmWrnSfPeHYwL9Zt9xDiWmMfW6y1mw8QD1o4Lp3yiahz74yW1tKq7vVX75eGEvt9gkSOBVoAFwL3AcuA1YKyJXGWN+yevBru/ZNtd1r7thKNfdNIy7H3iMM+npTB//BKmpKbTr1J0nJswA4N2FLzP/hSl5ur7y2JFY9v60i3tHjclr+IVq5KjRnElPZ+LYx0lNSaVj1x48O2UmAG+++hKzp0/iwNHUixzFcjjmIIcPxdCpxRUO2z5YspIWrXP/372gRj/xFOnp6Tz+6ChSU1Lo0bM3M2dbK/+8NPdFJk14ltR0qzvRtl173n7vQ6ZNmcSUSeNp2qw5q9duzDq3Ex4ezso1G3jogRH06dmV2nXq8umSpVx1tft7/sW1XQs2HqCEr/Bo99r4+/my6fdjTPnyNwBubFGFezrU4Iqx566zbFOnHHEnT/Pn0ZMOxzqbYbjjjZ08/b96vHTTVRxOTOOZJXvZ9Hv+rsXLL3e/V6+/uoCFr3nnIgC5SY9fr13B5vUrXR5LpmKxUICIVAP2A9WNMX/bygT4FlhqjJlsK2sOnL+8y4vGmBezHStfCwUUBflZKKAoyM9CAcoz8rtQgLfLz0IB3s7dCwV8ty9/y022rG1NUnJFnMXlG7Mh1g+Q3847GV0S+DXziTFmC+D+qaBKKaUuzPtGWItNgiwBZACNgbPnbXMcX1FKKeVVvPE6yOKSIH/BmpEbZOslIiJ+wPPAUuBfD8amlFLqIrxwjk7xuMzDGPMb8Dnwtoj0EpHWwALgVuB3jwanlFLqorzxMo9ikSBtbgJWAy8BK4FqQJfMSTtKKaVUXhSXIVaMMSeAEbaHUkqposQLh1iLTYJUSilVdOkkHaWUUsoJb5ykowlSKaWUx3lhftQEmZNxj410KGvfpQftu/R0UlsppYquL5ctZfmypZ4Ow+sUi6XmCpMuNVf06FJzRYcuNVd0uHupuR1/JV6sqlONq4UCutScUkqpYsobJ+kUp+sglVJKFVGFecNkEfEXkUUi8q2I3JatvLSIrBaRH0XkVbnInaQ1QSqllCpu+gPfAG2AoSKSOVp6LbAeuAowQOsLHUQTpFJKKY8r5KXmGgPbjTFnsdbirmIr3wd8YKzJN4cvFpOeg1RKKeV52bLdovfeYPH7bxbkaKFAjO3vGCAMwBjznVhuwuo9TrjQQTRBKqWU8rjsk3Suu3EY1904LFf7XVkl2FlxIlAJ2A9UBhIAbOccnwMqAP2MMekXOrYOsSqllPK4wpykA2wHGouIL1AR+MdW3t/2/0ONMRe9lk8TpFJKqeJmCdYQ6jfAQmCMiFwGNAM6AetFZIOIXHOhg+gQq1JKKY8rzKsgjTFpwEAnm0bbHrmiCTIHutScUupS4RVLzXnfOgG61Nz5dKm5okeXmis6dKm5osPdS83t/u9EvvZvWCkI0KXmlFJKFVPeeLsrnaSjlFJKOaE9SKWUUh7nhR1ITZA5CSyG5+uOJZ/ydAguoecgi47ieK4OoPOsrz0dQtHnhRmy+GUBpZRSRY433u5KE6RSSimP00k6SimlVBGhPUillFIe54UdSE2QSimlvIAXZkhNkDl4YtR9DmWduvWkU7deHohGKaVcJ27vZuL2fuvRGHIzSWfdqi9Zt2q5G6Kx6FJz58lc9mj/0VRPh1LojhfTyzwaVg71dAjqElccL/PY/GhbwH1Lze2LTcnX/rUrBAKuiVMn6SillFJO6BCrUkopj/PCU5CaIJVSSnkBL8yQmiCVUkp5nK6ko5RSSjmhK+kopZRSRYT2IJVSSnmcF3YgNUEqpZTyAl6YITVB5kBX0lFKXSqKyko67qbnIHMweeY8h8fFkqMxhlnTJtKhaQOaN6jO4w8MJzXl4qtDPPnISBa+PMeh/Ldf9nDLdb25qlYUra6oyZOPjCQxIT7fbcovYwwLZk2hf4er6d68LhMeH0FaqvN2GWP47MO3uL5rc9o0iGZw91YsXfQe2Vds+uO3vYy4pT8dr6pKr1aXMeXJh0hKTHBTa+xjnTh+HA3q1aJ65SiG33k7KRd4vxZ/uohWzRoTWa4MfXp248D+/Xbbf9m7l+5dOlKhbCjXtGzGyq9WuLgFzmm7LEWhXcYY/lm1kB1Th7B1fD/2fTyVs6fTcqx/MuZP9rz6MN8/3ZOd027i2I/rCiWO8MtaU2vgo3YPpQmyUM2dOZW3X5vP6HGTmTl/IT/s2Mqoe4flWN8Yw9fr1/DZx+85bEs5eZKh1/chICCANz/6nCkvzGfH1u8YccdNuHt5wNfnTufjtxdw/+jxjJ+5gJ9/2M7YUXc7rbtpzXKee3oUA264jdc+XkG/wbcw5amHWPH5xwCkppzk/qHX4h8QwItvfsqTU17kxx1bGDNiqNvbNXXyRObPm8PkqdNZ+Na7bN36PcOG3uy07qaNG7j1piEMGnIjS5auICg4mM4d25Kaai1JGBcXR9dO7ahSpSpLl6+iR89eXDegLzt37HBnkwBtFxSddv235m0ObV5MtV73UGfI0yT/s5d9H0x0WjfteAy7X3mA0tG1aHDHdMIbtOa398aT/O8vWXV2TBnMt2M62z32fTTFXc0pEJH8PVxJh1gLSXp6Ou+8/gqPPT2Bbr36AvDc7Je5tkd7DsX8R1R0Jbv6u3/cxY0DepCclOj0eBvXrSI+Po7pc14lJLQMABEVIunVoTn7//idmrXrurQ9mc6kp/PJO69x32Pj6NCtDwBjn5vLsGu7EHvoIBWiKtrVX/LR23Ts0ZfBQ4cDULfBlcT8+zefvP0qPfsN4ruNa0iMj+OZ6S8RHGK1q1xEBW7s1Ya/9++jWs06bmlXeno6r8yfx4RJU+nbrz8ALy9YSPs2Lfnvv/+oVMn+/Zr/0lyuHXg9Dzw0CoCrrn6XapUi+eLzJQwaPISPPngf/4AA5i94DV9fX5o1b87WLd+z8LUFNGr8ilvapO0qWu3KOHuGQ99+RtWed1P2cmvd09rXj+anufdwKuEIpcpE2NU/9N0Sgqs1pFqvewAIqnIZp5OOc+LfXwmuXJ8zaSmkxcVQ79ZJBERUydrPt1Rpt7SnoLxvgFV7kIXm91/2cOxoLO07d8squ+LqJpQJC+fbTRsc6teoVYePl61lxabtRFSIdNh+IjmZZi2vyUqOAOHlygEQc/C/Qo8/J3/+vpe4Y0do3b5LVtllVzQipEwY277d6FBfRGjUrLVdWVh4OWIPHQTgxIlkrm7WKis5Zm4HiI056IIWOLdn925iY2Pp1qNnVlmTpk0JDw9nw7q1DvXXrVltV9ff35/2HTuxdvUqa/va1XTp0g1fX9+sOt2692TNmlUubIUjbZelKLQr5fB+0k/EEVavRVZZUKV6lAgIIfEPx57s8R/XU+6KDlnPRYQ6Q54iqpX1gyH16D8gQpm6zQiMqJr1KBVazvWNKQTagyzGjh6JBaB8xLlk5+PjQ0RkFMeOxjrUDyxdmrr1GwDgV7Kkw/brbriF6264Jeu5MYa3Xp1PyZIlqd/g8sIOP0fHjh4BoGz5ClllPj4+lI+IJO7YUYf6M1/90O55UmICKz7/mAZXNgLgf9fdxP+uuylruzGGj95agF/JktSu39AVTXAqNvYwAJGR9u9XVFQ0sUfs36+UlBSSkpKIjo62K4+Orsj+P/8A4PDhw1xx5VX22ytW5EhsLMYYxE1XQWu7ik670pPjACgZHJ5VJj4+lAwpy+lk+7kGJuMspxKPknHmNHsWjOJEzD5KlYmgYvsbKH9VJwBSj/yDr39pfn9/PEn7f8KvdChlr+xA5Y434eNXyuXtKTjv60NekglSRNoD7Y0x4wrrmAkJcQQEBtr9IgUIKh1EfNzxAh07Pu440yY8zUfvvsHDTzxLufIRF9+pkCQlxOMf4NiuwKBgEuIv3K6fd21jwuiRHDl8iKnz3nLYnhAfx9xp4/j8o7e59+GnCS9XvlBjv5C4uDgCnb1fwcEcP3bMriw+3vqyKl06yK48ODiYY7a68fFxlA5y3J6amkpqaiqBgYGF3QSntF1Fp13pKUn4+PkjPvZt8i0VyJmT9qde0k8mgsngr2Xzqdr9Dqr0uJOkP3/g9w8m4lsygPDLWpF69G8y0k8TUv1KKnW8iZTDB/jry/mkJ8cVm0k3q79axpqvvnTb612SCdIVypQJJzUlhbNnz9r9I05OTiIkNCxfxzTG8PmnH/LsmIc5e/YsU2a+xPU3DS2kiHMnpEwYaamO7TqRnGQ3TJrdieQkZk16ks8/fof6l1/Nwk9XUaN2vaztxhi++vwTZjz7GGfPZvDklBfpe73zyRauEh4eToqT9yspKZGwMPv3K/P5iRPJduWJiefqhoWFcyLZcXvJkiUJCAhwRROc0nYVnXb5BYaQkZ6GyThrlyTPpp2kRGCwXV0fXz8AKnW4gajWAwAIrlyfk4f3c+jbzwi/rBUV2w4mus31+JUOzdou4sO+j6dS/X8j8S3p75Z25VduOu1de/Sma4/edmXvv73QRRFdYucgRaSZiPwKvA2MEJFfRWREYRy7fIQ1BBl7OCarLCMjg9jDh5yeY8yNqc8+wah7htGuU1fWbfmZQTff5rYhrUyZvdWjsYeyyjIyMjgWe4hyERUc6ifExzG0fydWLVvMk1Ne5M3P1tolR4AXp45l7Ki7aNmuC4vX7aDfoFvc3q4KtvckJsb+/ToUE0NkZJRd3cDAQEJCQog5aH+ONCbmYFbdyMhIx+0HDxIZFeXWtmm7ik67/GxDq6eTzvWATUYGp5OO4Rdc1q6ub0AQ4utHYFRNu/LACtU4lWANMZcIDM5KjplKV6oDJsPuNbyV5PPhSsUiQYqIEZF+2Z4H2craZ69njNlqjKkH3ALMNcbUM8bMLYwY6tRvQNnyEWxce+4k/487t5GUmEDLNu3yfLxvv97Aq/NmMfqZSbww/w23DqtmV7POZYSXLc+3G9dkle35cQfJSYk0bdnWof4LE8eQnJjAwkWr6DfoFnx87D9i277dyLuvzuH+0eOZ8MICtw6rZtegYUMiIiJYle3at21bt5KQkEC7Dh0d6nfo1JmVK8/VTUtLY+P6dXTsbE1e6tCxM2vWrOLMmTNZdVatXEGnTl0cjuVK2i5LUWhXYGQN/ILCiP91S1bZiX9/4UzaCcrUamRXV0QIqdaQE//8Yld+8uA+AspXBuCXt57kwLKX7Lanxv6FT4mS+Ifl70e6O+kknWLMz8+PW24fzvSJYylbrjylg4J5dswoevTpT8VKVUiIjyMhPp5qNWpe/GDAss8+ITIqms49+vD3AfsLnCOjovF30zBQCT8/rrvlTl6a/izhZcsRWDqYGc8+RqcefYmsWJnEhHiSEuKpXK0GZ8+eZe3yz+k3+BZK+Qfw71/n4vYt4Ut0paqsWraYiMho2nbuyX9/H7B7rfKRUfj7u2l4y8+P4feOYOxTYygfEUFwcDCjHhxJ/2sHUqVKFeLi4oiPi6NmrVoA3HvfSHp260yjRk1o0bIVL8ycTkhoKP36W8Ndg2+4kSmTxnPv3Xdyx13DWbVyBRvWr2PT5i0XCkPbdQm3y8e3BFGtBvD3igX4BYXhWyqQ/UtmUfby9pQKq0B6ShJnUpIIKGddwhLV5jp+f288fsFhBFdtQPwv33Hspw1cfq+1yEhYvRb8+ekM/AJDCK3dhLTjB/lr6TwqdrgB8fX+r3pvXElH3H1xtiuIiAH6G2OW2J4HAclAB2PMhmz1mgPnX5X/ojHmxfOORcMrG5FbQ24ZxpBbbrdW0nluAksWfUBqSiodu/Zg3JSZ+AcEMGvaRF6cPon9R1Md9m/TqC633TWCYcNHZpXdNrgfG9eudPp67y9ZSYvWjr23izmefCrP+4B1zvCVWZNZseRj0lJTadOxG4+Mm4a/fwALZk3h1RefY9v+BOKOHaVbs9pOjxFVsTJffP0zD9x2Hd9uXO20zsvvL6VxizZ5jq9h5dCLV3LCGMOEZ5/hg/ffJTUlhR49ezNz9hwCAgKYOH4ckyY8S2r6uX8fiz9dxLQpk/jrrwM0bdacOfNeplr16lnb9+7Zw0MPjOCHXTupXacuzzw7gS5duzl5ZdfSdrm/XZ1nfZ3vNv2zaiFHd64i4/Qpwi5rSY1+D+LrV4p/Vi3k39Vv0nr6pqz6R3asImbTR6Qe+4/AiKpU6X4HYXWbZR3r0DefcGjzZ5xKPIp/eCSRLf5HVOtrs85xHv7+Cw5vWZqr2E7+91vmcV2auTK/cw8lnM7X/lFlrKsAXBHnJZUg83Asp4msqMtvgvR2+U2QShWW/CZIb7b5UetHuNsSZGI+E2So6xKk9/e7c09y+FsppZSX88Yv7WIxScfmmmx/5338USmllMfoJB3XultENgGHgOmeDkYppVTueeMkneLUg/wAeB74Eljm4ViUUkoVccWpB/mlMebObM8f81gkSiml8sb7OpDFKkEqpZQqorwwP2qCVEop5XluXm0yV4pFgnT1dTpKKaVcSyfpKKWUUkVEsehBusITo+5zKOvUrSeduvXyQDRKKeU6cXs3E7f3W4/GkJsh1q+WL+Wr5e67H2SxWGquMOlSc0WPLjWnPE2Xmsu/zO/cuJNnLlbVqfDSVj9Pl5pTSilVLOkkHaWUUsoJnaSjHHzw9uueDsElFn/wpqdDcInXX13g6RBcoji2qzi2CaxbVin30ATpYR+8vdDTIbjEkmKaIBe+Vjy/dItju4pjm4Bc38+xqNHFypVSSiknvG+AVROkUkopb+CFGVKHWJVSSikntAeplFLK43QWaxG3dmXuV3DIS11Pv/6mtSsKtV5e5eW4ua375bLcT2TIS928yO1xXRWrK9rlDbEWpXbF7d1cqPXyKi/HdVUMueWNk3Q0QebgiVH3OTzefu3lXO+/duXyQo8pL8fMS92v135VqPXyKi/HzW3d5Xn4EstL3bzI7XFdFasr2uUNsRalduV2+TZXLfOW+9ffzF/L5vPHoulZD3eTfD5cSYdYczB55jyHMmfrsyqlVFEXfllr4vZ+S62Bj2aVxbr7chLvG2HVHqRSSqniRUT8RWSRiHwrIrddrDwnmiCVUkp5nOTzfznoD3wDtAGGikiJi5Q7j0nv5mEvc2V5pZRS7rubR0Flj1NEZgBLjDHfiMi7wFhjzP6cynM6pvYglVJKFTehQIzt7xgg7CLlTukknfO4+teSUkqpc1z0nZsIVAL2A5WBhIuUO6U9SKWUUsXNdqCxiPgCFYF/LlLulJ6DVEopVayIiD/wLlYSfAWoAizC6jlmlRtj3rzgcTRBKqWUUo50iFUppZRyQhOkUkop5YQmSOVRIq5eblgppfJHE6RyOxG5S0RmAxhjTFFPkiJyvYh09HQcyl5R/1wpz9MEqdxKRAKBakAPEZkARTdJiiUcmAU8LiLXeDikPBGRDiLS1dNxuIrtc1VsrvUWkSrZ/i7nyVguFZoglVsZY1KAF4G3gL4iMtVWXuSSpLHEAZ2ASGC0iLTzcFgXZUvsQcBsoL2Hwyl0th799bap/o+LSGVPx1RQtjYMF5HWIvIUMERE9PvbxfQ/sJcSkUYicpOIPCIiLTwdT2GwfTH7GmMOA8uBFcAdIjIail6StLXHzxjzC3ArUBsY6e09SVtiPwG8BNwqInU8HVNhEZEA4DegAbAbCDDG/OvZqArFUeALYBJwizFmjjEmQ0RCPBxXsaYJ0guJSH9gNXAXMAx4QUQmezaqgrN9MZ8VkX7AC0BrIAir51Ukh1uNMeki8j9gEFASGABMEJFWno0sZ9l6HhuAg0AjW7mvp2IqLMaYVKw1NvsC/wIbIeu8d21PxlYQxpg04DagDLBIRGrZvidGikhJjwZXjGmC9DIiUg+YDkw0xrQFhgDNgAQRKe/R4AqBiDQCXgYWA7cADYH3gOtEZDwUnSRpi7M91socR4BRQD/gMmC8iLT2WHBOiEh9EalqjMkAMMb8CvwBjLE9P+vJ+AqDLfnfB2zGSiiXi8gWoJcxZp9HgysAEakEnAZ6AK8DM4DngeXGmNOejK1YM8bow4seQE9gp+3v6lhrBb4ORAGPA1U9HWMB23c38DMQnq2sMlbSTAQe83SMeWzPNOArwC9bWX3gELAeuMbTMdpiuhz4EavH+CDQ1lbeENgF9LU9F0/HWghtLZft75XA29meF8n2AcK5lc/uxxo+bmh77uPp+IrrQ3uQ3qcUcEREqmP9Cl5jjLkdOIWVINt6MrhC4AcEYv2Dx3ZO8l9gLuALPCEiEz0YX67Yzj8KUA/riyvdVl7KWOckH8AaQn7UGybuGGN+xhqyn471I+U1EfkEK0GGAM1t9Yr82pPGmGMAInIf1g+XW2zPfYtq+4yNiJQFWgDDjDG7RcTH2EYEVOHTBOkFRMQ32/mfP4GOtv9fAtxhKw/CWmj3oNsDzKfMYVJb+zI/az9j9YwHgN2wng/WD4K7gAVuDjVXsg/7Zn5hAauAppnDqcaYU7Yq/sAe4CTwu7tjzS4zbmPMFmPMLKxhuruxFnAeivV+jPTm86Y5uchMziXGmI62er6mCA0hZ7bLyamGOOAOY8xWTY6uV2yuESqqRKQzMBgIFZFnjTE/icjtWEniEBBtS54PAhF4+Ms2t0REbL94OwL9gYoiMtoYs1FEngXm2q5RWw6cxTofWQJYbYyJ91zkzmVrT3Osc8Knsc6jLsZKOM+IyHhj3ak8CGtIcykwxVgTRzzm/F6TMeYv4C9bQmwOfI/1Q+weEUk1xuxyf5R5lz1BiEh9oDzwH3DEGHPCGHMwW70ikxwBjDVDtREwQESmGWOSbOUGSMms48kYLwV6Nw8PEpHuwCfAPqxzjAFAZ2PMdhG5B2umZyyQjDX8OMQY84OHws0z22zV97BmS1bB+gLrboz5wXZpx0SsyS1JWEPL/YwxP3om2pxlS479gVeBY1jvxy6sWcYNsCa6dAL2AgarV9bZNrTpdc7vfYjI9VjDwjHAVGPMDo8FlwuZ74nt74lYs4jDsX5UbgJmGmP+8GCIBWb797MYmIr1QyvZsxFdgjx9EvRSfgB3Ao8AYVhfqB9h3eG6iW37ZUBvoAsQ5el489i2y7G+bB+xPW8IZAB/A1fbyppg9Z6vAyp7OuaLtKcl1g+Vh7F+yDwGpAIfAKFYX843Y002GgPU9nTMuWyXT7a/b8RK+ld6Oq48xH+P7XN2re35W1g/YFoAJT0dXx7b4jCBCGv05SzwHBDk6RgvtYf2ID1ARBpjnV/8EeumnUtt5RWwVpnpgdX72OqxIAtIRHpgXYjeG6uH/BxWgqwMtMMaUo0zxmzzWJB5ICJjsGYYd8KaYPQK1pC3P9ZF3I8DscbDw6n5cV5vLNJYCzl4Ndu5OR+sHyg/GmMmiUhPrJvijsA61x1mjFnlwTDzTEQqAmnGmOPZygZi/XieAUwytuFW5Xo6ScczjgLbsb5wgyFryCsWawr3F8D3InK150IssDLAcazh03bAGaykMgZrqPV94DMRKVsUrnnEmiRVFmv4tDfWua6hwJdYPeAtwLsiElRE2pPFGLvrTmM9GkwumXO/7MsCm0WkE/ApMNoYsxDoAEyxrazj9UTER0TCsOYYjLfNVgXAGLMIuAFrtGmEWOv/KjfQSToeYIz5x3aOsQTwoojsMtalARhjYkXkEazLOlI8GWdu2CYQZdi+ZMOxptXHGmM+EJF/sL5w52D1lvcBXbF+Db8K/Jn9l7I3sE0cOmtrj7+xVjAB6zzQN1iXqFwPbDTGHBGROKzJOEuBr421hFuRk5lwsiUer2esVZmOYy3UEA6MNMa8Ztt8BojHmkzl9Yx1Pjje9r3wGnBSRKYbY47aqizHWtRhIlDKNqFPJ+m4mA6xuoltRtqVWL94fzbGrBSRUKxJOldhXbj9a7b6Xj0tXUT6ZA4N2573BqZgLbe22RgzzFZ+OdYF85fZEspcoLQx5jZPxJ0TEWlkjNmZ7XlX4CGsSUTLjTEf2cr/B7wJ1DDGJIjIK8B/xpgJHgj7kpVt4lQ1rIlgkcaYmiJSCmtU5nNgnzFmqAfDzLVsl+IYERmENXT8HDDLNrKEiHyAbSTGGPONx4K9hGiCdAPbbLRXsXpRdbEmd2w3xtxgS5IfY01i6WqM2eOxQHNJRGpi9QY/MMbcaBsKXo/1pXQSa3m8r4wxQ2z1twO1sGZ41gXaGWN2eyR4J0SkF9b5nbnGmHki0hJYB3yHdc40Fms1lgUicgXWexmDdU61NdDGFOFlzIoyW4+/AzAfa0TmP6wZxiWAlsaYM9nPsXqz85LkYKzTECuBzLkIDwJNjTG/6zWQ7qEJ0sXEulPCcmAe1gzHkkA3rN7WVmPMEBGJwJrOHQE0MLZVWbyV7SLmnlg9qc+AN7CS+zgR8cOadfsusMoYM1hEamCdWwXr1+8vHgg7R7b36Amsu3G8j/XlWsIY87xYa2BOAqoCbxlj3hCRZ7ASoy/woPHSSzkuJbbzd/dhTaCKA162DcGWMMac8Wx0uXdekrwWa4QJ4B3gNWPM15oc3UcTpIvZrnWcjTUr9V9bWQDW3QaeBe40xmyyzV7zMUXk1jy2f8g9sRJhKeBVY8wDtm0lsc41vou1mslQW7nXfVllG6qrATyJdblNJDDeGPOhrU51rPeqBvCiMeZjW3mgse5vqbyQt5+myMl5SbIP1sjMbOBRb/v3U9zpLFbXy8Ca2FEGsr6QU7GG8MpjreWJMeZgUUiO2f/xYi3SfSPWrNyGtp4lxrq7wEqsmXe3iMhC2+5e+WVle0/2Yy08fgCoBjTO3G6MOQA8jTWs/JSIDLVtKnKXdFxKimJyhHMTpWyfy6VYpyxGANNto03KTTRBuoCIhIp1N3OwvlQDgKEiEpDtXMhZ4Bes5OL1xLZerO1XbU0RqQVUMMYsxxraaoI15ApY90nEuqdlD6zJBl43QzLbualQW2/wN2Ac1jBrTxG5N7OuMeZvYALwLdb5Vq9rjyo+zkuSH2HdkPsBrNMAyk10iLWQ2S6QH4W1uspcY8zbItIX6xzjHOBDrNVy7sK6XKCVMeYfD4V7UWKtC7vfGLPe9rwf1oQIX6wE/5TtvEhvbNc2GmNu9VS8uZVtaDXz/SqDNXz6johUAZ7BWsnobWPM/Gz7ed0wsSq+zhtura2TwdxLE2QhEpE2WJNWtmItH9cUeNgYM9t2LuEVrF57PFYP8ibjxWurikhlrHtRRmMtZv0z1n3o3sMaXmyCNbFodLYk+RbWNYIDPBN17jl5v5pgLY03W0SqAmOBOlhJf6bnIlWXsmw/5jL/XyfpuIkmyEIiIoFY59yigOnGmDQReRprcsdDti/daKzzW4J1jdYRjwWcSyLSFmsItRZW8quJlUTSbQnmAaw2ZybJAcAsoIUxJsZDYV/URd6vB40xL9p6kjOwepfXG2MSPBWvUsr9dCWdQiAi5bHu37gL+NDYVl8xxkywTVx5QUTOAguNMd96MNRcy5wBaJthm451DdYTwK7My1Ayp5xjTSCYKNbqHotFZJXx4hVlcvF+zRKRDGPMXBF5GDijyVGpS48myEJgjDkqItcBK4C9IhJsbLemMcY8a0uOLwKnROS1IjK5I/M+e5WxLor/ButuFjeISFtjzCYAY93f0WBdIvGYiHznzckRcv9+iUi6MeYVjwarlPIYHWItRGLd/PgrrPN1H5lsd3YQkceApd52kbwz2c519MW6J6XBGhZejHUtYE3gPpNtuSsRaQ38bYz5zxMx50dxeb+UUq6hCbKQ2RYG+By4m/O+dIsSEWmGdVeRWcBvWAunb8F2ET3WdPORpoivCVlc3i+lVOHTIdZCZoz5ytbz+gTwF5E3zbk7QhQljbHWTn0x+2oxIhKFNRP3W6zbOw0yxmzxUIwFVozeL6VUIdOFAlzAGJO5wsw4rGXYiozM666wFunOWkpNrEWhweo5XoN1DeQmishCBxdSlN8vpZTr6BCrC4lIkLdPWMmJiHTEWi7uJttKHpnlA7GuD+wIJNmWlSsWivL7pZQqfNqDdKEi/mX7DTAX6/KN6yHrjgltsBY5yChOyRGK/PullCpk2oNUObItjPwUMJxzE3WigD7GmB89GZtSSrmaJkh1QWLdob0F1nnHg1jLyB3wbFRKKeV6miCVUkopJ/QcpFJKKeWEJkillFLKCU2QSimllBOaIJVSSiknNEEqpZRSTmiCVEoppZzQBKmUUko5oQlSKeV1RORNETHnPWJF5GMRqeqC1xsnIj/Y/h5qe71qhf06F3j9DSIyK4dtQ0UkIQ/H+ktEHixgPEZE+hXkGMWBJkillLfainX3mNpAPeBeoCWwRER8Xfi6i22vefBiFUWkt4joaivFlN4PUinlrVKNMX9ke/6biIQAC4EawD5XvKgxJglIcsWxVdGiPUilVFGSbPt/f8gaCuwuIl+JyE5bWSkReU5E/hSRkyKySUSuyTyAbft0EflbRI6IyOtAYLbt7W3HLWN7HmYb8j0kInEi8qGIVBCR9sDSbHEMtf0dLSIf2YaE42zDwpWyHT9KRBaJyHER+UNEHgEy78N6USJSV0RWiEi8iKSIyE4R6X1etQAReUtEjonIARGZLCIlzzvGclt8R0Tk1cz2qnM0QSqlvJ5YagNjgL+BX7NtngcsB262PX8V6Iw1JNsO+BlYKyL1bdufB+4AngZ6AQZ4KKfXBZZhDfEOBm4HrsTqxW4B7rZVrQ0sti3uvwnrlnB9gQFAeWCdiPjbhoZXAnWBIbbjXQu0ysN/jreAcNv+3YE9wCe21870NBBn2z4NeBCYaGtTOeB7rDv0dAVuBZoBn2e7YbpCh1iVUt6rrYik2f72xfq++g0YaIxJz1ZvkTHmRQDbxJqbgOrGmL9tZTuARkB/EfkX6/Ztw40xb2fb3jynGLDuZlPTGPOXrf4J4EngDBADkDkULCK32va72Rhz1la2EzhqiyEYaAjUN8b8Zts+EMjLHXI+Ab4yxuyx7X/W1uYo4C9bne3GmMykv11EQoGxIjIGuA/YlW07IrIf60dHVGablCZIpZT32g7cku15InDYON6C6IdsfzfEGq787bzOUEmsBFAbK9muzdxgjMkQkQ1YNwM/3+XAv5nJ0VZ/NbAawEmH63Ks86Mnnbx+baye37+ZydF2vIMi8hu5NxtoLiL3AFdj9RLPt/6856uAKUAVW4ztsv34gHNDvLXRBJlFE6RSylulGGN+vXg1UrP9XQLIABpjDXNmdxIok8MxMnIo97vANmdK4JjYMx0Fbsjj69sREX/gK6AS1mzbL23/v+K8quf/iMg8nXbKFuMXWMPV59PkmI2eg1RKFSe/YH2vBRljfrUl2D+xzkfWA/YD6UCnzB1s593a5nC8vUCV8ybZtBKR3SISnkP9Gli9xMzXTwFGY00E+gWoLCJ1sh2vgi223GiL1dO90hjzmDHmc5xP8Gl/3vOuwHHgkC3GOsDv2WIMBB7H8UfFJU0TpFKq2LANXX4OvC0ivUSkNbAAayLK78aYk8B8YLqI3CwiTbAm+VTP4ZBrgN3ARyLSTkR6Ai8Cx4wxcdgSiog0tl2C8j5WL+0jWyLtAXwIXIF1XeU64EdgkYh0EZE2WOcUU89/4RycxPrevl9EmorIA7b4AVqLSOaoYGMRmWGL6x7gKWCqbXh6HlANeM12jP7AB0AZ238fZaMJUilV3NyEdY7wJawZo9WALpmTdoBHsZLmRKzhykDgMWcHsk206Qb8i5XI3gH+wJpBCtaM1e+Ab4CexpgTWD28DKxLQN7BmoDTyxiTYYzJwDpn+BtWUnoLa5j001y27VvgGaxZqattsfW2xTYDCLLVGwlUtdUZZdvneVubDgLX2P67rLH9d1qL9SNCZSOO57uVUkoppT1IpZRSyglNkEoppZQTmiCVUkopJzRBKqWUUk5oglRKKaWc0ASplFJKOaEJUimllHJCE6RSSinlxP8BFH7tqMzWRS8AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# make confusion matrix of PF for comparison\n", + "batch_size = 50\n", + "\n", + "num_classes = 6\n", + "conf_matrix_pf = np.zeros((num_classes, num_classes))\n", + "\n", + "for i, batch in tqdm(enumerate(test_loader)):\n", + "\n", + " # make mlpf forward pass\n", + " target_ids = batch.ygen_id\n", + " pred_ids = batch.ycand_id\n", + "\n", + " conf_matrix_pf += sklearn.metrics.confusion_matrix(\n", + " target_ids.detach().cpu(),\n", + " pred_ids.detach().cpu(),\n", + " labels=range(num_classes),\n", + " )\n", + "plot_conf_matrix(conf_matrix_pf, \"PF\")" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3575aa0943ef44e2ba51ffb9b5c0f7b6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Time taken is 150.4s\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcgAAAGoCAYAAADCVXWwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAAsTAAALEwEAmpwYAACGf0lEQVR4nO3dd3wURRvA8d8AgSQkhAQISehSFRSpCgiEGhJAqjQVkF7tr4ogvSMgTbCBKHZAFOkdsdAtNAtNJBBKGpAEA5n3j92EXHKBS8iVhOfr5z5yu3N78zDHPTezs7NKa40QQgghLOVxdgWEEEIIVyQJUgghhLBCEqQQQghhhSRIIYQQwgpJkEIIIYQVkiCFEEIIKyRBCpFJSqlgpZRWSp3KxGvGmq/RSqkTSill4+u2pnrdh2n2fWhu357J+qeui7XHZaXULqXUVKVUoQyOceoOx0j76J2ZOgrhCiRBCuF45YCH71RIKVUUaGz32qTnBzQAXgWOKaVqOKEOQjidJEghnKOzDWUex77/Rk9rrVXqB5AXKA+8CFwFAoH3lFJ5MzjGuLTHyODxoR3jEMIuJEEK4VgnzP93smGYtWOa19id1jpJa31Caz0b6GNurgW0dFQdhHAVkiCFcKxfgJNAZaBqRoXMc38tgDhgg0Nqlt5y4JL55wzrKkRuJQlS5DpKqQJKqaFKqR+UUueVUleUUr8qpWYopYpk8JpqSqllSqmjSql4pdQZpdRGpVQrWyfU2EgDK8w/326YtTWQH1iHkSQdThsLNZ8yn97njDoI4UySIEWuopTyAHYB84H6QHHAC3gIeBn4RSnll+Y1XYDfgCeBKoA7UBKjB7cOeCmbq7nc/H+n25RJHl5dcZsydmX+MChrPv3HWfUQwlkkQYrcZiRQG4gCugHFgEJAKHAeI/GNSi6slPIBlgAK+AKoBnhizDR91yw2WSlVLBvruBf4F6imlKqcdqeZ5MOA/4A12fi+mdUJKGr++Tcn1kMIp5AEKXKbYPP/o7XWX2itL2mtr2it1wOTzH11U5Wvg5EQLwJPaq0Pa63jtdangCHmdjegenZVUGudBKw0n1obZm1p1mmT1jo2u97XFkqpPEqpckqp5zF+OAAcJOPzoGNsuAayrCPqLkR2y+fsCgiRzQqa//ewsu99jOHNm1bKgzG0ei35idb6plLqAYx/JzHZXM/lwLMYvbRJafY5ani1jFLqTjeEjQD6aa1v3qGcELmO9CBFbrPV/P8kpdQSpVT95Gv4tNYJWuvzWuuLqcrvwUiKxYC95uSelIk8Zg/0vNY6Ppvr+SNG8qmhlCqfvFEplR/j+sebwLfZ/J62ugkcAhYBD2qtD9ymrC3XQZ5ySK2FyGaSIEVuMwb4DmNYtDfwA3BZKbVWKTUg7SxWrfU54GmMc5b3Y0zuuaSUOqSUmqOUapDNs1iT3/cmt4ZZU0/WCQYKA9u11pez+33TSLdQgPnIp7V+UGs9OM2PCSHuKZIgRa6itb6K0QNrAryNMfvSB2OSzjvAcaVU1zSv+RqoBAzH6IH+h3Hd37MYM2LXKKUK26G6yUOoqROk02evCiEMkiBFrqMN27XWQzEuUygHDAOOYCTLpUqpMmlec0lrPV9r3QyjB9cU+BC4jpFcJ9uhqjuAy0BdpVRpcyi4Pca1kqvs8H5CiEyQBClyDaVUQaXUy+bDHVKS5Smt9QKgBkaPsgDGYtwopVqY5ZsmH8ecxbpNa/0M8Jy5uXl211drfYNbibAjUA/jus0fzaFfIYQTySxWkZskABMxEuARYG2a/TdS/Tl5VmodjFmke5RSj5qrx6SWmKZ8dlsB9MW43KNMqm1CCCeTBClyDfOyjL3AY8AipdQAjNmiiUBFYARQGmPptp/Ml+02/18XmKOUmg2cxRiKbQnMMPdvtlO1t2Ak3wYY50Hh1uQdW7nbcK1hnNb6QiaPK8Q9TRKkyG2GYKxUUwpjmbi0koCntNaRAFrrLUqpTzCWmRtuPtL6CRhnj8pqrf9TSn2LMZO2GLBfa306k4d5BGMB9Nv5BuP8phDCRnIOUuQqWuvfMS7XeA84DMSaj98xlo6rYs5aTa0XRoL6HmMJuP8wzlVuN/c11lon2LHay1P9WYZXhXARKv0pFyGEEEJID1IIIYSwQhKkEEIIYYUkSCGEEMIKmcWahg13NxBCiHuG1jrb1yJOLbu+c+1RT+lBCiGEEFZIDzID7k3S3qIPEo99jVuVDja93tay1/ctoEDtodl6zIzKnvn2FatlXxw+iFnzFt3xmLaWA2je6BE279x954KZPK61svnzpf+d9+zQgcxd8I5Nx8xM2cYN6rLjhz3Zelxr5dzyWv8xPHTwABYsfNem989M2QaP1uGHn/dm6zGtlc3oxihDBw1gwSIbj2tj2QaP1OaH3fuy9Zj2KpuZYzoqLg83u3Yc03F/2LbvwbQSflmQzTW5RRKkEEII51OuN6CZYYJUSh3EuKvAHWmta2ZbjYQQQtx7sv+2q3ftdj3IVY6qhBBCiHtcTupBaq3tsvakEEIIkU4O60FaUEo1AboCFYB+GAsk/6213m+nujlV4rG0y3UC+b1sfn2eolWysTaZP2ZmyoaEtsnWcpmVmePaWjY0zPZjZqZsZth63My8f1jrtnYp6+z3D2tjn7LOfn9by9ojpswcd813q/nnn9MMHTTALvXIqWxai1Up9TjGIspfAU8AVYGngFeB1lpre90KyOGSr8mxNovVHjIzi/VuZTSL1R4yM4v1blmbxWovmZnFejcymsVqL7bOYr1bGc1itYfMzPbMSRwVV/IsVkddB+le56UsvT5h70zAuddBjgbGa617YN50Vms9GpiDcYNaIYQQIuuUytrDjmwdYr0f467naX2Fcf89IYQQIutsmKRzM+pvkqL+dkBlDLYmyNNAGeDXNNvLAuHZWSEhhBD3IBt6g3n9KpLXr6LFtpsXf7NXjWxOkEuAmUqpSPN5kFKqOvAWxo1phRBCiFzF1gQ5E/AGNgIFgK0Y5yIXIOcg70reoDrOroJd9Ozdz9lVsIvefXJnXH369Xd2FbJdn365c0Zmbo3LFa+DtGkWa0phpQpgXOaRD+MSj2v2qpizOHoWqyM5charIzlyFqujOHoWq6M4charuDsOn8Va//UsvT7hx8mAfeqZmesgiwF9gCrAdeA3pdTS3JgkhRBCOJgL9iBtqpFSqiZwAngZ8ANKAOOBv5RS99mvekIIIYRz2NqDnA38AHRK7jEqpbyBlcB8IMw+1RNCCHFPcMHhd1v7tA9jLBSQMpyqtb4CjAPq2aFeTpd47Ot0j5uXjt7xdSN7NeTQx4M48dWzLHw5DI8CGf8G6f94TfZ90J8L373E1rlP07B6aavlgop6cXbVC1mO5W5prZk+eTx1qlehaoVSPDekP3FxcRmW/3bVCpo1rEv5kkV5on0Yp06esNgfEx3NkP69qFqhFA9WKsPk8aO5efOmvcNIR2vNlInjeLhqJSqVK8HQQf1uG9eqlctpVL8OpQP86NC2FSfTxPXx0sXUebgqAX5e1KtdneVffGbvEKzSWjNx/Fiq3V+RcqWDGDSg723j+nrFcuo/UpvAYr483roVJ0+csFruqy8+p0sn2+5Hag/JcVWtUoFypQIZ1P/2ca1csZz6dWsRULQwbcNC0sV19MgRWrVoSvEiPjxWry4b1q+zcwTpuUpMa75bzdBBAyweDqfyZO1hR7Ye/QTgY2V7IeDf7KuO63Cr0iHdI2/R+2/7mteeasDg9rV4/Z2t9Jn8LXUfKMHiEY9bLTusUx0m9m/C/BV7CHnhE7YfPM2307rx4H3+FuV8vd2ZMbQFefI479fVrOmTeX/RAsZOmMrC95eyf+8ehvTvbbXsD9/vYOAzT9G5S3c+X7EaLy9vHm/VlPj4eABu3rxJp8dDiI6K5v2ln/HKiDd4Z8Ec3n17rgMjMsyYOol3Fs5nwuRpvLv4I/bt2c2APj2tlv1+53b69nqSLl17sHzVGry8vAlrHpwS17atm3l2yED6DRjMxm276NLtSfr2foofdu10ZEgATJ08kUVvz2fSlOks/vBj9u7eTd/e1uPauWM7vZ7uQbfuPVi1ei1eXt60aNY4Ja5kf/35J5MnjndE9TM0dfJEFi6Yx+SpM1i8dBl79vxMn95PWy27c8d2ej3Vna7dn2TV6nV4eXvTvGmjlLgiIyNp2awxpUuXYfXajYSGteaJju04sN+xS0u7Skyt27RlwaJ3LR4O54IJ0ta1WFthDLMOAnZh3CfyMeB9YJTW+kt7VtKRsjqLNV/ePPz1xTDGL9nJkjW/AFCnShDb5/eiUrf5nL10xaL86RXPMfOzn5i7/Na6nktHteNq/H8MnWn86lvy+uN0aVqVPHkU0VcTCHx81t2ElqVZrImJiVSvUo4Ro8by9DPGJQ779+4mtFlDfjl6gqASJS3KP/NUVwoUyM+iDz4GICEhgaoVSjJj9nw6PtGNjevX8uygvvxy7CTu7u4ALJw3mz//OMbs+e9kKa6szGJNTEzkgYplGDl6HL37GJc47Nuzm+bBDTj85ylKlLSM6+nuT1CgQAHe/3BZSlyVygYxa84COnftTv9nnuba1at8+tWtRe5bNWvMQ9UfZvqsOZmuX1ZnsSYmJlLxvtKMHjuePn2NuPbs2U2ThvX54/hpSqaJq0dXI64lH92Kq1ypQObMe5su3boDUKl8Gf49cwaANm3b8eUKKwv52yirs1gTExOpWK4Uo8dOSLkkZc/u3QQ3rMefJ/5JF1f3rp0pkL8AH378CWDEVbZkAHPmL6Rrt+4sXDCfWTOnc+yvk+TNmxeAdm1CKVWqNPMXZu1zmNticvgs1uCs/QBL2D4acPBarEqpKKVUpLk4wKcYl3dsBRLMxzaMyTqvZXelcqKq5YoR4OfFht23lkHa90c4kVfiCa5Z1qJskUIe+PsWZO9Ry0WIdv12hnpVS6U8H7t4B3X6vc/YxTvsWvfbOXrkEBcvRNA8JDRlW41adfD19WPn9q3pyu/YtpnmLW+VdXd3p2HjpmzbsgmAb1Z+RUhYm5TkCDB4+AtZTo5ZdeTwIS5ERNAy5Nbp85q16+Dr58eO7VvSld++dTMtQyzjahTclK1mXP7+xWkZankqvmixYsQnWPbE7O3wISOukFa36lK7dh38/PzYvjV9XFu3bCKklWVcwU2asXnzppRtX3+zhr0HfqN1G+ujIY5w+NAhIiIiCEn1d1y7zm3i2rzJoqy7uzvBTZuxZdNGY/+WTbRoEZKSSABCWoWxefNGO0ZhKTfGlNvcbpLO846qRG5Q3K8gAOcjb131ojWcu3SV4r4FLcrGXLvO9f9uUNK/EBy+tb1McR8Ci966pdbp8zEA1KwcYMea396FiAgA/IvfqkOePHkICAzk4oULFmXj4uK4EhtLQGCQxfbAoCBOnjgOwNl/z1D94ZoMH9SXHdu24OlZkA6du/DC/0aQP39+O0dzy4WI8wAUD7CMKzAwiAsR6eOKjY0lIMgyrqCgEpw4YfwgmjTtzZTt169f56cfd7Fz+1YWvf+hnSKwLsKMK8BaXBciLMomxxWYNq4SQRw/fjzledVq1QDw9fUlOjraTjW/vdvFFZFBXEHW2uu40V7nz5/noeoPW+4vUYILERForR1yvWZujOmuuOBlHre7YfLSO71YKRUIPJOtNcqh/Lw9uBb/H0lJlkPWV+OvU8THw2LbjZtJrNhxlJG9GnL45EX++vcyIXXLM6h9LZf7EEdHReLp6WnxqxSgoJc3kZcvWWyLiY4y9hW0/EHglarshYgI3nl7LgMGD2fJsi/455/TjHzlReKuXWPc5Ol2jMRSVKT1uLy8vLmcJq7oqOS4LO8H6uXtzeVLly22/fzjD4Q0awRA/4FDCHNwryvDuLy9uXzJMq6o5Li80sTllb6ss0VmJa407eXt7c0ls2xUVGS6uL29vYmPjyc+Ph5PT8/sDiGd3BjTXXGx7z6w8TIPpZQH8BKQ9prHikA5YHI21yvHibwST0GP/OTJoyySZKGC7kRdSUhX/qV5m5j/Yih73+9HnjyKv/+N5P3VB2nToGK6ss5U2NePuLg4bt68afEP+UpsLD6FfS3KJj+/evWqxfbY2JiUffnzu9GoSTPGT5kBQK06jxAfF8eIl59jzMSp5MnjmF+Rvn7W44qNjaGwr2Vcyc+vXrU8jxwbE0Nh38IW2x56uAY/7v2FY0ePMPr1V/H182Pk6HH2CcKKDOOKSR+Xb3JcV9LEFRuTss9V+N2mvdLW1TeD9oqJuVXW19cvXdwxMTHkz58fDw/LH7T2khtjuisu2IO0tUbzgReAkkAvoCjwAPAIYH3K1T0mwhxaDSpy6xecUhBYxIvzkVfTlY++msBT47+maOsZlO00hwd7LuJi9DX+iYh1WJ1t4V+8OADnz906X5qUlMT58+EWw5MAnp6eeBcqxLnwsxbbz4eHU9wcoi1WPIAHHqhmsb9ylfuJi4sj8rJlb8yekoeMz4VbxnXuXLjFkBcYcRWyEte58LMEBAQCsHf3z0RevoynpydVqz1Ipye68tqo0SxbusTOkVgqfru4AgMtyibHFX7WMq7ws+EpcbmK5LjC08YVnr6uGcaVqr0CAgKsxH2WgMBAh43i5MaY7ooL3g/S1gQZBgzXWrfEWLB8vNb6UeBzoLa9KpeTHD55kYjIa7SsWz5lW50qQRT2cmfHwdPpyk/oH0zPVg8Rf/0GEVFGcu3Q+H6+2nbEYXW2xf0PVKNYMX+2bFyfsu3Avj3EREfTsHGTdOUbBTdjy6YNKc8TEhL4fuc2gps2B+CRR+tzYP9eUs+e/v23Xyjs60uRokXtGImlB6pWo5i/P5s23rpObP9eI65GjZumK9+4STM2bbj1d5CQkMDOHdtoYsb1dPcn+OKzTyxec/XKVfLls3k1x2xRtVo1/P392bDhVlz79u4hOjqa4OD0cTVp2pyNaeLasX0rTZs1d0h9bZUc18ZU1/Xt3WPE1biJlbiaNbf4O0hISGDHtq00bd7C2N+0OZs3b+TGjRspZTZuWEezZi3sGIWl3BhTbmPrv15f4Jj555+BWsA+4DPgTWDG3VTCnObbFKN32hr4D3hTaz3b3J8HGAYMxLgv5V/AVK31F+b+YGA9Rq/2XYye7Wngea31ZrNMXuBFjPVkywBHMRL9N3dT92Q3biaxaNU+xvcL5mJ0HFfi/mPW8JZ8vfMYZy7E4uvtjq+3ByfCjXMJV+MTmTm8JQDHTl+ib9sa+BQswFdbXStBurm50XfgECaOHUXRYv54eXsz4uXnadu+EyVLlSYqMpKoqEjuK18BgP6DhtKpbQgP16hJnUfqMX/OLAoV8qH148YF5k/16sP8OTMZ9dpLdOn2JH/+cYxJ495g2HMvO/RXrpubGwMGDWXc6JEUM+N65cXnaNehE6VKlybSjKu8GdfAIcNoF9aSGjVr88ij9Zgz+00KFfKhbfuOAIS2acvsmdMIDAqiYqXKHPr9V6ZPncjgoc86LKbkuAYOHsqYUa9TrJg/3t7evPT8s3To2PlWXJGRlK9gxDV46DBat2pBzVq1eLRefWbPfJNCPj6069DRofW+Ezc3NwYNGcboUSMo5m/E9eLzw+nQqTOlrcQ1ZOhwwkKaU7NmbSOuWTMo5ONDezOubj2eZMqk8QwZ2J9+AwaxccM6tm/bys4fdktMzmLLDZMvHiHpouO+I229DvJv4G2t9SylVAfgaa11R6VUY2CN1trrDoe40/E18DvGdZV7geeArsCDWutDSqnnMM5zvgTsAVpi3Garo9b6WzNBbsZYDm82EAdMxRgSLq611kqp8cCTGOvJngIeB0YDLbXWKXOqk6/JUd6Ws8VuJ29QHfIF1QXgjd6N6N68Kh7ubqz76W9enLeRhP9uMLJXQ0b1aohHU+N0bd48inF9g+nR8kHy5lHs+u0fXpq30WIWbLKnQh5kxtAWTrkOEozVPqZNGsfyLz4lLi6Olq3CmPLmHDw8PJg+eTwzpkzg4pXElPLfrlrB7BlT+Of0KWrWrsubby2gTNlyKft/+/Ugo159id9/+4XixQPp3bc//QcPTzdZwVZZvZuH1prJE8byxWefEB8fR0hoa2bMmouHhwdTJo5j6qTxxMTfWuFn1crlvDl9Cv+cOkmtOnWZPW8hZc24YmNjGT96JGtWf0Nk5GXKlC1H7z79GThkWJbiupu7eWitmTBuDJ9/9glxcXGEhrVm1lvz8PDwYOL4sUyeOJ64/5JSyn+9YjnTpk7m9KmT1Kn7CHPnL6RsuXLpjjug7zNER0c75TpIuBXXZ58uIz4ujtCwNsyacyuuSRPGEZ946/ts5YrlTJ8yiVNmXPMWLLKI68jhw7zw3DB+OXiAipUqM2bcBFq0DMly/XJCTB+89y6L37dtEYADB/Yn19Ex10G2zFo/K2Hj/wD71NPWBPk/YBpGD24tcAhYiDH0Gqu1bnRXlTD+guZrrYebz/2BCKCD1nqVUuos8I7Wenyq13wAVNJaNzQT5Dags9Z6hbm/C/AFRu83AYgCQrTWO1Md4zPgitZ6QKptcrurHEZud5Vz5IhzYQJwwkIBITOz9PqEDS8BTrzdldZ6hlLqNyBKa31WKdUHoyd2AciuMaSfUr3fheR/SEopHyAISLtm1w6gXUbHMOuWrALgDmxKbgyTm5XjCiGEcDQX/PFk8wwCrfWGVH/+Gsj6OIt1Ga/Qa10S6euf0TGSy4UBZ9Ps+y+T7yuEEOIekGGCVErZ3DPUWtttpWmtdYxS6hzQENiealdjwNaztX8DN4CiyecbldFFHYcxWcf67QuEEEI4hgteB3m7HqSt91fSgL1vxTADmKCUisCYxNMSYwWfJ2x5sdb6qlJqAfCWOcR6CuiMEWN9u9RYCCGE7XLSEKvWOv00NueZg5GIXwBKY/QInzaHem31ChCLMRs2CGOiURut9d5srqsQQojMymE9SIexNvso9TatdRLwlvmw9vrtgLrdNq31fxiXdYy++xoLIYTIVi6YIF2vRkIIIYQLcIkepBBCiHtcTjoHKYQQQjiMCw6xZipBKqUCMG5xdRC4qbV27O3SHSjxWPr5P3mKViFv0fudUBshhLCfNd+tZu13q51bCRfsQdq61JwPsArj2sMkjEXBXwMKAb211unv55RDyVJzOY8sNZdzyFJzOYfDl5pr/16WXp+wqj9gn3ra+s0yAygM3I9xwT0Y94isBkzJ7koJIYQQzmZrgmwHvKa1/gPjekS01geAEYBr3RdHCCFEzuOCN0y29RxkPiwX/052BvDMvuoIIYS4F9ky/H7z3C/cPPerA2pjsDVB/oxxL8WD5vPkE5eDgf3ZXSkhhBD3FlsSZL6gGuQLqmGxLe6U/W7IZGuCfBH4SSn1MMYtoqYqpSoBZYHH7FM1IYQQwnlsOgdpnnusBHyPcZurPMC3QGWtteP6u0IIIXInlcWHHWXmfpCXMG4PJYQQQmQrV7wEyKYEqZSadbv9WusXs6c6Qggh7kU5NkECNdI89wSqAHHAumytkRBCiHtOjk2QWusmabcppbyBD4AfsrtSrkCWmhNC3CtcYqk5F2TTUnMZvlipKsByrXW17KuSc8lSczmPLDWXc7hiL0FY5+il5gp1+yhLr4/9vCdgn3re7d08KgJlsqMiQggh7mEu+Nvpbibp+AHtgV3ZWSEhhBD3HlccXcjqJJ1ky4HR2VQXIYQQ96icnCA7AdFa6yR7VsaVXFz3mrOrkO2KPfqss6tgF1F75zu7CkKIXMjW2Q3HgTr2rIgQQoh7l1IqSw97sjVBfgx0s2dFhBBC3LuyM0EqpdyVUsuVUj8qpZ5Jtd1XKbVNKfWrUqrPnepk6xDrv8DzSqlHgV+A+NQ7ZSUdIYQQdyV7O4MdMCaQzgO2KqU+1lrfwLgr1VLgI+BHYPHtDmJrggwF/jD/XCXNvqxfSCmEEEKQ7ZN0agGrtNY3lVJngNLACYzV3/yAgtiQu7K8ko4QQghhDwl/biHhzy13cwgfINz8czjga/55JfA3MBK47RrjcJsEqZS6CVTUWp+4m1rmVMOHDEy3LbR1G8Jat3VCbYQQwn5cYam51D1Ij8rN8ajc3KbXXfrQ6vSYGKAkRq+xFBBtbp8CdAT2AKuVUu+Yd6qy6nY9SNe7KMWB5r39jrOrIIQQDtG6TVtat7H88b/4g/ccWodsHmLdB9RSSv0AlAD+Mbf7AReA68A1oBCQYYLMfYtYCiGEyHmy94bJq4AGGBN1FgMjlFIPAJMwrso4CBy+0wjpnc5BdlRKXbhDGbTWWVtlVgghhCB7e5Ba6wSgcwa7bb6m/04JcrotdcGYMiuEEELkGndKkBXu1Uk6QgghHCcnr8UqhBBC2I0kSCGEEMKKnJYglwJXHFURIYQQ9zDXy48ZJ0it9TMZ7RNCCCFyOxlizYCspCOEuFe42ko6rkIWCsjAvLffSfewJTlqrZk8YRzVH6hEhbIlGDKwH3FxcRmWX7VyOQ3r1aFkcT/at2nFyROWk4Z/2PU9zYMbEljUhwerVGDMqNeJj4/P4Gj2M3JgGIe+GcOJjZNYOKYHHu5uGZbt/8Rj7PvqdS7sepOtS16gYa2KFvvvK1WU5W8N5NzO6RxbM47RQ1rjli+vvUNIR2vNxPFjqVqlAuVKBTKof9/bttXKFcupX7cWAUUL0zYsJF1bHT1yhFYtmlK8iA+P1avLhvXr7ByBdRKXISfE5SoxtW7TlgWL3rV4OFpOvh+ksNH0KZNYtHA+E6ZM4/3FH7F3z2769+lptez3O7bzTM8n6dKtByu+WYOXtzetWgSnJMDws2fp+HgYD1StytqNW5k4ZRpffv4pL7/wrCND4rX+rRjcrTGvv/U1fUYtpe6D5Vg8sZfVssN6BDPx2XbM/3QbIf3nsH3vn3y7YAgPVioBgHsBN76dP5QkrWkzaD4vTV9O7/b1eX1AqCNDAmDq5IksXDCPyVNnsHjpMvbs+Zk+vZ+2Wnbnju30eqo7Xbs/yarV6/Dy9qZ500YpbRUZGUnLZo0pXboMq9duJDSsNU90bMeB/fsdGRIgcUHOiSs3xpRVrpggldZyt6rUlFIa4ErCzUy/NjExkSoVyjBq9Die6dsfgL17dtOscQOO/nWKEiVLWpR/qvsTFMhfgA+WLgMgISGBCmWCmD13AU907c7ct2Yxf+5sjvx5knz5jNHw71Z/Q48unTh3KYaCBQtmqn7FHs18Ys2XLw9/rZvI+Le/Y8nXPwJQp1oZti99iUqhozl7Idqi/OktU5i5ZBNzl21N2bZ0Sm+uxl1n6ITPaFynEuvffZaAhv8j5qrxD3twt8Y837MZlcNGZ7p+AFF752f6NYmJiVQsV4rRYyfQp5/RVnt27ya4YT3+PPEPJdO0VfeunSmQvwAffvwJYLRV2ZIBzJm/kK7durNwwXxmzZzOsb9Okjev0Rtu1yaUUqVKM3+h49b1lbhyTlyuHpOHm5F8tNZ2zULJ37klh6zK0uv/fbs9YJ96Sg8yGx05fIgLERGEtApL2Vardh18/fzYvi39rVu2bdlMy1a3ek7u7u40btKULZs3AZAQH0/TZi1SkiNA0aLF0Fpz7lx4uuPZQ9XyQQQULcSGXYdTtu07/A+RMXEEP1LZomyRwgXx9/Nm7++nLLbvOvA39R6+DwCPAm7cuHGT+OuJKftjryXcdsjWHg4fOkRERAQhobfaqnadOvj5+bF9a/q22rp5k0VZd3d3gps2Y8umjcb+LZto0SIk5YsJIKRVGJs3b7RjFOlJXIacEFdujOmuZO9arNlCJulko4iI8wAUDwhI2ZYnTx4CA4O4cMFySdu4uDhiY2MJDAyy2B4UVIITx/8G4JURIy32JSUl8d6itylarBilS5exRwjpFC9aCIDzl2NTtmmtOXcxhuJ+3hZlY67Gc/2/REoG+MKvt7aXCSxCYDEfAH7+9QQXo64yblhbxi/8juJFCvHcU035evMvdo8lteS2CrDSVhEXIizKJrdVUFDGbXX+/Hkeqv6w5f4SJbgQEYHW2mETECSunBNXbozpbthSv/iTe4g/udcBtTG4RIJUSo0F2mutH85gfzCwDfDVWkc76n0zKyoyEk9PT4tfcABeXt5cvmx5R5XoqCgACnp5WSl7Od2x/z1zhpdffJY1q7/l/SUfkT9//uyo8h35FfLkWvx1kpIsh+KvxiVQpLDlEO+NG0ms2HiQkQNDOfx3OH+dvkDIYw8wqFsjlPlTL/pKPP1Hf8x3C4fxfM9mABz/5yL/m7HCIfEki8yorby9uXzJsq2iktuqoGVbeXt7c8ksGxUVma4tvb29iY+PJz4+Hk9Pz+wOwSqJK+fElRtjuhu2JEjP+x7B875HLLZdO2K/HrIMsWYjXz8/4uLiuHnT8vxlbGwMhQv7Wmwr7Gs8v3rlipWyhVOeJyUlMX/uW9R86H727d3D58u/pmv3J+0TgBWRsXEU9ChAnjyWH95CXh5EXUk/m/al6V9x6K9w9n45gti9c5jyQgfeX76LcxdjAKhxfyk+nNybye+u47Enp9N60HzOXohm1bzBDv2F63ebtvL1tWyr5OdXr1q2VUzMrbK+vn7p2jImJob8+fPj4eGR3dXPkMSVc+LKjTHdDVecpOMSPcjconhxY6jkXHg4JUuVAowEd/5cuMUwCoCnpyeFChUiPPysxfbw8LMUDwhMee0zPXuwcvlXDH32eUa+MRZvb8thTXuLuGQMrQYV8+HfiGjA+CAHFvPhvJn0Uou+Es9Try7Gw92NQgXdibh8hZd6N+efc5EA9O30GD/9cpwJC9ekvObAkdP8u20adaqVYU+a85f2ktxW4eHhlErVVufCwwkw//6TpbTV2fRtlVw2ICAg/f6zZwkIDHRo4pe4ck5cuTGm3MahPUillK9S6kOl1DmlVKRS6nOlVPFU+zsppQ4ppa4qpTYppUqmOcT9Sqld5v7flVKNU722slJqnVIqSikVp5Q6oJRqk2p/AaXUDKXUaaXUBaXUB0C2jjk8ULUaxfz92bjh1rVH+/buITo6msbBTdOVD27ajE0b1qc8T0hIYOf2bTRt1hyAZR99yMrlX7F02WdMnT7T4ckR4PDxcCIux9KyQdWUbXWqlaGwtwc79v6ZrvyEZx+nZ7tHiU9IJOKy8Wu2Q4uafLXBmGpe0CN/uuHaJHMmdUHPAvYKI52q1arh7+/PxlTXie3dY7ZVk/Rt1aRZczakateEhAR2bNtK0+YtjP1Nm7N580Zu3LiRUmbjhnU0a9bCjlGkJ3EZckJcuTGmu3FP9yCVEcl3QF6gG+AHTMa42/NeoAIwCBgIlAXmAuOAvqkO8y4wEriOcWfoxUB5c99SjDlN3YE4oD/wlVKqsNb6OjATeBJ4Djhqvs8LwGGyiZubGwMHDWXs6JEU8/fH28ubl198jvYdO1GqdGkiIyOJioqkfPkKAAwaPIy2YS2pUas2jzxajzmz36RQIR8eb98RgBVffUHtOnWpXqMmx80T8clKly6Dm5v9Z37euJHEoi92Mn7441yMvMKVuARmvfoEX2/+hTPno/At5ImvjycnzhjnQa7GXWfmK08AcOzEefp2egwfL/eUBLls9W6+mT+E1/q3Yu2O3ylcyJPX+rXi+JmL/Pyr4+6s5ubmxqAhwxg9aoTRVt7evPj8cDp06kzp5LaKjKR8BaOthgwdTlhIc2rWrM2j9eoze9YMCvn40L6D0VbdejzJlEnjGTKwP/0GDGLjhnVs37aVnT/sdlhMElfOiis3xnRXXLCT67DrIM3e3lagvNb6lLmtBUbC+wH4H1BKax1h7lsI3K+1Dk41Saej1vprc39X4PPka1+UUi8B67XWh83nDYBdQDngEhANDNJav2/uz4Mx1/Jm6kk6ydfk1KhZy+bYevfpR59+AwBjhuek8WP54rNPiIuPo1Voa96cPRcPDw8mTxjHlEnjLa6xXLVyOTOmTeH0qZPUrlOXt+YtpGy5cgDUfrgafxw7avU9Dx07TpmyZW2uI2TtOshkbwxuTfewOni452fd94d4cdpXJFxPZOTAMEYNCsOjxjAA8ubNw7ihbenRpi558+Zh1/6/eGn6cs5fujULNqxRNV7tG8L95QO5Gned7Xv+YOyC1fxzLipLdcvKdZBgtNWEcWP47NNlxMfFERrWhllz5uHh4cHE8WOZNGEc8Ym3/n2sXLGc6VMmcerUSerUfYR5CxaltBXAkcOHeeG5Yfxy8AAVK1VmzLgJtGgZkqW63Q2JK+fE5eiYPnjvXRa/b9sqOQcO7E+uo0Ougyz3wpo7FbXq5OzWgH3q6cgEOQx4WWtd1sq+sUAPrXWlVNveBGqnSZAltNbh5v42wOpUCTIf8AjwEFADaAWUwkiQvsAB4D6t9clU7zEPaGgtQWZloQBXdzcJ0pVlNUEKITLm6IUC7ntxbZZef2KWcW1oTl8owA1Ius3+jBcgvEMZpZQ7sBljmLUcsAYYkKrIDWuvu0N9hBBC3MMcmSCPAKVTT7xRStVXSh3COB95NxoBDYHqWutXtNbfYDmifQJIBJqlem9lvk4IIYSTKZW1hz058jKPzcAh4Aul1OtAQWA8xvnByLs89jWMZP+sUmozUB9jMg5AA+ALYCEwQyl1HWOSTh+M3uapu3xvIYQQd8kVL0VxWA9Sa30TCAHOAF8BHwN/Y8w6vVs/AmOA54FN5vu0Md/nTcALYxLQu8BEYD3GJR6vZMN7CyGEuEuu2IOUu3mkIZN0ch6ZpCNE9nP0JJ1Kr6y/U1Gr/pzeCsj5k3SEEEKIHEOWmhNCCOF0LngKUhKkEEII50t7QwRXIAlSCCGE00kPMgcZPmRgum2hrdsQ1rqtE2ojhBD2s+a71az9brVT62DLZR5X/vyJK3/97IDaGGQWaxoyizXnkVmsQmQ/R89irToyazc+PjypJWCfekoPUgghhNPJEKsQQghhhSuupCMJUgghhNNJghRCCCGscMH8KCvpCCGEENZID1IIIYTTyRCrEEIIYYUL5kdJkEIIIZxPepBCCCGEFS6YHyVBZkSWmhNC3CtcYak5VyRLzaUhS83lPLLUnBDZz9FLzdWeuC1Lr983qgkgS80JIYTIpWSIVQghhLDCFSfpyEIBQgghhBXSg8xAsS7vOrsK2S63nqsLj4p3dhWyXYCPu7OrYBeueNd44Rps6UBGHf2RqKM/2r8yJkmQQgghnM6WIVa/Bxrg90ADi20X935nrypJghRCCOF8LngKUhKkEEII55NJOkIIIUQOIT3IDCT++km6bXmKP0jegIecUBshhLAfV1hJxwU7kJIgM+JW/UlnV0EIIRyidZu2tG5juYzm4g/ec2gdXHGIVRKkEEIIp5MEKYQQQljhgvlRJukIIYQQ1kgPUgghhNPJEKsQQghhhQvmR0mQQgghnE96kEIIIYQVLpgfZZKOEEKI3EUp5a6UWq6U+lEp9Uyq7XmVUouUUgeVUs/e6TiSIIUQQjhdHqWy9MhAB2AX0BDorZRKHi1tDsQCtYAWSqm8t6uTDLFmQJaaE0LcK3LhUnO1gFVa65tKqTNAaeAE0Bj4XmudpJR66k4HkR5kBtyqP5nuYWtyHNmtNocWdefEkqdZOKwxHvkz/h0ytM2D/PZ2Ny583ocNEx+nRvmiFvt9Cubng+ebcmLJ0xxf/DRjnqzj8JvOaq2ZOH4sVatUoFypQAb170tcXFyG5VeuWE79urUIKFqYtmEhnDxxwmL/0SNHaNWiKcWL+PBYvbpsWL/OzhFYp7VmzvSJNKlbjUerleO15wcRf5u4ko16eThL3kl/8+k/jh6m1xNtqVExiAbVKzDq5eHEREfZo+q3pbVm0oSxPHh/Re4rE8Tggbdvr69XLqfBo7UJ8vfl8dat0rVXsq++/JyunTvYq9p3lBs/h64SU+s2bVmw6F2Lh6MppVIeZ3/4ht3Tn7HpkQEfINz8czjga/65KNBBKfUTMExrffN2dZIEmc1e61KTwa2r8fqHP9Fn9lbqVi7O4heaWi37TIv7GftUXaZ8uZ+Wr3/LsX+j+G5cG/x9PADj7utrxrehsFcBnp6xiYmf7WP44w8xrM2DjgyJqZMnsnDBPCZPncHipcvYs+dn+vR+2mrZnTu20+up7nTt/iSrVq/Dy9ub5k0bER8fD0BkZCQtmzWmdOkyrF67kdCw1jzRsR0H9u93ZEgAzJ81lY8+WMRrYyYx8+3F/LJ/Ly8N7Zthea0132/bzNdffZpuX9y1azzT9XHcPTxY8vkqJs9+m/17f2Z4v6fRWtszjHSmTZnIorfnM2nqdD5Y8jF7d++m3zM9rZbduWM7vZ/uQdduPfj627V4e3vTsnnjlPZK9teffzJl4nhHVD9DufFzmBtjyqo86tajdMP21H/tQ5seGYgBSpp/LgVEm3+OB34GGgGPKqUq3a5OMsSajfLlzcPAsGq88dFuvv35FACD5m1n+7QOlChSkLOXr1mU79GkIks2HeWz7X8B8MK7u2hf7z5Capfm4y1/0LJGKUoX86bZa99wPfEmPxw5j7eHG5VL+eIoiYmJvLNwARMmTaVde6P3sOjdxQQ3rMe///5LyZIlLcovfHs+nTp34bkXXgTg4RrLKFsygG+/WUXXbt354rNPcffwYOG775M3b17qPvIIe3b/zOL336VmrXccGteyxe/wv1HjCWndDoCpby2kc1gTzoX/S2CQZVyHfjvIUx3DuBIbY/V4O7duIioqkhnz3qWQT2EA/P0DaNP0UU78/SflK1a2azzJEhMTeWfR24yfOIXH2xnttfDdD2jSqD5n//2XEmna652FC+jYuQvPPm+01wcffsx9pQNZ/e0qunTtDkDlCmX498wZACpWckwcaeXGz2FujMmF7ANqKaV+AEoA/5jbfwGuaa0TlVJXuEMnUXqQ2ahqaV8CfD3ZsP+flG37/rpA5NUEgh8qka68R/58XIn/L+V5UpIm7noingWM3y2dHivPmj2nuJ54axRg7re/MXTBDjtGYenwoUNEREQQEhqWsq12nTr4+fmxfeuWdOW3bt5kUdbd3Z3gps3YsmmjsX/LJlq0CCFv3lvnxkNahbF580Y7RpHen8cOc+niBYKbh6Rse6hGbQr7+vHjzu3pyt9XvhJfrN7M2h178S8ekG7/1aux1H30sZTkCOBX1BguP3f232yvf0YOHz7EhYgIQlrdaoNatY322rbNSntt2URIq9CU5+7u7gQ3acaWzZtStq38Zg17DvxG6zaP27fyt5EbP4e5Maa7kXqINTOPDKwCGmBM1FkMjFBKPQB8CfRUSh0Azmmtj92uTtKDzEbFfT0BOB996xyC1nAuMo7ihT3TlV/10wmGtHmQb346ydEzUfQLeYAihdzZfND4Qi1VzIuDf1/knWeDaVa9JNeu3+Cr7/9m2lcHSLyR5JCYIiLOAxAQcCsp5MmTh8DAICIuRFiUjYuLIzY2lqCgIIvtQUElOHH8bwDOnz/PQ9UfttxfogQXIiLQWjvsYuGLZt2L+VvG5V88kEsXL6Qr71mwIJXvrwqAW/786fZ37t6Tzt1vDWNqrfno/UXkz5+fKlUdNyQecd5or+Jp2isgMIgLEdbbKzAwbXsFcfz48ZTnVatWA6Cwry8x0dF2qvnt5cbPYW6M6W5kZ/W01glA5wx2t7H1ODkmQSqlPgQKa63bZ/NxTwFvaa3futtj+Xm7cy0hkaQky3NOV+MTKVLIPV35N1f8Qlidsux+64mUbQPnbuP4OWMYr3hhT4Y9/hALVv9O92kbKePvzYx+DShYIB8jPvz5bqtrk8jISDw9PS1+lQJ4eXtz+dIli21RUcaElIIFvSy2e3t7c8ksGxUVSUGv9Pvj4+OJj4/H0zP9Dwl7iImKwsNKXAW9vIiKvHxXx46KvMz0iW/w5bIPeen1sRQt5n9Xx8vUe0dZby9vb28uX7Zsr2izvbzStIeXlbLOlhs/h7kxpruhcL0EnmMSZE4QeSWBgu5u5MmjLJJkIc/8RF29nq78e881wcvDjc6T1hF++RqPVQ1iyjP1OB8Vx8YDZ/jvxk22/vovry35CYC9f17Ao0A+Zg94jNeX/owj5n74+fkRFxfHzZs3Lf4hx8bG4OtreS40+fnVq1cstsfE3Crr6+vH1Svp9+fPnx8PDw97hGCVj68v8VbiunIlFp/ChbN0TK013674gnGvv8TNmzeZPGsBXZ7snT0VtpGvr/X2iomJoXCa9kp+fiVNe8TGxFC4sOPOc9siN34Oc2NMd8PBk/NtIucgs1FElDG0GuR365eaUhDo58n5KMup20F+BXmqaWX6vbWVNXtOc/D4JeZ9+xsfb/mDoW2NIbmI6HgOn460eN2xM1EUdHejiHf6Hqk9FDfPt4WHh6dsS0pK4lx4OAEBgRZlPT09KVSoEOFnz1psDw8/m1I2ICAg/f6zZwkIDHToEFAx/+IARJw/l7ItKSmJC+fPUczKOUZbTB03kheH9KFx05Zs+fk3uj71jMOHtZKHVs+laa/z527TXuFp2yucgEDLss6WGz+HuTGm3MZlEqRSSiulmiilPlRKXVRKnVVKvWCl3BCl1F9KqStKqRVKqUKp9vVSSh1RSiUopS6Z+wNS7a+klNqglIpRSh1WSlmfT51Fh/+JIiI6jpY1S6dsq1PRn8IFC7DjN8sProc5ESftcKzWGi93NwB+Onqe2hUth+eq31eUyCsJXIpNyM6qZ6hqtWr4+/uzMdX1VHv37CE6OprGTdJfvtKkWXM2bLhVNiEhgR3bttK0eQtjf9PmbN68kRs3bqSU2bhhHc2atbBjFOlVqlKVIkX92bFlQ8q2Xw/sIzYmmvqPBWf6eD9+v533336LV0dPYtbCxQ4dVk2tatVqFPP3Z2OqNti312iv4GAr7dW0ORs3rE95npCQwI7tW2narLlD6mur3Pg5zI0x3Y1snqSTLVxtiHUO8D7wDvAcMEsptUlrfcjc3wS4DjwF1ATeMstNMGcoLQFmAl9hXPsyA5gM9FFKeQM7gWMYJ2kLmq+3nEt9F27cTGLRmkOMf/oRLsbEcyU+kVkDGvD1jyc4c+kqvl4F8PUqwInzsRw/F8MPh8/x3vNNGf3Rbs5HxdHggQAGta7G8+/sAuDDTUd5oUN1ZvStz6fb/6RySV/GPVWX2V//kl1VviM3NzcGDRnG6FEjKObvj7e3Ny8+P5wOnTpTunRpIiMjiYqMpHyFCgAMGTqcsJDm1KxZm0fr1Wf2rBkU8vGhfYeOAHTr8SRTJo1nyMD+9BswiI0b1rF921Z2/rDbYTElx/V034G8OWkMRYoWo6CXN+Nff4nQth0IKlmK6KhIoqOiKHtfeZuO992qrwgIDKJFaBtOn7S8eDsgMAh3Bw1xubm5MWjwUMa88TrFivnj5e3Nyy88S4eOnSllpb0GDRlGm9AW1KxZi0fr1eetWW9SyMeHdu07OqS+tsqNn8PcGNPdcMVOrqslyB1a67kASqnjQFegApCcIBOAXuYMpd1KqceB5G+wJOAVYLa5OsIepVQroKK5vwdGUuyotY403+MpYK+1ilzfMcXmSuct8xj5yjYEYOqXB3DLl4dpferjUSAf6/ad5sV3jYQ3pM2DjOpeG492iwDoMmU9b/Sow4KhjShSyJ2/zsbw7KLv+XjLH4Ax+7XFiG+Y0a8BPZtX4XxUHFO/PMCC7363uW7Z4bXXR5GYmMir/3uR+Lg4QsPaMGvOPADenj+XSRPGEZ9o9IQbNQ7mo08+Z/qUSUyZNJ46dR9h05YdKedA/Pz82LB5Oy88N4y2YS2pWKkyK1at5uEaNRwaE8CwF1/jRmIik0e/Rnx8HE1bhjJm8iwAlr73NnPfnMzxC3deWQfgfPhZzp8Lp3m96un2ffL1eh5t0Chb6347r44w2uu1V14iLj6O0LDWzJxttNfCBXOZPHE8164bs6AbNQ5m6bLPmD51MlMnT6B2nUfYsGm7S56zyo2fQ0fH9MF777L4fcevkmOL26yr6jTK0at8ZEQppYEntdafptnWQWu9ypzFGqS1bplq/3Lgqta6t/m8GPAI8BBQF2gF/Ky1DlZKzQEe1lo3TvV6hbFw7RvJs1jN98T98YV2jNY5olYMcnYV7CI8Kv7OhXKYAB/HnGN2NEcvkyiyzsPNaCuttV0bLfk7t8P7+7L0+q/71QbsU09X60He6ed6hvuVUvWBb4EDwDrgTeAccL9Z5EYGL73tWnxCCCHszwU7kC6XIO/GMGCz1rpb8gal1PBU+48CA5RSfslDrBg9TR8H1lEIIYQVrjjT1mVmsWaDa0AjpVSIUqqpOaTaESillLoP+Ay4AqxQSjVUSrUAlpnbhBBCOJFSWXvYU25KkKOBX4GVGDNhr2Och8wLvKq1voZxL7D/gNXAXIxh2D1Oqa0QQogU2XzD5GzhMkOs1k6wpt6WPBEnzf7Oqf58DghNWwYom6rMX0BImv1LM19bIYQQjhZ+cAfhv+x02Pu5TIIUQghx77KlL1iiRmNK1Ghsse3Ejq/tUyEkQQohhHABrjhJRxKkEEIIp3PFS2QlQQohhHA66UHmIIm/fpJuW57iD5I34CEn1EYIIexnzXerWfvdamdXw+VIgsyAW/UnnV0FIYRwiNZt2tK6TVuLbYs/eM+hdXDBDqQkSCGEEM4nQ6xCCCGEFTJJRwghhLDCFXuQuWmpOSGEECLbSA9SCCGE07le/1ESpBBCCBdg74XHs0ISpBBCCKdzwfwoCVIIIYTzueIkHUmQGZCVdIQQ9wpZScc6SZAZkJV0hBD3CllJxzpJkEIIIZzOlkk6p/Zt4/S+7favjEkSpBBCCKezpQdZrk4TytVpYrHt6ObldqqRJEghhBAuQCbp5CTn/nJ2DbKd1trZVbCLIl75nV0FIUQuJAlSCCGE07niuqeSIIUQQjidDLEKIYQQVrji7a5csVcrhBBCOJ30IIUQQjidK/YgJUFmIPHUpnTb8hS+j7yFyzuhNkIIYT+usNScnIPMQdzKtnB2FYQQwiFcYak56UEKIYQQVrhgB1Im6QghhBDWSA9SCCGE09myWLmjSYIUQgjhdK44nCkJUgghhNO5YAdSEqQQQgjns2WI9c+ft/LX7q0OqI1BEqQQQogcodKjTan0aFOLbQfXf2m395MEKYQQwulkiFUIIYSwwhUXCnDFiUMuIfHUpnSPm9HH7/i6kf1bcmjFCE6sHcPCUV3wKOCWYdn+neqx77OXubBtElvfG0bDmpbL2CmlGNU/hF++fJVzWyaydMJTFPEpeNexZZbWmonjx1Lt/oqUKx3EoAF9iYuLy7D81yuWU/+R2gQW8+Xx1q04eeKExf6jR44Q2rIZAUUL07D+I2xYv87OEVintWbKxHHUqFaZyuVKMmxQ/9vG9c3XK2hcvy6lA4vQ8fFQTp20jOvjpUuoW6MagUW8qV/nYZZ/+bm9Q7BKa82kCWN58P6K3FcmiMED79BeK5fT4NHaBPlbb69kX335OV07d7BXte8o+XNYtUoFypUKZFD/28e1csVy6tetRUDRwrQNC7H6OWzVoinFi/jwWL26TvkcukpMa75bzdBBAywejpZHqSw97Fonux49B3Mr2yLd407rsL7WpzmDuzzG6/NW02fMp9StVobF43pYLTusW0MmDm3D/M+/J2TwQrbv+5tv5/TnwYqBKWWmPd+Wbq1q8tqcb+k75lMeqhTEoje6Zmuctpg6eSKL3p7PpCnTWfzhx+zdvZu+vXtaLbtzx3Z6Pd2Dbt17sGr1Wry8vGnRrDHx8fEAREZGEtI8mFKly/Dt2g20Cg2jS6f2HDiw35EhATBj6iTeXbSA8ZOm8e7ipezbu5uBfXtZLfv9zu307fUkT3TrzvJV3+Hl5U1YiyYpcW3fupnnhg6k74BBbNj6PV269aBf76f4YddOR4YEwLQpZntNnc4HS4z26vdMxu3V++kedO3Wg6+/XYu3tzctm99qr2R//fknUyaOd0T1MzR18kQWLpjH5KkzWLx0GXv2/Eyf3k9bLbtzx3Z6PdWdrt2fZNXqdXh5e9O8aSOLz2HLZo0pXboMq9duJDSsNU90bMeB/Y79HLpKTK3btGXBonctHo6mVNYedq2T1tq+75DDKKU0gHudFzP1unx58/DXd28wftF6lnyzG4A6VUuz/YPhVHp8ImcvxFiUP71+LDM/2srcT299gS6d8BRX468zdPJX+Pl4cmLNGIL7zuWXP84C0KhWed4YEEK7594nLuG/TMcW+cObmX5NYmIiFe8rzeix4+nTtz8Ae/bspknD+vxx/DQlS5a0KN+j6xMUKFCAJR8tAyAhIYFypQKZM+9tunTrzsK35zN75gyO/nmCvHnzAtCubRilSpVi/tvvZLp+AP/dSMpSXFUrlmXk6HH06tMPgH17dtOiyWMc+uMkJdLE1bNHFwoUKMB7Sz5OiatSuRLMmrOAzl26MaBPT65evcqnX65MeU1o88Y8WL0G02e+len6ueXN2m/XxMREKpUvzegx43nGbK+9e3bTpFF9/vj7dLq4nuz2BPkLFGDJ0lvtdV/pQN6a9zZdunYHoHKFMvx75gwAbdq244vlX2epbgB5sjiOlpiYSMVypRg9dgJ9+pmfw927CW5Yjz9P/JPuc9i9a2cK5C/Ahx9/AhhxlS0ZwJz5C+narTsLF8xn1szpHPvr5K3PYZtQSpUqzfyFWfsc5raYPNyMttJa2zUNJX/njt/0V5ZeP7pFRcA+9ZQeZDapWj6QgCKF2PDj0ZRt+46cITI2nuA6FS3KFvEpiL+fN3sP/WOxfdcvx6n3UFkAWjesytkL0SnJEWDn/uO0GPh2lpJjVh0+dIgLERGEtApL2Va7dh38/PzYvnVLuvJbt2wipFVoynN3d3eCmzRj82bj7ihbN2+meYuWKf+AAVq1CmXL5vR3T7GnI4cPceFCBC1CbtW1Zu06+Pr5sWN7+mnk27dutijr7u5O4+CmbDXrXczf3+LvCKBoMX8S0vTE7O3w4fTtVctsr23bbG+v1O2x8ps17DnwG63bPG7fyt/G4UOHiIiIICQ01eewzm0+h5s3WZR1d3cnuGkztmzaaOzfsokWLUIsPochrcLYvHmjHaOwlBtjuht5VNYe9iSTdLJJ8SLeAJy/fCVlm9aacxdjKO7nbVE25mo81/+7QcnihS22lwn0I7CYDwClAnz5NyKa//VuRo/QWhQq6M62fX/x+tzVXIi8at9gUomIOA9AQEBAyrY8efIQGBjEhQsRFmXj4uKIjY0lMCjIYntQiSCOHz+ecryHqle32B8YVIILERForR12y5sLZlzF08QVEBjExQziCggMtNgeGFSCk8f/BmDS1Fu98+vXr/Pzj7vYsX0ri9770E4RWBdxPuO4LkRk0F6Bador6FZ7AVStWg2Awr6+xERH26nmt3e7z2FEBu0VlPZzGFSCE2Z7nT9/noeqP2y5v4RjP4e5Maa7oXC9+kkPMpv4FfLkWvx1kpIsh6yvxl2nSGHLiTU3biaxYvMvjOwfwgP3BeCWLy9tGlVl0BMNyO9m/PorXsSb+tXL8ViN+xg+dTl9x31GxVLFWDGzb5aHqbIiKjIST09Pi1+lAF7e3ly+dMmybFQUAAW9vCzLet0qGxUViVea/d7e3sTHx6c772VPUVFRVuPy9vLi8mXLuKLNuLwKet2x7M8//UBx34K0ax1C125PEpbmFkL2FhVlvb28vb0zjitte1kp62yRWfkcpm0vb28upfocpv2cOvpzmBtjuhvSg8zFImPjKOhRgDx5lEWSLOTlTlRs+llpL81cxfwRndn76UvkyZOHv/+5yPsrf6JNI+PXemLiDWKvXaf7q0tThlR7nr3MsVUjqXV/KfYe/ifdMe3B18+PuLg4bt68afEPOTYmhsK+vpZlzedXr1yx2B4bG5Oyz9fXjytp98fEkD9/fjw8POwRglW+vr7W44qNpXBhy7iS47xyNW1csen+Dh6qXoMf9hzkj2NHGf36qxT29WXk6HF2iiI9X1/r7RVjpb1S4rLSHmn/DpzNL6PPYarPVrKUz2Ga9oqJsfwcpv2cxjj4c5gbY7obcplHLhZhDq0GFfVJ2aaUIrCoD+cvXUlXPvpKPE+9/jFFG79O2dCxPNh5KhejrvLPeeOX4vnLVzh59pLF+cbT4ZHEJyRSKqCwfYNJpXhxY/jnXHh4yrakpCTOnQtPN+To6elJoUKFCD971mJ7+NlwAgICU44XHp5mf/hZAgIDHToE5J8c17n0cRUPsB5X6r8DMOqdXHbvnp+JvHwZT09PqlZ7kI6du/DqyNF88tGH9g0kjeSh1bTtdf7crTZIltJe6dojfds6W/LnMDzt5zD8NnGl/RyGn00pGxAQYOVz6tjPYW6MKbe5JxKkUmq9UirdNC6l1B6l1NTseI/Dx88RcfkKLetXSdlWp2opCnu7s2Nf+tlZE4a2pmfbusRfT0xJrh2aVeerjQcB+Om3U1Qs7U+hgu4pr6lUxh8Pdzf+PH0xO6psk6rVquHv78+GDbeup9q3dw/R0dEEBzdNV75J0+Zs3LA+5XlCQgI7tm+labPmADRtZkwAuXHjRkqZDRvWp+x3lAeqVqOYvz+bU9V1/749xERH0yi4SbryjZs0Y/NGy7i+37GNJk2Nevfs3oUvPv/E4jVXr14hbz7HDtJUNePamE3t5SqSP4cbU13Xt3ePEVfjJlbiatbc4jObkJDAjm1badq8hbG/aXM2b95o8TncuGEdzZq1sGMUlnJjTHdDKZWlhz3dK0Osi4BPlFKvaq2jAZRSdYDaQLZcWHjjZhKLvtrF+KFhXIy6ypW468x6qT1fb/2NMxHR+BbywLeQJyf+vQzA1fjrzHypPQDHTp6nb4d6+BR056tNvwDww8ET/H3mIp9N68Wk9zZSIH8+pj7/OBt/Osahv89lR5Vt4ubmxsDBQxkz6nWKFfPH29ubl55/lg4dO1OqdGkiIyOJioykfIUKAAweOozWrVpQs1YtHq1Xn9kz36SQjw/tOnQEoGv3J5kyaQJDBg2g34CBbFy/jh3btrJj188Oiyk5rv4DhzBuzEiKFvPH29uLV156nnYdOlGqVGmiIiOJiorkvvJGXAMGD6V96xAerlmLRx6tx9zZMylUyIe27YwL50Nbt+WtN6cTFFSCChUrcej335gxdRKDhgx3eFyDBg9lzBtGe3l5e/PyCxm316Ahw2gT2oKaNY32emuW2V7tOzq03nfi5ubGoCHDGD1qBMX8jc/hi88Pp0OnzpS2EteQocMJC2lOzZq1jc/hrBkU8vGhvfk57NbjSaZMGs+Qgf3pN2AQGzesY/u2rez8YbfE5CSuOMR6T1wHqZTKB5wCpmut55rblgDFtdZhacpm6TrIZG8MCKF7aC083N1Yt+soL765koTrNxjZvyWj+ofgUfclAPLmzcO4waH0CKtN3jyKXQdP8NKbX1vMgvXz8WTuq51oWLM8N24k8d33h3lj/hpiryVkqW5ZuQ4SjNm4E8aN4fPPPiEuLo7QsNbMemseHh4eTBw/lskTxxP3361rEb9esZxpUydz+tRJ6tR9hLnzF1K2XLmU/UcOH+bF54fzy8EDVKxUmdFjx9OiZUiW6gZZuw4yOa7JE8by5eefEh8XR0hoa6bPmoOHhwdTJo5j2uQJRMfd+jX+zdcreHPaFE6fPkntOnWZNfdtypY14oqNjWXCmFGsWf0NkZGXKVO2HL369GPg4GHpJmHYIqvXQSbHNXH8GD7/9BPi4o32mjnbaK9JE4z2unY9VXutXM50s71q10nfXskG9HuGmOhop1wHCbc+h599uoz4uDhCw9owa86tz+GkCeOIT7z1fbZyxXKmT5nEKfNzOG/BonSfwxeeG5byORwzbsJdfQ5zW0yOvg5y5o47r1RmzUuNjQVcUtdTKeUOLAOCgPe01kvSvGcX4D6t9W1HEO+JBAmglBoN9ADuB/yAf4EuWuvVacppAOVZ3OZj5y32IPn8H8q+ytpJVhOkq8tqgnRld5MgXZkjZ2CL9D54710Wv2/bKjnJq1s5KkHO3ml9icM7eaHRfUC6BNkdKA7MA7YCzbTWN8x9bsBBYNmdEuS9MsQK8D4wGmgK1AAuAmszKlyg6pMOqpYQQjhG3/4D6NvftnVWk3uQOVQtYJXW+qZS6gxQGkjOwAOADbYc5J5JkFrrcKXUN8Bw4EHgXa31TSdXSwghBJbnIH/89jN+/PazuzmcD5A8PTgc8AVQSnkDTYAFwCN3Osg9kyBNC4FNwA3gAyfXRQghhCn1hNQG7brToF13m1737GP3WdscA5TE6DWWAqLN7S8Cs4GMb7OUyr2WILdi/IX9qrV23FRQIYQQt5Une5ea2wfUUkr9AJQAkldWqYzRgywMFFJK7dFap1982XRPJUitdZI5Hu2YZWiEEEI4wyqMWaxdgHeAEUqp5VrrHgBKqWDg0dslR7jHEqQQQgjXlJ3X/GutE4DOt9m/Hdh+p+NIghRCCOF0rngF0D2XILXWwc6ugxBCCEt5XHC92HsuQQohhHA9LpgfJUFmJPFU+jvc5yl8H3kLl3dCbYQQwn7WfLeatd+tvnPBe8w9s9Scre52LVZXJkvN5Ryy1JxwNkevxfr+7tNZen2/R8oA9qmn9CCFEEI4nQyxCiGEEFa44piJJEghhBBOZ++bH2eFKyZtIYQQwumkBymEEMLpXK//KAlSCCGEC5CFAoQQQggrbEmPB3Zu4uD3m+1el2SSIIUQQjidLR3IWo1bUKtxC4tt277+1E41kkk6QgghhFXSg8yALDUnhLhXuMJSc654mYcsNZeGLDWX88hSczmHLDWXczh6qbnPD/ybpdd3q1kSkKXmhBBC5FKu2IOUBCmEEMLpXC89yiQdIYQQwirpQWbg2LpJzq5CtvN/+iNnV8EuLi7r5ewqZLsbN3PfeVWAPC7ZT7h7CYk3nV2FHE+GWIUQQggrXHE4UxKkEEIIp5MepBBCCGGF66VH1+zVCiGEEE4nPUghhBBO54IjrJIgMzLihaHptjULCaN5q9ZOqI0QQtjPujWr2bB2jVPr4IoznGWpuTSSlz06dSne2VXJdtWGfuHsKtiFXOaRc+TLpUvo5cbLPHw9jf6To5aaW/37+Sy9vu2DAYB96pk7P61CCCHEXZIhViGEEE6nbBhi3b19A3u2b3RAbQySIIUQQjidLZN0Hm0SwqNNQiy2bVixzE41kgQphBDCBbjiJB1JkEIIIZzOFS/zkEk6QgghhBXSgxRCCOF0rtiDlAQphBDC6WyZxepokiCFEEI4XR7Xy49yDjIjI14Ymu6xef3tl2LSWjN72kQa16lKnQfK8cpzg4iPi7vje73+0nA+WDQvw/0H9u0mNPiRTMeQnUZ0rs4vb3Xgr4VPsGBgfTzy582w7ODQ+zk4uz3hS3qwdnQID5fzs9j/dHAF9s1sR8TSJ/l5+uN0rl/O3tW3SmvNxPFjqVqlAuVKBTKof1/ibtNeK1csp37dWgQULUzbsBBOnjhhsf/okSO0atGU4kV8eKxeXTasX2fnCKzTWjN5wjiqP1CJCmVLMGRgv9vGtWrlchrWq0PJ4n60b9MqXVw/7Pqe5sENCSzqw4NVKjBm1OvExzt+panc2F5aa6ZOHEfNapWpUq4kwwf1v21M33y9guD6dSkTWIROj4dy6uSJDMsO6NOTJ7t0tKke69as5vmhgywejqay+J89SYLMwJTZC9I97rQO67yZU1n6/kJGjJ3MW4sWc3DfHl4Y0ifD8lprdm7bzMovP8mwzPlzZ5k6dmSW48gOr3R8iEEhVXjjk/30W/A9dSoU472hDa2W7d20IqO71mDayt8IHbeeP87G8M3IlhTzcQcguFog8wbU472Nf9B89Fq+2HWCJc82okGV4o4MCYCpkyeycME8Jk+dweKly9iz52f69H7aatmdO7bT66nudO3+JKtWr8PL25vmTRulJIrIyEhaNmtM6dJlWL12I6FhrXmiYzsO7N/vyJAAmD5lEosWzmfClGm8v/gj9u7ZTf8+Pa2W/X7Hdp7p+SRduvVgxTdr8PL2plWL4JS4ws+epePjYTxQtSprN25l4pRpfPn5p7z8wrOODAnIne315tRJvLtoAeMnTeOdxUvZt3c3g/paXzpx187t9Ov1JE90685Xq77Dy8ub1i2aWP2x8tUXn/HV55/aXI/Q1m15a8Eii4eQtVjTyeparImJidSrXpGXXhtN955GUjy4fw8dWwXz469/EhhU0qL8oV8P0r1DKFdiYwB4Y+J0+g4ablHmiTbN2PvzjwDcX+0h1m3fnbWgTFlZizVfXsWxBU8w8cuDfLj1LwBqVyjKlvFh3D9sOeGRlr92149pxcGTlxnx0V4A8ijF34ueYPSnB1i242/eG/oYXu5udJ+5LeU1G8a24teTkbyydE+W4srKWqyJiYlULFeK0WMn0KdffwD27N5NcMN6/HniH0qWtGyv7l07UyB/AT782Pgxk5CQQNmSAcyZv5Cu3bqzcMF8Zs2czrG/TpI3r9G7btcmlFKlSjN/4TuZrl9W12JNTEykSoUyjBo9jmf6GnHt3bObZo0bcPSvU5RIE9dT3Z+gQP4CfLB0WUpcFcoEMXvuAp7o2p25b81i/tzZHPnzJPnyGWdkvlv9DT26dOLcpRgKFiyYqfpldS1WV2+vrKzFmpiYSLWKZXl99Dh69ekHwL49u2nZ5DF+/+Nkurbq1aML+QsU4L0lH6fEVLlcCWbOWUDnLt1Syv1z+hSN69ehePEAyleoyCdfrsx03cDxa7FuPXYpS69vWqUoIGuxurQ/jh7m0oUImrS4tcpD9Rq1Kezrxw87t6crf1+FSixfs4UN3+/Dv3iA1WNOe2sRG77fx1PPDLBXte/ogVK+FC/swYaDZ1O27T9+iair1wmuFpiuvHv+vFyNT0x5nqQ1167fwKOA8SV0MSaBDQf/tXjNxZiE2w7Z2sPhQ4eIiIggJDQsZVvtOnXw8/Nj+9Yt6cpv3bzJoqy7uzvBTZuxZZOx7NXWLZto0SIk5csWIKRVGJs3O25ZLIAjhw9xISKCkFa36lqrdh18/fzYvi19XNu2bKZlq9CU5+7u7jRu0pQtmzcBkBAfT9NmLVKSI0DRosXQWnPuXLgdI7GUG9vr6OFDXLgQQYuQW3//Nc222rl9a7ry27Zutijr7u5O4+CmbDPbCuDmzZsM7NuLZ/oOoGbtOvYNIJvJEGsudvFCBADF/G8luzx58lA8IJBL5r7UPAsWpPL9Val8f1Xc8ue3esz7KlSk8v1VKVK0mH0qbYPihT0AiIi+1aPWGs5FxeFvDpum9u3u0/RqUpEHy/jiljcPA0OqUMTbna2/GV+mry/bl9ITzZ8vD42rBdC4WiDrDvyb7lj2FBFh3DkgIMCyvQIDg4hI015xcXHExsYSFBRksT0oqAQXzLLnz58nMO3+EiW4EBGBI0dpkuMqbiWuCxcuWJRNjiswMH1cyZ/nV0aMZNF7i1P2JSUl8d6itylarBilS5exVxjp5Mb2yqitAgKDUuqZLC4ujiuxsQQGWv4oDUwVE8DsGVOJj4vntVFj7Fhz+8ijsvawJ5nFmk1ioiLx8PS0+EUKUNDLi6jIy06q1d3zLZifawmJJKX50riacIMi3ukT5KxvDxFaqxQ/Tns8ZduQRT9w/PwVi3KPVirGpvHGL/x3Nhxj7f4zdqh9xiIjI/G00l5e3t5cvmQ51BMVFQVAwYJeFtu9vb25ZJaNioqkoFf6/fHx8cTHx+Pp6ZndIVgVlVFcXt5cvmwZV3RyXGnqbZRN/5n998wZXn7xWdas/pb3l3xE/gx+2NlDbmyvqKioDNrKK+O2Kpi2rbyINMvu27ObuW/NZPOOHx3aNtlFLvNwEUqpYCBYaz02u47p4+tHfFwcN2/etPjAX4mNpVBh3+x6G4eLuvYfBd3dyKOURZIs5OlG9LXr6cq/M7gBXu756DpjK+GR12hwfwATn6rN+eh4Nv1ya5j211ORPPK/b6hSsjATn6xF1NXrTPrqF0eEBICfnx9xVtorNjYGX1/L9kp+fvWqZZKPiblV1tfXj6tX0u/Pnz8/Hh4e9gjBKt/bxFU4zeewcHJcaeptlC2c8jwpKYm3589l/JhRFPLx4fPlX9O6zeM4Um5sL19f3wxiis24ra6mbatYCvv6Eh8fz4A+PRk7YQqVKlexf+XvETLEmk2K+RuzMCPO3zovk5SURMT5cxmeY8wJkodWA/1ufWkoBQG+npyPtpzIFOjrSY/GFRjw9i7W7j/DLycjWbD2CJ9s/5vBre4HoE6Fovh5FSD+v5scORPNyp9OMWX5rzwdXMFxQQHFzTYJD7dsr3Ph4QQEWA5jeXp6UqhQIcLPnrXYHh5+NqVsQEBA+v1nzxIQGIhy4BIhyXGdSxPX+XPhFsOTkCqu8PRxFTfjSkpK4pmePRjxykv06T+Qg78fc3hyhNzZXiltdc5aW6WPybtQIYv4Ac6ZbXXxQgQnTxxnxP9eIMC3IAG+Bfni02WsW7OaAN+C/LBrp/0DuktKZe1hT/dUglRK1VVKHQM+AoYppY4ppYZlx7Er31+VosX82Z7qJP8vB/YSGxNN/YaNs+MtnOLImSguRMfT8uESKdtqly9KYc/87DhkeQdwT3MiTlKaUzga8HI3BiuWvdiErg3vs9jv5eHGjbQvsrOq1arh7+/PxlTXvu3ds4fo6GgaN2marnyTZs3ZsOFW2YSEBHZs20rT5i2M/U2bs3nzRm7cuJFSZuOGdTRr1sKOUaT3QNVqFPP3Z2Oquu7ba8YVnD6u4KbN2LRhfcrzhIQEdm7fRtNmzQFY9tGHrFz+FUuXfcbU6TPx9va2fxBW5Mb2ut9sq82p/v7379tDTHQ0DYObpCsf3KQZmzemaasd2whu2pzAoBLsPniI73cfYOfP+9n5835CW7elYeNgdv68nxo1azskpruhbHj8sHU900c9b/Gwp1wzxKqUKgCMBzoDAcB+4HWt9a7kMlrrPUAVewyxurm50bPvIKZPHE2RYsXw8vJmzGsvEvZ4B0qULE10VCTRUVGUva98dr2lQ9y4qXln4zHGdqvJxZgEriQk8mbvR1i1+zT/Xr6Gb8H8+HoV4ETEFY6fv8KPxyJ4Z3ADxn5+gPNR8dS/vzgDWlbhpSU/A7B2/xlefLwa4ZFx/BUew4Nl/Hi1w0O8vf6oQ+Nyc3Nj0JBhjB41gmL+/nh7e/Pi88Pp0KkzpUuXJjIykqjISMpXMHq2Q4YOJyykOTVr1ubRevWZPWsGhXx8aN/BuBC7W48nmTJpPEMG9qffgEFs3LCO7du2svOHu7s0JytxDRw0lLGjRxpxeXnz8ovP0b5jJ0olxxUVSfnyRlyDBg+jbVhLatSqzSOP1mPO7DcpVMiHx9sbca346gtq16lL9Ro1OX78b4v3Kl26DG5ubg6LK7e1l5ubG/0HDmH8mJEUK+aPl7cXr770PO06dKJUqdJEmW11n9lWAwYPpX3rEGrUrEXdR+sxb/ZMChXyoW27Dri5uaUbWi3k44PWOscMueaxoTvYqFkojZqFWmxb/eVH9qpS7kmQwHtAVWAIcBl4BtiilHpYa53pb9+2zRrYXLZ7zz706NWX4S+9xo3ERCa+8Srx8fE0axHKuKmzAFjy7tvMmTEp09dXuoLpK3/DLW8eJj9dB88C+Vh/4AwvLzGuWRwUej+vd34Y725LAej+5jZGPvEwc/vXo4i3O3+fi+WFD35m2Q7jy3X0p/u5maSZ1rMOft4FOH3hKtO//o1F6485PK7XXh9FYmIir/7vReLj4ggNa8OsOcaKRm/Pn8ukCeOITzR6to0aB/PRJ58zfcokpkwaT526j7Bpy46U81V+fn5s2LydF54zEk7FSpVZsWo1D9eo4fC4XhkxksTERF5/5WXi4uNoFdqaN2fPBWDRgnlMmTSeKwnGdXsNGwfz4cefMmPaFKZNnkDtOnVZt2lbSlxnz57lj2NHebhq5XTvc+jYccqULeuwuHJje738mtFWI197mfi4OEJCWzNt1hwA3nl7HtMmTyAqzujlPtYomA8++pSZ06YwfcpEatepy5qNWzN1zvTDD95j6eL37RLL3XK9KTq5ZKEApVRZ4ARQTmt92tymgB+B1Vrryea2R4C0y9bM1VrPTXWsLC0UkBNkZaGAnCArCwW4uqwuFODqsrpQgKvLykIBrs7RCwX89FdUll5fr6Ixgcke9cwtPchqGD9A/khzgj0/kNI10VrvBhw7G0QIIcSduWAXMrckyHxAElALSPtT7prjqyOEECIz5DpI+zmKMSPXy+wlopRyA2YCqwHHXoUuhBAiU1zxhsm54oSA1voP4BvgI6VUa6VUA+BdoBfwp1MrJ4QQ4o5suczD2sOeckWCND0FbALeBjYAZYEWyZN2hBBCiMzILUOsaK2vAsPMhxBCiJzEBYdYc02CFEIIkXPJJB0hhBDCClecpCMJUgghhNO5YH6UBJmRES8MTbetWUgYzVu1dkJthBDCftatWc2GtWucXQ2XkyuWmstOstRcziNLzeUcstRczuHopeb2n4rJ0utrlfUBLOuplHIHlgFBwHta6yXm9oLAKsAf2AMM0LdJgrnz0yqEECJHUVn8LwMdgF1AQ6C3Uip5tLQTsA14GONOfLe9K4UkSCGEEE6XzTdMrgXs01rfxFhJrbS5/S/gM7PXeD6jFyeTc5BCCCFcyopPl7Di0w/v5hA+QLj553DAF0Br/ZMyPIXRe5xwu4NIghRCCOF0qTuDnXs8Q+cez9j0uofLFLK2OQYoiXEbxFJANKTcBnEaUBxor7VOvN2xZYhVCCGE82XvYqz7gFpKqbxACeAfc3sH8/+9tdZX7lQlSZBCCCGcLpsn6azCGELdBSwGRiilHgDqAs2AbUqp7Uqpx25XJxliFUII4XTZuZKO1joB6Gxl12vmwybSgxRCCCGskB6kEEIIp5Ol5nIQWWpOCHGvcIml5lwwQ8pSc2nIUnM5jyw1l3PIUnM5h6OXmjv079Usvb5aSS/APvWUHqQQQginc8XbXeXOn3NCCCHEXZIepBBCCKdzwQ6kJMiM5MZTs3nyyICBEPbgil/uOY4L/iVKghRCCOF0t1kVx2kkQQohhHA6maQjhBBC5BDSgxRCCOF0LtiBlAQphBDCBdiQIbduXMu2jWvtXxeTJMgMjHgxg6XmQmSpOSFE7rJuzWrWO3mpOVsm6TRr2ZpmLS2/g79ctsReVZKl5tJKXvbo5MXct9Rc9We/cnYV7CLio6edXYVsJ0vN5SzXc+FSc4UdvNTcXxFxWXp9xeKegH3qmTs/rUIIIcRdkiFWIYQQTieTdIQQQghrXDBDSoIUQgjhdLKSjhBCCGGFrKQjhBBC5BDSgxRCCOF0LtiBlAQphBDCBbhghpQEmQFZSUcIca/IKSvpOJqcg8zAlFkL0j3ulBy11rw1fSLBdapSt2o5Xn1uEPFxd14dYuTLw1m8aF667X8cPczTT7SheoVA6j1UnpEvDycmOirLMd2N1zo9xMFZ7fhjQSfm96+HR/68GZYd1KoK+2e2498PuvHdqBY8XM7PYn+9yv5sGBPCvx9049e32jOmaw3c3TI+nr1orZk4fixVq1SgXKlABvXvS9xt2mvliuXUr1uLgKKFaRsWwskTJyz2Hz1yhFYtmlK8iA+P1avLhvXr7ByBdVprJk8YR/UHKlGhbAmGDOx327hWrVxOw3p1KFncj/ZtWqWL64dd39M8uCGBRX14sEoFxox6nfh4x680lRvbS2vNlInjqFGtMpXLlWTYoP63jembr1fQuH5dSgcWoePjoZw6aRnTx0uXULdGNQKLeFO/zsMs//Jzm+oR2rotcxYssngISZDZat6sqSx9fyEjxk5m9sLFHNy/hxeH9MmwvNaands2s/LLT9Lti7t2jV5d2uLh4cGHX3zD1NkL2b/nJ4b2ewpHLw/4vw4PMrBlFd749AAD3v6B2hWL8u6QBlbL9mpSgTe6PMz0r3+j9YSN/Bkew6oRzSlWyB2AQF8PVrzajKP/RtNm4kbe+OQATzQox4zedRwZEgBTJ09k4YJ5TJ46g8VLl7Fnz8/06W192bqdO7bT66nudO3+JKtWr8PL25vmTRulJIrIyEhaNmtM6dJlWL12I6FhrXmiYzsO7N/vyJAAmD5lEosWzmfClGm8v/gj9u7ZTf8+Pa2W/X7Hdp7p+SRduvVgxTdr8PL2plWL4JS4ws+epePjYTxQtSprN25l4pRpfPn5p7z8wrOODAnIne01Y+ok3l20gPGTpvHu4qXs27ubgX17WS37/c7t9O31JE90687yVd/h5eVNWIsmKTFt37qZ54YOpO+AQWzY+j1duvWgX++n+GHXTkeGlGVKZe1h1zrJWqyWsroWa2JiIvWrV+TFEaPp/rSRFA/u30On0GB++OVPAoNKWpQ/9OtBenQM5UpsDABvTJhOn0HDU/avW/01zw9+hr2HT1HIpzAARw79Rusmj7D5x18oX7FypmPLylqs+fIqjszrxKSvfmHptr8BqFW+KJvHtaLqsysJj7T8tbv2jZb8cjKS15ftAyCPUvz5dmfGfH6AT3YcZ1jY/QwNe4Bqz67kZpLx2QurVZJPXgimRN/Pibt+I9N1zMparImJiVQsV4rRYyfQp19/APbs3k1ww3r8eeIfSpa0bK/uXTtTIH8BPvzY+DGTkJBA2ZIBzJm/kK7durNwwXxmzZzOsb9Okjev0Rtu1yaUUqVKM3/hO5muX1bXYk1MTKRKhTKMGj2OZ/oace3ds5tmjRtw9K9TlEgT11Pdn6BA/gJ8sHRZSlwVygQxe+4CnujanblvzWL+3Nkc+fMk+fIZZ2S+W/0NPbp04tylGAoWLJip+mV1LVZXb6+srMWamJhI1YplGTl6HL369ANg357dtGjyGIf+OJmurXr26EKBAgV4b8nHKTFVKleCWXMW0LlLNwb06cnVq1f59MuVKa8Jbd6YB6vXYPrMtzJdP0evxfrP5YQsvb50EePHt6zF6sL+PHqYSxcjaNI8JGVb9Rq1Kezrx487t6crf1+FSnz13RbW79yHf/GAdPuvXrlC3XqPpSRHgCJFiwIQfvbfbK9/Rh4oWZjihT3Y+MvZlG0HTlwi6tp1GldNX2+P/Hm5mpCY8jxJa+Ku30gZknXPn4+tv4enJEeAS7HXyZNHEeDrYcdILB0+dIiIiAhCQsNSttWuUwc/Pz+2b92SrvzWzZssyrq7uxPctBlbNm009m/ZRIsWISlftgAhrcLYvHmjHaNI78jhQ1yIiCCk1a261qpdB18/P7ZvSx/Xti2badkqNOW5u7s7jZs0ZcvmTQAkxMfTtFmLlOQIULRoMbTWnDsXbsdILOXG9jpy+BAXLkTQIuTW339Ns612bN+arvz2rZstyrq7u9M4uClbzbYq5u9v0e4ARYv5k+CE4fCscMUepEzSySYXL0QAUMz/VtLIkycP/gGBXLwYka68Z8GCVL6/KgBu+fOn2/9Ej5480ePWsJjWmqXvLSR//vzcX/XB7K5+hvwLG0krIvrWrzut4VxUPP4+7unKf7vnHwa2qsLqvf9w7N8YnmlWET/vAmz9/RwAb6763aK8UtC/ZWUuxiRw5uI1O0ZiKSLiPAABAZbtFRgYRMQFy/aKi4sjNjaWoKAgi+1BQSU4cdzoVZ8/f56Hqj9sub9ECS5ERKC1RjnoKujkuIpbievChQsWZZPjCgzMOK5XRoy02JeUlMR7i96maLFilC5dxh4hWJUb2+tCBm0VEBiU8n2SLDmmgMBAi+2BQSU4acY0aeqbKduvX7/Ozz/uYsf2rSx670M7RZDd7vx3vmn9d2xe77jJRPdkglRKBQPBWuux2XXM6OhIPDw9LX6RAngV9CIq8vJdHTsq8jLTJrzBF8uW8PLr4yhazP+ujpcZvl4FuJZwg6Q0Q/FX4xMp4p0+Qc5efZhWNUuya0qblG1D3/mRE+evpCtbws+T6b3r0qZ2Kfot2EWiA2/xFBkZiae19vL25vKlSxbboqKMiVEFC3pZbPf29uaSWTYqKpKCXun3x8fHEx8fj6enZ3aHYFVURnF5eXP5smVc0clxpam3UTb9Z/bfM2d4+cVnWbP6W95f8hH5rfyws5fc2F5RUVFWY/L28sqwrbzSxmSl7M8//UCrZo0B6D9wCGFt2mZ31Z2mRas2tGjVxmLbpx8tttv73ZMJ0h4KF/YjPi6OmzdvWnzgr1yJxcfHN0vH1FrzzYrPGTviJZJu3mTKrLfp+lTvbKqxbaKuXqegez7yKGWRJAt55if62n/pyi8cVB8vDze6vbmNc1Fx1K9SnIlP1uJ8dDybfzWG5JSCwa3u540uDxMb9x/d3tzGugOOGzYG8PPzI85Ke8XGxuDra9leyc+vXrVM8jExt8r6+vpx9Ur6/fnz58fDw3FDx763iatwYcu4CifHlabeRtnCKc+TkpJ4e/5cxo8ZRSEfHz5f/jWt2zxuvyCsyI3t5evrm0FMsRm21ZWradsqNmVfsoeq1+CHPQf549hRRr/+KoV9fRk5epydosg+stSckyml6iqljgEfAcOUUseUUsOy49jF/IsDEHH+1nmZpKQkIs6fs3qO0RZTxr3OC4P7ENysJVt3/063p59x2FBdsgvRxvmLwFTnB5Uynp+PtpygE+jrQY9G5Rm08AfWHfiXX05G8va6oyzbcZzBraqkvHbxsIZMebo2i7f8Sa2XvnF4cgQobrZJeLhle50LDycgwHIYy9PTk0KFChF+9qzF9vDwsyllAwIC0u8/e5aAwECHtllyXOfSxHX+XLjF8CSkiis8fVzFzbiSkpJ4pmcPRrzyEn36D+Tg78ccnhwhd7ZX8vdC6nO5SUlJnDsXnvL3nyw5ptTtCpZttXfPz0RevoynpydVqz1Ix85deHXkaD756EP7BpJNVBYf9pQrEqRSSiul2qd67mVuC05dTmu9R2tdBegJzNdaV9Faz8+OOlS6vypFivmzfcutk/y/HthLbEw09Ro2zvTxfvx+O+8teIvXxkxi9sIlDh1WTe3Iv9FciImnxcMlUrbVKl8UH8/87Dx83qKsRwFjQCIpKc3MaK0p6O4GwJONytOxXll6z9nJyGX7uZqQ+Vmr2aFqtWr4+/uzMdW1b3v37CE6OprGTZqmK9+kWXM2bLhVNiEhgR3bttK0eQtjf9PmbN68kRs3bsWzccM6mjVrYcco0nugajWK+fuzMVVd9+014wpOH1dw02Zs2rA+5XlCQgI7t2+jabPmACz76ENWLv+Kpcs+Y+r0mXh7e9s/CCtyY3slt9XmVH//+/ftISY6mkbBTdKVb9ykGZs3WrbV9zu20aSp0VY9u3fhi88tLxm7evUKefPljIFCmaSTi7m5udGz7yBmTBxN0aLFKOjlzdgRLxLatgMlSpYmOiqS6Kgoyt5X3qbjrf76KwICg2gR2pbTaS4GDggMwt1Bw0A3bmre3fgHY7rW4GJsAlfjE5neqw7f7DnNv5fj8C2YH1+vApyIuMKJ81f48VgEiwY3YOznB4mIjqdeFX/6t6zCyx/uAaBTvbLs+/sSv56K5L7ill+2/1y6yo2bjrnsyM3NjUFDhjF61AiK+fvj7e3Ni88Pp0OnzpQuXZrIyEiiIiMpX6ECAEOGDicspDk1a9bm0Xr1mT1rBoV8fGjfoSMA3Xo8yZRJ4xkysD/9Bgxi44Z1bN+2lZ0/7HZIPKnjGjhoKGNHjzTi8vLm5Refo33HTpRKjisqkvLljbgGDR5G27CW1KhVm0cercec2W9SqJAPj7c34lrx1RfUrlOX6jVqctycDJKsdOkyuLm5OSyu3NZebm5u9B84hHFjRlK0mD/e3l688tLztOvQiVKlShNlttV9ZlsNGDyU9q1DeLhmLR55tB5zZ8+kUCEf2rbrABgX+7/15nSCgkpQoWIlDv3+GzOmTmLQkOG3q4bLcMWVdHLFdZDmdTQdtNarzOdewBWgidZ6e6pyjwBpr8qfq7Wem+ZYPFi9ps3v361nH3r07IvWmtnTJrBq+WfEx8XTtGUo46bMwt3Dg7emT2TOjElWr698rGZl+gwYZnEd5DPd2rN9ywar7/fZqg082qCRzfVLlpXrIJO93rk6XR8rh0f+fGw4+C//+3AvCYk3ea3TQ4zoVB2fHsa1Wb5e+RnZ+WFa1SxJEe8C/H0ulkUbjvHJjuMA7J7eliolC1t9jwefXck/lzI/kzUr10GCcY53wrgxfPbpMuLj4ggNa8OsOfPw8PBg4vixTJowjvjEW/8+Vq5YzvQpkzh16iR16j7CvAWLKFuuXMr+I4cP88Jzw/jl4AEqVqrMmHETaNEyxMo731lWr4NMjmvS+LF88dknxMXH0Sq0NW/OnouHhweTJ4xjyqTxXEm4dd3eqpXLmTFtCqdPnaR2nbq8NW9hSly1H67GH8eOWn2fQ8eOU6Zs2UzVLavXQYJrt1dWroNMjmnyhLF8+fmnxMfFERLamumz5uDh4cGUieOYNnkC0XG3ernffL2CN6dN4fRpo61mzX2bsmWNmGJjY5kwZhRrVn9DZORlypQtR68+/Rg4eFjKOc4PP3iPDxe/b1Pdfjm4P7mODrkO8lx0+jkNtggsbEwWs0c976kEmYljZXqhgJzgbhKkK8tqgnRld5MgXdndJEhXltUE6cocvVDAuZgsJkgf+yXI3DTEqjL4sxBCCBfnil/auenn3GOp/pz58UchhBBOI5N07GugUmoncA6Y4ezKCCGEsJ0rTtLJTT3Iz4CZwBrgOyfXRQghRA6Xm3qQa7TW/VM9f8VpNRFCCJE5rteBzFUJUgghRA7lgvlREqQQQgjnc8W1WHNFgrT3dTpCCCHsSybpCCGEEDlEruhB2sOIF4em29YsJIzmIa2dUBshhLCfdWtWs36t425EbI0tQ6zr1zq2nrliqbnsJEvN5Tyy1FzOIUvN5RyOXmou8lrW7uzjV9B+9ZQepBBCCKeTSTpCCCGEFTJJR6Tz6UcfOLsKdvHf39ucXQW7+OC9d51dBbtY/H7uiyu3ttWHH7zn7CrcMyRBOtnnHy12dhXs4r/juTNB5sZEAth8j8CcRNoqZ5HFyoUQQggrXG+AVRKkEEIIV+CCGVKGWIUQQggrpAcphBDC6WQWaw63eYPtKzhkpqyz3z/x7MFsLZdZmTmurWXXfLfa5mNmpmxm2HrczLz/2jX2Kevs93d2e9nr/dfZ+Hdga7nMysxx7VUHW7niJB1JkBkY8eLQdI+P3l9k8+u3bFib7XXKzDEzU/aGjUnH1nKZlZnj2lp2bWaSjp0SpK3Hzcz7r1tj+73AM1PW2e/v7Pay1/vbuiyavZZPs/W469asZvTI13hu6KCUh6OpLD7sSYZYMzBl1oJ026ytzyqEEDldaOu2rF+7hjkLbnUCli5x8OUkrjfCKj1IIYQQwhrpQQohhHA6V5ykI3fzSCN5ZXkhhBCOu5vH3bJHPWWIVQghhLBCepBCCCGEFdKDFEIIIayQBCmEEEJYIQlSCCGEsEISpBBCCGGFJEghhBDCCkmQwqmUsvdyw0IIkTWSIIXDKaUGKKXmAGitdU5PkkqpLkqpps6uh7CU0z9XwvkkQQqHUkp5AmWBUKXUBMi5SVIZ/IC3gFeVUo85uUqZopRqopRq6ex62Iv5uco1y2kqpUqn+nNRZ9blXiEJUjiU1joOmAssBdoppaaa23NcktSGSKAZEAC8ppRq7ORq3ZGZ2L2AOUCwk6uT7cwefRellDvGD5dSzq7T3TJjGKSUaqCUGgV0V0rJ97edyV+wi1JK1VRKPaWUelkp9aiz65MdzC/mvFrr88BaYB3QTyn1GuS8JGnG46a1Pgr0AioCw129J2km9qvA20AvpVQlZ9cpuyilPIA/gKrAIcBDa33GubXKFheBb4FJQE+t9TytdZJSqpCT65WrSYJ0QUqpDsAmYADQB5itlJrs3FrdPfOL+aZSqj0wG2gAeGH0vHLkcKvWOlEp9TjQFcgPdAQmKKXqO7dmGUvV89gOnAVqmtvzOqtO2UVrHQ+EA+2AM8AOSDnvXdGZdbsbWusE4BmgMLBcKVXB/J4YrpTK79TK5WKSIF2MUqoKMAOYqLVuBHQH6gLRSqliTq1cNlBK1QQWASuBnkA14BPgCaXUeMg5SdKsZzCwDLgAvAi0Bx4AxiulGjitclYope5XSpXRWicBaK2PAX8DI8znN51Zv+xgJv+hwA8YCeVBpdRuoLXW+i+nVu4uKKVKAv8BocAHwJvATGCt1vo/Z9YtV9Nay8OFHkAYcMD8czngH4x/EIHAq0AZZ9fxLuMbCPwO+KXaVgojacYArzi7jpmMZzqwHnBLte1+4BywDXjM2XU06/Qg8CtGj/F5oJG5vRpwEGhnPlfOrms2xFo01Z83AB+lep4j4wMUt24u8SzG8HE183keZ9cvtz6kB+l6CgAXlFLlMH4Fb9Za9wWuYyTIRs6sXDZwAzwx/sFjnpM8A8wH8gKvK6UmOrF+NjHPPyqgCsYXV6K5vYA2zkk+hzGE/D9XmLijtf4dY8h+BsaPlPeVUl9hJMhCwCNmuRx/ex+t9SUApdRQjB8uPc3neXNqfNqklCoCPAr00VofUkrl0eaIgMh+kiBdgFIqb6rzP8eBpub/VwH9zO1ewAmMHkCOkDxMasaX/Fn7HaNn3BEshvXyYPwgGAC86+Cq2iT1sG/yFxawEaiTPJyqtb5uFnEHDgPXgD8dXdfUkuuttd6ttX4LY5huIFAa6I3RHsNd+bxpRu4wk3OV1rqpWS6vzkFDyMlxWTnVEAn001rvkeRof7nmGqGcSinVHOgG+Cilxmmtf1NK9cVIEueAIDN5Pg/44+QvW1sppZT5i7cp0AEooZR6TWu9Qyk1DphvXqO2FriJcT4yH7BJax3lvJpblyqeRzDOCf+HcR51JUbCGaOUGq+13mVeQvEgsBqYoo2JI06TttektT4FnDIT4iPAzxg/xAYrpeK11gcdX8vMS50glFL3A8WAf4ELWuurWuuzqcrlmOQIoI0ZqjWBjkqp6VrrWHO7BuKSyzizjvcCuWGyEymlWgFfAX9hnGP0AJprrfcppQZjzPSMAK5gDD9211r/4qTqZpo5W/UTjNmSpTG+wFpprX8xL+2YiDG5JRZjaLm91vpX59Q2Y6mSYwfgPeASRnscxJhlXBVjoksz4AigMXplzc2hTZeTtvehlOqCMSwcDkzVWu93WuVskNwm5p8nYswi9sP4UbkTmKW1/tuJVbxr5r+flcBUjB9aV5xbo3uQs0+C3ssPoD/wMuCL8YX6BRAN1Db3PwC0AVoAgc6ubyZjexDjy/Zl83k1IAk4DdQwt9XG6D0/AZRydp3vEE89jB8qL2H8kHkFiAc+A3wwvpyfxphsNAKo6Ow62xhXnlR/fhIj6Vd3dr0yUf/B5uesk/l8KcYPmEeB/M6uXyZjSTeBCGP05SYwDfBydh3vtYf0IJ1AKVUL4/zir8A7WuvV5vbiGKvMhGL0PvY4rZJ3SSkVinEhehuMHvI0jARZCmiMMaQaqbXe67RKZoJSagTGDONmGBOM3sEY8nbHuIj7VSBCO3k4NSvS9MYCtLGQg0szz83lwfiB8qvWepJSKgxYDgzDONftq7Xe6MRqZppSqgSQoLW+nGpbZ4wfz28Ck7Q53CrsTybpOMdFYB/GF643pAx5RWBM4f4W+FkpVcN5VbxrhYHLGMOnjYEbGEllBMZQ66fA10qpIjnhmkeMSVJFMIZP22Cc6+oNrMHoAe8GlimlvHJIPCm0trjuNMKplbGRvvXLvgjwg1KqGbACeE1rvRhoAkwxV9ZxeUqpPEopX4w5BuPN2aoAaK2XAz0wRpuGKWP9X+EAMknHCbTW/5jnGPMBc5VSB7VxaQBa6wil1MsYl3XEObOetjAnECWZX7J+GNPqI7TWnyml/sH4wp2H0Vv+C2iJ8Wv4PeB46l/KrsCcOHTTjMddGyuYgHEeaBfGJSpdgB1a6wtKqUiMyTirge+1sYRbjpOccFIlHpenjVWZLmMs1OAHDNdav2/uvgFEYUymcnnaOB8cZX4vvA9cU0rN0FpfNIusxVjUYSJQwJzQJ5N07EyGWB3EnJFWHeMX7+9a6w1KKR+MSToPY1y4fSxVeZeelq6Uaps8NGw+bwNMwVhu7QetdR9z+4MYF8w/YCaU+UBBrfUzzqh3RpRSNbXWB1I9bwm8gDGJaK3W+gtz++PAh8B9WutopdQ7wL9a6wlOqPY9K9XEqbIYE8ECtNbllVIFMEZlvgH+0lr3dmI1bZbqUhytlOqKMXQ8DXjLHFlCKfUZ5kiM1nqX0yp7D5EE6QDmbLT3MHpRlTEmd+zTWvcwk+SXGJNYWmqtDzutojZSSpXH6A1+prV+0hwK3obxpXQNY3m89Vrr7mb5fUAFjBmelYHGWutDTqm8FUqp1hjnd+ZrrRcopeoBW4GfMM6ZRmCsxvKuUuohjLYMxzin2gBoqHPwMmY5mdnjbwIsxBiR+RdjhnE+oJ7W+kbqc6yuLE2S7IZxGmIDkDwX4Xmgjtb6T7kG0jEkQdqZMu6UsBZYgDHDMT8QgtHb2qO17q6U8seYzu0PVNXmqiyuyryIOQyjJ/U1sAQjuY9VSrlhzLpdBmzUWndTSt2HcW4VjF+/R51Q7QyZbfQ6xt04PsX4cs2ntZ6pjDUwJwFlgKVa6yVKqTEYiTEv8Lx20Us57iXm+buhGBOoIoFF5hBsPq31DefWznZpkmQnjBEmgI+B97XW30tydBxJkHZmXus4B2NW6hlzmwfG3QbGAf211jvN2Wt5dA65NY/5DzkMIxEWAN7TWj9n7suPca5xGcZqJr3N7S73ZZVqqO4+YCTG5TYBwHit9edmmXIYbXUfMFdr/aW53VMb97cULsjVT1NkJE2SbIsxMjMH+J+r/fvJ7WQWq/0lYUzsKAwpX8jxGEN4xTDW8kRrfTYnJMfU/3gxFul+EmNWbjWzZ4k27i6wAWPmXU+l1GLz5S75ZWW2yQmMhcdPAmWBWsn7tdYngTcwhpVHKaV6m7ty3CUd95KcmBzh1kQp83O5GuOUxTBghjnaJBxEEqQdKKV8lHE3czC+VD2A3kopj1TnQm4CRzGSi8tT5nqx5q/a8kqpCkBxrfVajKGt2hhDroBxn0SMe1qGYkw2cLkZkqnOTfmYvcE/gLEYw6xhSqkhyWW11qeBCcCPGOdbXS4ekXukSZJfYNyQ+zmM0wDCQWSINZuZF8i/iLG6ynyt9UdKqXYY5xjnAZ9jrJYzAONygfpa63+cVN07Usa6sCe01tvM5+0xJkTkxUjwo8zzIm0wr23UWvdyVn1tlWpoNbm9CmMMn36slCoNjMFYyegjrfXCVK9zuWFikXulGW6tKJPBHEsSZDZSSjXEmLSyB2P5uDrAS1rrOea5hHcweu1RGD3Ip7QLr62qlCqFcS/KIIzFrH/HuA/dJxjDi7UxJha9lipJLsW4RrCjc2ptOyvtVRtjabw5SqkywGigEkbSn+W8mop7Waofc8n/l0k6DiIJMpsopTwxzrkFAjO01glKqTcwJne8YH7pBmGc31IY12hdcFqFbaSUaoQxhFoBI/mVx0giiWaCeQ4j5uQk2RF4C3hUax3upGrf0R3a63mt9VyzJ/kmRu+yi9Y62ln1FUI4nqykkw2UUsUw7t94EPhcm6uvaK0nmBNXZiulbgKLtdY/OrGqNkueAWjOsE3EuAbrdeBg8mUoyVPOMSYQTFTG6h4rlVIbtQuvKGNDe72llErSWs9XSr0E3JDkKMS9RxJkNtBaX1RKPQGsA44opby1eWsarfU4MznOBa4rpd7PIZM7ku+zVwrjovhdGHez6KGUaqS13gmgjfs7aoxLJF5RSv3kyskRbG8vpVSi1vodp1ZWCOE0MsSajZRx8+P1GOfrvtCp7uyglHoFWO1qF8lbk+pcRzuMe1JqjGHhlRjXApYHhupUy10ppRoAp7XW/zqjzlmRW9pLCGEfkiCzmbkwwDfAQNJ86eYkSqm6GHcVeQv4A2Ph9N2YF9FjTDcfrnP4mpC5pb2EENlPhlizmdZ6vdnz+gpwV0p9qG/dESInqYWxdurc1KvFKKUCMWbi/ohxe6euWuvdTqrjXctF7SWEyGayUIAdaK2TV5gZi7EMW46RfN0VxiLdKUupKWNRaDB6jo9hXAO5kxyy0MHt5OT2EkLYjwyx2pFSysvVJ6xkRCnVFGO5uKfMlTySt3fGuD6wKRBrLiuXK+Tk9hJCZD/pQdpRDv+y3QXMx7h8owuk3DGhIcYiB0m5KTlCjm8vIUQ2kx6kyJC5MPIoYBC3JuoEAm211r86s25CCGFvkiDFbSnjDu2PYpx3PIuxjNxJ59ZKCCHsTxKkEEIIYYWcgxRCCCGskAQphBBCWCEJUgghhLBCEqQQQghhhSRIIYQQwgpJkEIIIYQVkiCFEEIIKyRBCiFcjlLqQ6WUTvOIUEp9qZQqY4f3G6uU+sX8c2/z/cpm9/vc5v23K6XeymBfb6VUdCaOdUop9fxd1kcrpdrfzTFyA0mQQghXtQfj7jEVgSrAEKAesEopldeO77vSfM+zdyqolGqjlJLVVnIpuR+kEMJVxWut/071/A+lVCFgMXAf8Jc93lRrHQvE2uPYImeRHqQQIie5Yv7fHVKGAlsppdYrpQ6Y2woopaYppY4rpa4ppXYqpR5LPoC5f4ZS6rRS6oJS6gPAM9X+YPO4hc3nvuaQ7zmlVKRS6nOlVHGlVDCwOlU9ept/DlJKfWEOCUeaw8IlUx0/UCm1XCl1WSn1t1LqZSD5Pqx3pJSqrJRap5SKUkrFKaUOKKXapCnmoZRaqpS6pJQ6qZSarJTKn+YYa836XVBKvZccr7hFEqQQwuUpQ0VgBHAaOJZq9wJgLfC0+fw9oDnGkGxj4Hdgi1LqfnP/TKAf8AbQGtDACxm9L/AdxhBvN6AvUB2jF7sbGGgWrQisNBf334lxS7h2QEegGLBVKeVuDg1vACoD3c3jdQLqZ+KvYyngZ76+FXAY+Mp872RvAJHm/unA88BEM6aiwM8Yd+hpCfQC6gLfpLphukCGWIUQrquRUirB/HNejO+rP4DOWuvEVOWWa63nApgTa54CymmtT5vb9gM1gQ5KqTMYt28bpLX+KNX+RzKqA8bdbMprrU+Z5a8CI4EbQDhA8lCwUqqX+bqntdY3zW0HgItmHbyBasD9Wus/zP2dgczcIecrYL3W+rD5+ptmzIHAKbPMPq11ctLfp5TyAUYrpUYAQ4GDqfajlDqB8aMjMDkmIQlSCOG69gE9Uz2PAc7r9Lcg+iXVn6thDFf+kaYzlB8jAVTESLZbkndorZOUUtsxbgae1oPAmeTkaJbfBGwCsNLhehDj/Og1K+9fEaPndyY5OZrHO6uU+gPbzQEeUUoNBmpg9BLT2pbm+UZgClDarGPjVD8+4NYQb0UkQaaQBCmEcFVxWutjdy5GfKo/5wOSgFoYw5ypXQMKZ3CMpAy2u91mnzX5SJ/Yk10EemTy/S0opdyB9UBJjNm2a8z/r0tTNO2PiOTTadfNOn6LMVydliTHVOQcpBAiNzmK8b3mpbU+ZibY4xjnI6sAJ4BEoFnyC8zzbo0yON4RoHSaSTb1lVKHlFJ+GZS/D6OXmPz+ccBrGBOBjgKllFKVUh2vuFk3WzTC6OlW11q/orX+BusTfILTPG8JXAbOmXWsBPyZqo6ewKuk/1FxT5MEKYTINcyhy2+Aj5RSrZVSDYB3MSai/Km1vgYsBGYopZ5WStXGmORTLoNDbgYOAV8opRorpcKAucAlrXUkZkJRStUyL0H5FKOX9oWZSEOBz4GHMK6r3Ar8CixXSrVQSjXEOKcYn/aNM3AN43v7WaVUHaXUc2b9ARoopZJHBWsppd406zUYGAVMNYenFwBlgffNY3QAPgMKm38/wiQJUgiR2zyFcY7wbYwZo2WBFsmTdoD/YSTNiRjDlZ7AK9YOZE60CQHOYCSyj4G/MWaQgjFj9SdgFxCmtb6K0cNLwrgE5GOMCTittdZJWuskjHOGf2AkpaUYw6QrbIztR2AMxqzUTWbd2ph1exPwMssNB8qYZV40XzPTjOks8Jj597LZ/HvagvEjQqSi0p/vFkIIIYT0IIUQQggrJEEKIYQQVkiCFEIIIayQBCmEEEJYIQlSCCGEsEISpBBCCGGFJEghhBDCCkmQQgghhBX/B1/mgCFwKAHWAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "batch_size = 100\n", + "conf_matrix_ssl = evaluate_mlpf(model_ssl, with_VICReg=True)\n", + "plot_conf_matrix(conf_matrix_ssl, \"ssl MLPF\")" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3d6d098528b44bcfb0a61c90c573dccf", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Time taken is 100.82s\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcgAAAGoCAYAAADCVXWwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAAsTAAALEwEAmpwYAAB8HUlEQVR4nO3dd3wURRvA8d8TCCQhhYSWUEJHEWw0Bem9iiAKWJGOgGJ57SIdBEFBEGwo9gKIoiC9iIUiWEBFFFQkEEoakAQDmfeP3YRccoG0KwnPl899yO3O7T3DhHtuZmdnxRiDUkoppRz5eDoApZRSyhtpglRKKaWc0ASplFJKOaEJUimllHJCE6RSSinlhCZIpZRSyglNkEplIiJTReQ3EfnN07FkJiKtRcRkeFTN4evGZnjNX5n2DUjbl89YMj9Oi8hOEXlTRGplc4w3L3KMzI83cxOjUvmhCVJdskRknP0onWlXBHCZ/fB2vQu4XEEKAK4F7gZ2i8jdHohBqTzTBKkuZc/Yj9IejiM/+lysgIjUBK52cRzVjTGS9sD6bAm349sPlATmikjlbF6/KePrL/AY4OJ6KJVOE6RSmRhjBmT4oPdWCUAs0ExEKl6kbC/77/2uDek8Y4k2xiwB2gGJQCAw2l0xKJVfmiCVKpxSgGX2zxcbPk3bv8Rl0VyAMeYvYIX9tJ4nYlAqLzRBKq+QYcLH2gzP14hIjIjEici3ItJPRJz26kSkqYh8LCK/i0iSiBwTkV0iMkFEKmQquzHThJQD9nu3tvenTVrZaD+PEJFUe1urbN5fRGS/XeaJTPv8ReQREfleRE6KyAkR+VpE7hGREnn9N+N8wst2mFVEKgFNgaPA1/l4r/z6y/67hgdjUCpXins6AKUyE5F+wDtAsQybr7cf1YGpmcrfBSzKdBg/oCxwDXC3iDQxxkTnJR5jzGER2QK0AHoCm5wUu9aODeC9DLFVAlYDV2Qq38x+3CkiNxpjTuUhtLVYQ60tRKRCNvW7yf57GXAuD+9RUKrZf//jwRiUyhXtQSpvUwV4FViDlRCDgebAT/b+8SLin1ZYRIKABfbTpcBVWLMnK2DNnjwFRAL3p73GGNM60/nFtAkmGy8Q12L7757Z9GJvtv/+yh5STLMAKzket+OpBFS140kC2tj1zTVjzBlgOdb/45uyKebR4VUAEakGdLWf/nSBokp5FU2QytvUAXYA3Y0xW40xJ40xXwO32vt9cTyPdSXgD0QB/Y0xPxtjkowxR40xbwHT7XLX5jOupfbfNTK9P3bCTBvmfCfD9pZAd+AM0NwY85YxJsoY848xZg7QxS7aT0Qa5DGubIdZRaQs0AqIAzbm8fh5Yg85lxeRXsA6rC8tp4EXs3lJqxxcAznAXfErBZoglXd6xhjjMBxojNkLHLOfBmbY9S9wC9DPGPOfk2OlDTv6O9mXY8aYf4Fv7ac9M+2uh5XY/+N8TxNgsP33y3b8mY+5CasHCOeTZW6twko8bUSkTKZ9PbCGqT/L5t+mIB3ImMyAVKx/+6VYXypSgFHGmIMujkOpAqMJUnkbA2zNZl9ilsJWb2yxMearjNtFpLSI9AZGFmBs6cOsmban9d6+MMbEZNhe2/574wWO+Y39d8O8BGSMScSaIVrMSVxpw74eG17FmpyzBGhijHnzAuVych3khV6vVIHTSTrK2xyxz63lmD3E2QFrOLM+VmLK7oL0/FgCzAQai0hFY0yUvT0tEb2TqXxaglyazeTbjErnI67FWL3oPsBCABEJxvo3OY11PtfVqmc696pUoacJUnmbXA0F2ongM6xzbWD1Mn/EmuG5GwjCWi0n34wxf4vIdqAxcCOwQEQuw0rK8Zy/1i+Nby4OH3jxItlaASQD7UWktDEmDmtSTAlgmTEmKR/HVuqSpUOsqrB7Dis57gc6ASHGmGbGmHuMMTM5f/1dQUkbZr3R/jut9/iRMSY5U9k/7L/b5mD4sEleA7IvEVmFlZB72Js9PntVqcJOE6Qq7G6y/x5tjFltjDmbaX/NAn6/tITTzr7EJMvs1QzSEuSV2R1MRKqKSCP7esn8SEvcN9uXwXTFmj27Mp/HVeqSpQlSFXYl7b+zDM3aK+jcmfb0AsfI8Zqrxpg/gV1Yw5fDsS4f+QfY4qT4+/bfD4lIliFUe9sWYDtZFxLIrc+xZop2xurVlgJWGWNO5vO4Sl2yNEGqwu5H+++pItLAXtatsr26ztdYF+UD1BeRaiJSzMkxroH0yT45kdZbG2///a4xJtVJuU+xZqlGAl+LSFcRKSsiQSLSBliPNZnoZ6xrBfPMPu+4BusLw3P25lwPr9r/Rhd6ROYnTqUKE02QqrB7HOuau0bA91iTdA5iLT0XBtyDdelIGHCA8+fo4Pz5yaUiksD5iT4Xk5Z40q6tfNdZIWOMAQZiXat5FfAF1rWcCVjJsTHwN9AtmwSbW2lxVQDOcv4ay9w4cJGHroSjLhmaIFWhZq+ycx3WEOMhrPNuv2OtoHOZfe3cUOAEVm/z7wwvHwjswRqePYV1SURO3nMvVq8PYJcxZs9Fyl6F1dtcg7XkXAzWtZ6PAvUK8OL5Tzm/3up6Y0xsAR1XqUuSWF9ylVJKKZWR9iCVUkopJzRBKqWUUk5oglRKKaWc0KXmMsl0p3mllLqkZbp3aoErqM9cV8SpPUillFLKCe1BZsPvxvlZtqX8+C6+V9+eo9fntOyZTVMp2erxAj1mdmX/fXew07IPjB7O8y8uuOgxc1oOoF3L61i3Obu7VuX9uM7KBpTMeu3/yBFDmTf/lRwdMzdlb7i+MV9/t71Aj+us3KnkzCvmWR66bwQz52T93cxv2Q6trmfNpu8K9JjOygb5O1+/feTwocxbkMP2ymHZG65rxNdbdxToMV1VNjfHdFe9/H1d2nHMwu+avN2ZLvmHeQUcyXmaIJVSSnmeeN+AZrYJUkR2Ya1AclHGmAYFFpFSSqlLT45XenSfC/Ugl7krCKWUUpe4wtSDNMaMz26fUkopVaAKWQ/SgX33gb5ALWAw1vqXfxhjvndRbB6V8qOT9adLBOX49T4Vsr0FYJ7l5pi5KdupS/cCLZdbuTluTst27dbj4oXyUDY3cnrc3Lx/xy7dXFLW0+/ftXsu2isXZT39/jkt64o65ea4X3y+nH/++ZuRw4e6JI7CKkdrsYrIjVh3CvgYuAWoB9yBtdhyN2PMWlcG6U5p1+Q4m8XqCrmZxZpf2c1idYXczGLNL2ezWF0lN7NY8yO7WayuktNZrPmV3SxWV8jNbM/CxF31SpvF6q7rIP0aP5Sn1ydvnwl49jrIscAEY8xtWLfRwRgzFpgNTCrooJRSSl1iRPL2cKGcJsi6wGdOtn8M1C+4cJRSSl2SxCdvDxfK6dH/5vyd2TOqBkQVWDRKKaUuTV7Yg8zpJJ03gJkiEmM/rygiVwMvAK+6IjCllFLKk3KaIGcCQcBqoCSwHutc5Dz0HGS+FKva3NMhuMRdA9w3IcidBg4e4ukQXOLOItheAwcXzRmZRbVe3ngdZI5msaYXFimJdZlHcaxLPE67KjBPcfcsVndy5yxWd3LnLFZ3cfcsVndx5yxWlT9un8Xa7Ik8vT75mymAa+LMzXWQ5YCBwOXAGeAnEVlUFJOkUkopN8tBD/JczO+kxuxzQzCWHCVIEWkAbAKSgW+wJvf0AZ4SkebGmP2uC1EppZSCYmF1KBZWx2HbuehdLnu/nPYgnwe+Bm5O6zGKSBCwFJgLdHVNeEoppS4JXrjUXE7Pil6DtVBA+nCqMeYkMB5o6oK4PC7lx3ezPM4d+SlHr32yXyN2L+jP/jfuZP6oVviXyP57yMjuV/LTS/04+sFAVk26kWtrlnXY3/bqynw1ozfHPhjE9tm3cGuLWvmqV14YY3h2ygQaX305V9Sqwv33DiExMTHb8p8tW0LbFk2oUbkst9zUlb8OOA4wxMfFMWLI3VxRqwr161RlyoSxnDt3ztXVyMIYw6QJ46hftzbVIysyfOigC9brkyWLaXZdIyLKhXJjt84c2O9Yr19/+YUuHdsRXrY0LZpdx6ovV7q4Bs4ZY5g+ZQJNrq5L/dqRjBk59IL1Wr5sCe1bXketKuW49aZuTtvr3iEDqF87kqsuq8ZUD7dXvctrUb1KBMOHXLi9li5ZTLMmDQkvW5oeXTs5ba/OHdpSoUwIzZs28Uh7eUudvvh8OSOHD3V4uF0hvg5yPxDiZHsw8G/BheM9fK++PcujWPhVF33dY7c2YES3+jzx5rcMfH49TS6rwMIH2jote0+Huoy7owlTP/qejk98xm//xvL5+O6UD/EHoG6VUD55ugurd/1D+8eX8dqqX3j9gba0vbpygdb1YmZOn8JrC+YxbuI0Fry2iB3bt3HvkAFOy275ahND77mDPrf258MlywkMDKJH57YkJSUBcO7cOXrf2Im42DheX/Q+jzz+NAvmzebll+a4sUaWaVMmseCluUyeOp2Fb77N9q1bGTTgLqdlN2/ayN133ka//rexbPkKAgOD6NCuVXq9YmJi6NS+NVUiq/LZilV07tKVW2++iZ073b9U8azpU3j95Zd4ZtJUXnr1Tb7fvpWRQwc4Lfv1V5sYNvBObr61P+8vXk5gUCA9u7RzaK8+PTsTHxfHa4ve43+PP83LL83hlfkvurFGlmlTJjF/3otMmTaDhYveYdu27xg44E6nZTdv2sjdd/Snb//bWbZ8JYFBQbRv29KhvTq2a0VkZFWWr1hNl67duKV3T3Z+79728pY6deveg3kLXnF4uJ0XJsicrsXaGWuYdTiwBes+kc2B14CnjDEfuTJId8rPLNbixXzY9/odTHh3O2+s+RWAxnXKs/HZXtQZ/A6HTjjOZ1oz5UZ2/XmcR17/BgAfH+HAG3fx1Fvf8fa6vTx9WyO6Nq5G0wcWp7/m/Uc7cio5hSGzN+Q6vrzMYk1JSeHqy6vz2FPjuOse6/Xfb99K53Yt+PHX/VSs5Jis77mjLyVKluDl198GIDk5mStqVWbG83O5+ZZ+rP5yBaOHD+LH3w7g5+cHwEsvPs/ve3/jhbkv5zo+yNss1pSUFGrXiGTsuAkMHGRdurFt21batGjG3j//pnJlx3rd1vcWSpYsyRtvvZNer+pVIpj94kvc2q8/81+ay/MzZ/Dr7/spVsyKp2ePrlSpUoW5L+W+XnmdxZqSksI1dWvw2FPjuHPAIAC+376Nru1bsOuXP7O018A7+1KiREkWvP5Wer3q167C9Fkv0vuWfqz5cgX3jRjMrl/3p7fX/LkvsG/vb8x6cUGu48vrLNaUlBRqV6/C2HET0y+12bZ1K61bNOX3/f9kaa/+fftQskRJ3nz73fR6Vasczuy58+nbrz/z581l1szp/LbvwPn26t6FKlUimTs/b7+HRa1Obp/F2npCnl6fvHEs4Oa1WEUkVkRi7MUB3sO6vGM91kSdZGADUAl4rKCDKqzqRYYSHhrAqu//Sd+2Y99RYk4l0/qqSlnK+5cozsmk/9Kfp6YaEs+kEFCyePr+Uxn2AyQk/Ze+3x1+/WU3R49G06FTl/Rt1zZsTGhoGJs3rs9SfuOGtbTveL6sn58fLVu1ZeO6NQB8uvRjOnftnv5hC3Dv6AfynBzzas/u3RyNjqZT5/Onzxs1akxYWBgb16/LUn79ujV06uxYr9Zt2rF2rVWv9WvX0r5Dx/QPJoDOnbuwzt7vLr/9sptjR6Np37Fz+rZrGzbKtr02bVjnUNbPz48Wrdqwcb11/4FPP1lMp0ztNWLUmDwlx/zYs3s30dHRdOqSob0aX6C91q5xKOvn50frtu1Yt2a1tX/dGjp06OTQXp06d2Xt2tUurIWjolinouZC/dMxwAMZHoOxLvMYDAyxf74Xa8FyBVQIDQDgSNz5cwjGwOGYRCqUDshSftm3+7mnQ12uql4G3+I+jOhWnzLBfqzdZY1af7HtLxrXqUCvZjUoXsyHG66IoFezGiz71n2Tho9GRwNQvkJ4+jYfHx/CIyI4evSoQ9nExEROJiQQHlHRYXtExYocO2aV/fffg4SUDmX08EFceVk1rru2HtMmj+e//xy/CLhadPQRAMLDHesVEVGRo0ejHcomJiaSkJBAREXHelWsdL5sdPQRIrLUuxJHo6PJzbXG+ZVde1WIiEhvgzTZtldEJY7ZbXvIbq/7Rgzm6sur07RBPZ71svaKzqa9KmZur4qV0tvryJEjTtrTve1VFOuUL144xHqhGyYvutiLRSQCuKdAIyrEwoL8OJ2cQmqq4y/jqaQUygT7ZSn/3JIf6Nq4GltfuCV927A5G/jzcDwAX/9yhNnLfuS9Rzum7393w14+/upPF9Ugq9jYGAICAhy+lQIEBgYRc+K4w7b4uFgASpUqlaXsCbvs0ehoXn5pDkNHjObNdz7kn3/+5slHHiTx9GkmTJnuwpo4io3Jpl5BQZw47liv2Fi7XoGBjmUDz5eNjY0hMNP+oKAgkpKSSEpKIiAg6xckV8hbe2WqV1Bgetmj0Ud45aU5DBkxmoVvf8jBf/7iyUcfIjExkfGTn3VhTRzF5KW9SmVtj+MZ2itze7q7vYpinfLFC2ex5vQ6SH/gIaBGpl21gerAlAKOq1CKOZlMKT9ffHzEIUkGB5Qg9tSZLOVfvb8Ngf6+9Jm8kqgTp2leryJT72nKkdhEVu88yIAOl3NPx7qMmLuRH/48Tu1KIUy883qe7NeIyR+45z53oaFhJCYmcu7cOYf/yAkJCZQuHepQNsR+furUKYftCQnx6WVLlPClVZt2TJw6A4CGja8jKTGRxx6+n3GTpuHj457lpkLDsqlXfDylQx3rFWo/P3XypMP2hIT49H2hoWGczLw/Pp4SJUrg7+/viio4lV17nczQBmnOt1fmuBPS9/mWKEHL1u3Sv7w0bNyEpKQkHn/4fp6ZONVt7RWWXXtlaIM06e2VqV7x8Y7tlbk9493cXkWxTvnihUvN5TSiuVjDrJWBu4GywBXAdYDzKVeXoOhYa2i1Ytj5b2oiEBEWwJFYx6nbFcNKcUfbyxj8wnq+2PY3u/48zouf/cTb6/YysseVADzY6xqmf7yTN9f8xg/7j/PxV3/y4KtbuK/nVZQo7p5fpvIVKgBw5PD5m7akpqZy5EgUFTIMDQEEBAQQFBzMkahDDtsPR0VRwR7yK18hnLpXON4h7bLL65KYmEjMiROuqIJTafEcjnKs1+HDUYRHRDiUDQgIIDg4mKhDjvWKOhRFeHhE+vGiMtU7KuoQ4RERiBu/GWfbXocPUyE8a72CgoM5nLm9Dh9Kb9vy5cOpW8+xvepcZrdXjPvbKypze0Wdb4M02bZX1KH0suHh4U7a073tVRTrlC9eeDePnH7KdgVGG2M6Yi1YPsEYcz3wAdDIVcEVNnv+iSU6LpGODSLTtzWuXZ7SpUqy6SfHX1x/e6JN5uFYYwyBftZMv1J+vqSarPv9SxSjmJu+ude9oj7lypVn7eov07ft3LGN+Lg4WrRqk6V8q9btWLtmVfrz5ORkvtq8gVZt2wPQ5Ppm7Px+u8M5kZ9++oHSoaGUKVs2y/FcpV79+pQvX55Vq85fJ7Zj+zbi4uJo3TrrZTlt2rZn9arz/wbJycls2rietu2serVt1451a9dw9uz52aerVn2Zvt9dLr+iPmXLlWddhjbYuWM78fFxNG/ZOkv5lq3bsj5Te23ZvJFWbdoB0OT6puzK1F4///QDpUuHUqaM+9trdYbr+rZvs9qrVRsn7dWuvUPbJicns2nDetq272Dtb9uetWtXO7TX6lUradeugwtr4ago1qmoyel0yFDgN/vn74CGwA7gfeA5YEZ+grCn+bbF6p12A/4DnjPGPG/v9wFGAcOw7ku5D5hmjPnQ3t8a+BKrV/sKVs/2b2CMMWatXaYY8CDW5KKqwK9Yif7T/MSe0dlzqSz4YjcT7ryOY/FJnExKYdbQG/jkm/0cPH6K0MCShAaWZP+RBP48HM/Xew7z6pi2jH1rK0diE7nhinCGd6vPmJe3APDWut94om8j4k7/x4/7j1O7YghTBjTl46/+JOk/9yxm7evry6Bh9zJp3FOUK1eewKAgHnt4DDfedDOVq0QSGxNDbGwMNWpaCxgMHj6Sm3t04pprG9DkuqbMnT2L4OAQut/YC4A77h7I3Nkzeeqxh7il3+38vvc3Jo9/mtH3P+zWb7m+vr4MGzGSZ556gnLlyhMUFMRDY+6jV+8+VImMJCYmhtiYGGrWsuo1YuQounXuQIOGDbm+aTOen/kcwSEh9OzVG4C+/W9n6uSJ3Dt8KIOHDmP1lyvZtGE9m7Z857Y6pdVr0NARTB73FGXLlSMwMIgn/jeGHjf1dt5ew0bS58bOXN2gIY2bNGXenJkEB4fQLUN7zZszi6cfe9hur1+ZMv5pRo15yO3tNfzeUYx96nHKlbfa68Exo+l1cx8inbTXvSNH07VTexo0aGS116wZBIeEcJPdXv1uu52pkydw77AhDB46nNWrVrJxw3o2f71V6+QpXjjEmtPrIP8AXjLGzBKRXsCdxpjeItIK+MIYE3iRQ1zs+Ab4Geu6yu3A/UBf4EpjzG4RuR/rPOdDwDagI9ZttnobYz6zE+RarOXwngcSgWlYQ8IVjDFGRCYAtwMPA38BNwJjgY7GmPQ51WnX5EjI+V7gxRSr2pzi1VqkP3/6tkb0b1UH/5LFWbnjbx58ZQvJ/53jyX6NeKp/I/x7WlPkw4JK8vRtjenWuCplgv3YdyieeZ//zNvr9lrH9RFG9biSAR3qUrV8EIdjTvPRV38w/eNdeUqQeb2bhzGGaZPHs/jD90hKTKRD565Me242/v7+PDtlAjOmTuT4yZT08p8tW8LzM6by999/0bBRE557YR5Vq1VP3//Tj7t48tGH+PmnH6hQIYIBg4YwdMToLJMVciqvd/MwxjBx/DN88P67JCYm0qVrN2a98CL+/v5MmjCOKZMmkPhfanr5T5Ys5tlpU/j7rwM0bnIdc+bOp1r18/X6Zc8eHhwzmh927aR2ncsYO24CHTp2ylNs+bmbhzGGZyePZ/FH75OUmEjHzl2ZMuMF/P39mT5lAs9Nm8TRhPOzUJcvW8Lzz03jn7//okHDJsx4Ya5De/384y6eeuxhu73CuXvgUIaMGJWn9srP3TzS2uv9994hKTGRLl27M2v2+faaPHE8SSnnP8+WLlnM9KmT+cturxfnLcjSXg/cPyq9vZ4ZPzHP7VVY6vT6q6+w8LWcLQKQtsiF266D7Ji3flby6v8Brokzpwnyf8CzWD24FcBuYD7W0GuCMaZlvoKw/oHmGmNG28/LA9FAL2PMMhE5BLxsjJmQ4TWvA3WMMS3sBLkB6GOMWWLvvxX4EKv3mwzEAp2MMZszHON94KQxZmiGbXq7q0JGb3dVeOjtrgoPty8U0Glmnl6fvOohwIO3uzLGzBCRn4BYY8whERmI1RM7CtxXQLF8m+H9jqYN34hICFAR2Jyp/CagZ3bHsGNLUwvwA9akNYbN18lxlVJKuZsXTiTK8ZIsxphVGX7+BPikgGPJfoVe51LJGn92x0gr1xU4lGmfe694VkoplSfnju4h9eget71ftglSRHLcMzTGuGylaWNMvIgcBloAGzPsagX8ksPD/AGcBcqmnW8Uq4s6Hmuyjt7PUimlPCkHk3SKVbiSYhWudNh27l/XTYS7UA/ygRwewwCuvhXDDGCiiERjTeLpiLWCzy0XfJXNGHNKROYBL9hDrH9h3fD5AaCZSyJWSimVc4VpiNUYUz27fR4wGysRPwBEYvUI77SHenPqESABazZsRayJRt2NMdsLOFallFK55YWXebjvthAX4Gz2UcZtxphU4AX74ez1GwG50DZjzH9Yl3WMzX/ESimlCpQXJkjvi0gppZTyAl7Rg1RKKXWJK0znIJVSSim38cIh1lwlSBEJx7rF1S7gnDEmySVReYGUH9/Nss2nwpUUC7/KA9EopZTrfPH5clZ8vtyzQXhhDzKnS82FAMuwrj1MxVoU/DEgGBhgjDmV/asLF11qrvDRpeYKD11qrvBw+1JzN72ap9cnLxsCuCbOnPZpZwClgbpYF9yDdY/I+sDUgg5KKaWU8rScJsiewGPGmL1Y1yNijNkJPA70dlFsSimlLhVeeMPknJ6DLI7j4t9pDgIBBReOUkqpS5E77y+aUzntQX6HdS/FNGknLkcA3xdoREoppS45IpKnhyvltAf5IPCtiFyDdYuoaSJSB6gGNHdNaEoppZTn5KgHaZ97rAN8hXWbKx/gM+AyY8yPrgtPKaXUJUHy+HCh3NwP8jjW7aGUUkqpAuWN5yBzlCBFZNaF9htjHiyYcJRSSl2KcpIgz0bt4mzUD64PxpbTHuS1mZ4HAJcDicDKAo1IKaXUJScnCdK3UgN8KzVw2HZy/yZXhZSzBGmMaZN5m4gEAa8DXxd0UN5Al5pTSl0qvGKpOS+Uo6Xmsn2xyOXAYmNM/YILybN0qbnCR5eaKzx0qbnCw91LzQX3eytPr0/44C7ANXHm924etYGqBRGIUkqpS5j3zdHJ1ySdMOAmYEtBBqSUUurSU2hnsZJ1kk6axcDYAopFKaXUJaowJ8ibgThjTKorg/Emxz4a6ukQCly5G4rm1Tix373g6RAKnJ6rU8rzcroW659AY1cGopRS6tLljWux5jRBvg30c2UgSimlLl3emCBzOsT6LzBGRK4HfgCSMu7UlXSUUkrli/edgsxxguwC7LV/vjzTvrxfSKmUUkpRiCfpOFtJRymllCrKsk2QInIOqG2M2e/GeLzG6HuHZdnWpVt3unbr4YFolFLKdbxhqbmC7EGKiB/wDlAReNUY84a9PRRYinUd/2xjzMILHie7peZEJBWodaklyLRlj04mn/N0KAVOL/NQSuWUu5eaK3fPh3l6/bE3+gKOcYpIf6AC8CKwHmhnjDkrIqOAU8BbwDfGmOsvdOyczmJVSimlXKdgb5jcENhhjDkHHAQi7e2JWL3HUuRg/szFzkH2FpGjFzuIMSZvq8wqpZRSOA6xJv22lqS9a/NzuBAgyv45Cgi1f14K/AE8CVzwPsdw8QQ5PQeBGKzuqlJKKZVv/pe3x//y9jkqe/T1W51tjgcqA/uBKkCcvX0q0BvYBiwXkZeNMcezO/bFhlhrGWN8LvIoevcaUkop5VYFvFDADqChiBQDKgH/2NvDgKPAGeA0EHyhmPQcpFJKKY8r4AS5DLgB625TC4HHReQKYDLWynC7gD0Xm4Sa3/tBKqWUUvlWkJd5GGOSgT7Z7M7xuuIXSpCLgJO5CUoppZTKE+9bSCf7BGmMucedgSillFLeRIdYs6Er6SilLhVFbSWdgqKTdLLx4ksvZ3nkJDkaY5gycTxXX1GHWtUqce+wwSQmJmZbftnSxbRo2pjKFcK4qXtnDux3PGf89ZavaN+6BRFlQ7jy8lo889QTJCUlZXM013lyaGd2f/Ik+78cz/yn++FfMvsb+g65+QZ2fPgoRzdNY/3r99GiYS2H/TUql2XxrMEc3jCV3z4by9jhXfAt7v7J0MYYJk0YR73La1G9SgTDhwy6YFstXbKYZk0aEl62ND26dsrSVr/+8gudO7SlQpkQmjdtwqovV7q4Bs5pvSyFoV7eUqdu3Xswb8ErDg9388bbXWmCLGDTp05mwfy5TJz6LK8tfIvt27YyZOBdTst+tWkj99x1O7f2u40ln35BYFAQnTu0Tk+AUYcO0fvGrlxRrx4rVq9n0tRn+eiD93j4gfvcWSUeG9SREbe24InZnzHw6XdoUr8qCyfe4bTsqP6tmDS6B3Pf20SnYXPZuH0fn80ZxpW1KwLgV9KXz14cRqoxdB/5Eg89t4QBPa/niSGd3FklAKZNmcT8eS8yZdoMFi56h23bvmPggDudlt28aSN339Gfvv1vZ9nylQQGBdG+bcv0toqJiaFju1ZERlZl+YrVdOnajVt692Tn99+7s0qA1gsKT72KYp3yyhsTZLZrsV6q8rMWa0pKCpfXqspTY8dzz6AhAGzftpV2rW7g131/UalyZYfyd/S/hZIlSvL6oncASE5OplbVijw/Zx639O3PnBdmMXfO8/zy+wGKF7dGwz9f/im33Xozh4/HU6pUqVzFl5e1WIsX82HfF+OYsGAFbyz7DoDG9aqy8Y37qdN9PIeOxjuU/3v1RGYuWsecdzemb1s0+S5OJZ5h5OQPadWoNl8uGEl468eJP2X9xx7RtwVj7mjDZT0m5Do+yNtarCkpKdSuXoWx4yYycLDVVtu2bqV1i6b8vv8fKmdqq/59+1CyREnefPtdwGqrapXDmT13Pn379Wf+vLnMmjmd3/YdoFgxqzfcs3sXqlSJZO78l/NUr7zQehWeenl7ndy9Fmvle5fl6fX/vnQT4Jo4tQdZgH7Zs5uj0dF06tw1fVvDRo0JDQtj44Z1WcpvWLeWjp27pD/38/OjVZu2rFu7BoDkpCTatuuQnhwBypYthzGGw4ejshzPFerViiC8bDCrvv41fduOX/4hJiGR1o3rOJQtE1KK8mFBbN/9t8P2LTv/pOnV1QHwL+nL2bPnSDrzX/r+hFPJ+PuVcGEtstqzezfR0dF06nK+rRo1bkxYWBgb12dtq/Vr1ziU9fPzo3Xbdqxbs9rav24NHTp0Sv9gAujUuStr1652YS2y0npZCkO9imKd8qVg12ItEJogC1B09BEAKoSHp2/z8fEhIqIiR486LmmbmJhIQkICEREVHbZXrFiJY0ejAXjk8SdZ8Or5u7Gkpqby6oKXKFuuHJGRVV1VDQcVylgLTRw5kZC+zRjD4WMJVCgT5FA2/lQSZ/47S+UKpR22V60YRkS5EAC+++kAx+JOMf7ebviX9KVapTLcf0cbPln3o2srkklaW4U7aato+98/TVpbVayYta2O2mWPHDlCROb9lSpxNDoad47SaL0KT72KYp3ywxuHWL0iQYrIOBH54QL7W4uIEZHS7nzf3IqNiSEgIMDhGxxAYGAQJ044LvcXFxsLQKnAQCdlT2Q59r8HD3Jb35v56MP3mTZ9JiVKuKfHFRYcwOmkM6SmOv4HO5WYTJnSjkO8Z8+lsmTtLp4c2pkraobjW7wY3VvVZ/itzSnha/2bxJ1MYsgz7zHmzrbEfD2DXz99mgA/X/43c6lb6pMmJru2CgrixHHHtopNa6tSjm0VFBTEcbtsbGxMlrYMCgoiKSnJrZOqtF6Fp15FsU75oQmyiAsNCyMxMZFz5xzPXyYkxFO6dKjDttKh1vNTJ086KVs6/Xlqaipz57xAg6vqsmP7Nj5Y/Al9+9/umgo4EZOQSCn/kvj4OP4iBgf6E5uQ9T/dQzOWsntfFNvff4SE72Yy9f6evLbkGw4fs85VXnt5Zd6cfCdTXl1F87tm0m3kSxw6Gs+y2cPcOs077AJtFRrq2FZpz0+dcmyr+PjzZUNDw7K0ZXx8PCVKlMDf37+gw8+W1qvw1Kso1ik/vDFB6nWQBahCBWuo5HBUFJWrVAGsBHfkcJTDMApAQEAAwcHBREUdctgeFXWICuER6a+9567bWLr4Y0beN4Ynnx5HUJDjsKarRdtDqxXLhfBvdBxg/SJHlA3myPGELOXjTiZxx+OL8C/pS3CgH9EnTvLQ3e3457D1DXhQ72Z8+8MBJr58fvr5zl8O8u/ayTSuF8m2TOcvXSWtraKioqiSoa0OR0URbv/7p0lvq0NZ2yqtbHh4eNb9hw4RHhHh1sSv9So89SqKdSpq3NqDFJFQEXlTRA6LSIyIfCAiFTLsv1lEdovIKRFZIyKVMx2irohssff/LCKtMrz2MhFZKSKxIpIoIjtFpHuG/SVFZIaI/C0iR0XkdSCgIOt3Rb36lCtfntWrzn/479i+jbi4OFq1bpulfOu27Viz6sv058nJyWzeuIG27azbvLzz1pssXfwxi96xhlXdnRwB9vxxmOgTJ+nYrG76tsb1Iikd5M+m7b9nKT9xVHfuuvE6ks6kEH3C+jbbq93VfLx6JwCl/EuQmprq8Jq04dtSASVdVY0s6tWvT/ny5Vmd4Tqx7dvstmqTta3atGvPqgztmpyczKYN62nbvoO1v2171q5dzdmzZ9PLrF61knbtOriwFllpvSyFoV5FsU75kZPeYtKBbZxYP9fh4Upu60GK9RXmc6AY0A/rtiNTsFZa3w7UAoYDw4BqwBxgPDAow2FewbrR5RmsVdkXAjXtfYuw5jT1x7pr9BDgYxEpbYw5A8wEbgfuB3613+cBYE9B1dHX15dhw0cybuyTlCtfnqDAIB5+8H5u6n0zVSIjiYmJITY2hpo1rQvnh48YRY+uHbm2YSOuu74ps59/juDgEG68qTcASz7+kEaNm3D1tQ34888/HN4rMrIqvr7ZX6xfUM6eS2XBR18xYWR3jsWc4mRiMrP+dzOfrPuRg9FxhAYHEBocwP5/rfMgp5LOMPNhK/7f9h9hUO9mhAT68/HqXQC88/l2Pp0zjMcGdWTFV3soHeTPY4M68ue/x/nuxwMur08aX19fht87irFPPW61VVAQD44ZTa+b+xCZ1lYxMdSsZbXVvSNH07VTexo0aMT1TZvx/KwZBIeEcFMvq679brudqZMncO+wIQweOpzVq1ayccN6Nn+91W110noVrnoVxTrlSw46uQE1ryOg5nUO207tXuWigNx4HaTd21sP1DTG/GVv64CV8L4G/gdUMcZE2/vmA3WNMa1FpDWwAehtjPnE3t8X+CDt2hcReQj40hizx36edquT6sBxrBtmDjfGvGbv9wF+BM4ZY67JEKcBuLZBwxzXbcDAwQwcPBSwZnhOnjCOD99/l8SkRDp36cZzz8/B39+fKRPHM3XyBIdrLJctXcyMZ6fy918HaNS4CS+8OJ9q1a1LIhpdU5+9v/3q7C3Z/dufVK1WLccxQt6ug0zz9LAu9O/SEH+/EqzcsocHZywl+UwKTw7tzFNDO+PfaAwAxYr5MP7ebtzWrRHFfHzYsvNPHpqx1GEWbNcW9Xh0YAfq1gjnVOIZNm7fx7iXvuCfI7F5ii0v10GC1VYTxz/D+++9Q1JiIl26dmfW7Bfx9/dn0oRxTJ44nqSU8/8/li5ZzPSpk/nrrwM0bnIdL85bkN5WAL/s2cMD94/ih107qV3nMp4ZP5EOHd2/AILWq/DUy911ev3VV1j4Ws5Wydm58/u0GN1yHWT1B77I0+sPPN8NcE2c7kyQo4CHjTHVnOwbB9xmjKmTYdtzQKNMCbKSMSbK3t8dWJ4hQRYHrgOuAq4FOmPdSbo6EArsBGoYYw5keI8XgRbOEmReFgrwdvlJkN4srwlSKZU9dy8UUOPBFXl6/f5Z1rWhhX2hAF8g9QL7s1+A8CJlRMQPWIs1zFod+AIYmqHIWWevu0g8SimlLmHuTJC/AJEZJ96ISDMR2Y11PjI/WgItgKuNMY8YYz7FcUR7P5ACtMvw3mK/TimllIeJ5O3hSu68zGMtsBv4UESeAEoBE7DOD8bk89insZL9fSKyFmiGNRkH4AbgQ2A+MENEzmBN0hmI1dv8K5/vrZRSKp+88VIUt/UgjTHngE7AQeBj4G3gD6xZp/n1DfAMMAZYY79Pd/t9ngMCsSYBvQJMAr7EusTjkQJ4b6WUUvnkjT1IvZtHJjpJp/DRSTpKFTx3T9Kp88iXFyvq1O/TOwOFf5KOUkopVWjoUnNKKaU8zgtPQWqCVEop5XmZb4jgDTRBKqWU8jjtQRYio+8dlmVbl27d6dqthweiUUop1/ni8+Ws+Hy5R2Pwxss8dBZrJjqLtfDRWaxKFTx3z2Kt9+TqPL1+z+SOgGvi1B6kUkopj/PCDqQmSKWUUp6XkyHWhN+/JeH379wQjUUTpFJKKY/LSYIMuawZIZc1c9gWuytvdwHJCU2QSimlPM4bh1h1JR2llFLKCe1BKqWU8jhvvMxDE6RSSimP88L8qAlSKaWU52kPUimllHLCC/OjJsjs6FJzSqlLhTcsNeeNdKm5THSpucJHl5pTquC5e6m5RpM25On1O55qA+hSc0oppYooHWJVSimlnPDGSTq6UIBSSinlhPYgs1F9xMeeDqHAFdVzdUn/Fb3zxcdPnvF0CC5RpUyAp0NQXsoLO5CaIJVSSnmeNw6xaoJUSinlcV6YHzVBKqWU8jztQSqllFJ5FPPL18T+8o3b3k8TZDZOfv1Klm0lqjSkZGRDD0SjlFKu4w0r6eSkA1mm3g2UqXeDw7boba6LWxNkNoJuGOrpEJRSyi26de9Bt+6Oy2gufP1Vt8agQ6xKKaWUE5oglVJKKSe8MD/qSjpKKaWUM9qDVEop5XE6xKqUUko54YX5UROkUkopz9MepFJKKeWEF+ZHnaSjlFJKOaM9SKWUUh7n44VdSE2Q2dCl5pRSl4rCstScu+kQazaCbhia5ZHT5Pi/nvXYNrUru2fdyAv3NMa/RLFsyw5tX5vvpnTlwEu9WfZIG66qGuq03E1NqrBo1A1O97maMYZJE8ZR7/JaVK8SwfAhg0hMTMy2/NIli2nWpCHhZUvTo2snDuzf77D/119+oXOHtlQoE0Lzpk1Y9eVKF9fAOWMM0yaPp+GVl1G3RmVGjxhywXp9+skS2tzQhGoVy3DzjV3468D+bMt6kjGGOTMm0/76K7nhqho88cAIki5QrzRjH7mPN1+Z64YI86Yo/h56S526de/BvAWvODzcTUTy9MjmWH4islhEvhGRezJsLyYiC0Rkl4jcd7GYNEEWsAd7XMHgdrUZ9/GPjHj1OxrWKMNLg69zWvaOljV4oveVzFy+h57TNvD74QQWP9SKcsElHcrVqBDI/26s747wnZo2ZRLz573IlGkzWLjoHbZt+46BA+50Wnbzpo3cfUd/+va/nWXLVxIYFET7ti1JSkoCICYmho7tWhEZWZXlK1bTpWs3bundk53ff+/OKgHw3LOTeXXBPMZPfpYFry9ix/atDB98t9OyWzZvZMiA2+nTtz8fffI5gUFBdOvYJr1eAA3q1yEirJTDY+SwQe6qTrqXnn+WdxYu4JGxk5kx73V++H47/xs9ONvyxhi2bFzLso/fc7r/rpu7UL9qmMPjrpu7uCr8bBXF38OiWKe88pG8PbLRC9gCtAAGiEjaaGl7IAFoCHQQkex7L4AYYwqmdkWEiBiAsvd8kOvXFi8m/PhcD6Z+spt3Nlvf7BrUCGPlE+255n/LORyb5FD+s0fb8OPfsTz9wQ+ANQa/+/kbmbj4J97fcgCAXTO6U7lMKQBW7PyXu+d+nee6HXylb65fk5KSQu3qVRg7biIDBw8BYNvWrbRu0ZTf9/9D5cqVHcr379uHkiVK8ubb7wKQnJxMtcrhzJ47n779+jN/3lxmzZzOb/sOUKyY9bvZs3sXqlSJZO78l/NUr6T/zuWpXlfWqcbjY8dz9z1W8tixfSud2jTnp70HqFTJsV53334rJUuU5JU33k6v1+U1KjHzhXncfGs/Tp48SdXwUN75cCm1atdJf11QcDARERVzHd/xk2dy/Zq0erVqUIf7Hx1L3zusL84/7tzOrd3asOn7vYRXrORQfs9Pu7irTzdOJsQD8MSEZxkwdJRDmSZXRHL//57i+uat0rf5+flTqUpkruOrUiYg16+BwvF7WNTq5O9rZR9jjEsHP9M+czu/9F2eXv/lvdcDjnGKyHPAMmPMFhF5BxhrjNkvIlOAb40xy0UkBDhljMn2A0R7kAWobqUQyof4s/anw+nbdh2IIfb0f7SsWyFLeb8SxTmVfDb9eaoxJJ456zAk2/+Fr2jx9Jes3HXItcFnY8/u3URHR9OpS9f0bY0aNyYsLIyN69dlKb9+7RqHsn5+frRu2451a1Zb+9etoUOHTun/gQE6de7K2rWrXViLrH7ds5ujR6Pp0Ol8T6hBw8aEhoWxecP6LOU3rl/rUNbPz4+Wrduyft0aAP7YtxcRoU27DtS57PL0R16SY37s+20Px48dpVW7TunbrrymISGhYXzz1YYs5avXrMP7n67h8w3bKF8hPMv+mBPHiYs5QbOWbahZ+7L0R16SY34Uxd/Dolin/Mg4bHpwyzK+mTYgR49shABR9s9RQNq5q7JALxH5Fhh1oeQIOkmnQJUP8QPgaHxy+jZj4EhcEuWC/bKU//z7gwxpX4cvvv+XvVEJ3N26JmWCSrJx95H0Mr8dsr7Zx5/+j+AAXxfXIKvoaCuW8PDzH54+Pj5ERFQk+mi0Q9nExEQSEhKoWNExKVSsWIn9f/4BwJEjR7jq6msc91eqxNHoaIwxbrtYOK1eFSo41is8vCLHnNTrZEIC4RERDtsz1mvf3r0Eh4QwbOCdfPP1V5QpU5aevfvw4P8ex88va9u7Slrs5cqf/0Lm4+NDhQrhnDh2NEv5gFKlqFO3HgC+viWy7N//x+8AzHt+Gl9vXEdAqUBatevEmEefJjiktAtq4FxR/D0sinXKj4zhVW3Zi6ote+XodSuGOz2FFQ9UBvYDVYA4e3sSsAcYBiwVkTrGmN+zO3ahSZAi8iZQ2hhzUwEf9y/gBWPMC/k9VulSJTl95iypmYatTyefJSwo64fPnBW/0enqSmwYf/7b/v0Lt7H/6Kn8hlJgYmJiCAgIcPhWChAYFMSJ48cdtsXGxgJQqlSgw/agoCCO22VjY2MoFZh1f1JSEklJSQQE5G0ILrfiYmOzqVcgJ0441isuzq5XprgDAwOJscvu+/03kpOSuL5Zc8Y8/Ci/7tnDuKcf49jRozz/4nwX1sRRfFws/v5Z61UqMIjYmBO5Pt7+P35HRKhSpRoL3lpM1L//MGPS0/y1fx+vv/+p2z50i+LvYVGsU34IBfq7tANoKCJfA5WAf+ztPwCnjTEpInKSi4yiFpoEWRjEnT5DqZLF8RFxSJJB/r7En07JUn7uoCaU8ivOHXO+4nBsEk3rlGPcrVcTHZ/Eup+PZCnvCWFhYSQmJnLu3DmH/8gJCfGEhjrOuE17furUSYft8fHny4aGhnHqZNb9JUqUwN/f3xVVcKp0aKjzesUnULq0Y73SnmeOOyEhgRB738j7HmTEqDGElSkDWMO1xYoVY9TwQUx+dqbbPpxCSoeSlJS1XicT4gkuXTrXx+t6Y2/adexKmXLlAbi6QSPKlivP7b068df+P6hes3ZBhX5BRfH3sCjWKT8uMOEmL5YB7wC3Ai8Dj4vIYuAj4EMReQzYYIz57UIH0QRZgNKGVsNL+xFlT8gRsZ5HxztO0Akv7U/fG6rT5plV7D4YB8BPf8dSMSyAoe3reE2CTBuCjIqKokqVKgCkpqZyOCqK8HDHIceAgACCg4OJOuR4vjQq6lB62fDw8Kz7Dx0iPCLCrUNAafU6fDiKypXP1+vIkSgqOKlXUHAwh6OiHLYfzlCv0pk+0ACuuuZa65iHo6hRs5YrqpFF2tDq0SOHibAnGqWmpnI0+gjly2c9x3gxgUHBBAYFO2y74qprAIg6dNBtCbIo/h4WxTp5C2NMMtAnm93dc3ocr5mkIyJGRNqIyJsickxEDonIA07K3Ssi+0TkpIgsEZHgDPvuFpFfRCRZRI7b+8Mz7K8jIqtEJF5E9oiI8/nUefTroXiOxifT7qrzv9wNqpchJKAEX/3qeP4nwJ6Ik3k41hhDKT/v+d5Sr359ypcvz+oM11Nt37aNuLg4WrVpm6V8m3btWbXqfNnk5GQ2bVhP2/YdrP1t27N27WrOnj0/OWn1qpW0a9fBhbXIqm69+pQrV561q79M3/b9jm3Ex8XRsnWbLOVbtWnnUDY5OZnNmzbQum17AO7q34exTzzi8Jq9v/2Kn58fkVWruaYSTtS+vB5lypZj0/rzEzN+2rWDhPg4rm/eOtfHm/DEQzw4YgAZZ7v/sdf60l2z1mX5jjeniuLvYVGsU34U5HWQBcV7Pokts4HXsLrE9wOzRGSNMWa3vb8NcAa4A2gAvGCXmygiVwBvADOBj7FOzM4ApgADRSQI2Az8hvUNopT9ese51Plw9pzh9fX7eOrmqziecIZTySlMvb0By78/yKGYREqXKkFoqRIcOHqK/UdP8d3vx5g76DomLfmJ6Phkrq9dlkHtavPYO95z3ZKvry/D7x3F2Kcep1z58gQFBfHgmNH0urkPkZGRxMTEEBsTQ81aVg/p3pGj6dqpPQ0aNOL6ps14ftYMgkNCuKlXbwD63XY7UydP4N5hQxg8dDirV61k44b1bP56q9vrNXj4vUx85knKlStPYGAgjz48hht73UzlKpHExsQQGxuT3vMbOnwkvbp34poGDWlyfVNefGEmwcEhdO9pTSRo17EzD44eQWhoGK3btefA/v08/djD3Pfg/yhe3H3/zXx9fblj4HBmTXmGMmXLUSowkIlPPEzn7r2oWLkKcbExxMfFUrV6zRwdr23HLgzs15PyFSLo3KMXsSdO8OzEJ+l16+1ZLhlxpaL4e1gU65Qf3tjJ9bYEuckYMwdARP4E+gK1gLQEmQzcbXeft4rIjUDa//RU4BHgeXvq7jYR6QykjQHdhpUUextjYuz3uAPY7iyQuM+eyHHQfpe1xe8yqycxa/kv+BbzYUK/awgoUYzVPx7m8Xd3AjCkfW0e6VmfcgM/BOCuF7fwWK/6zLy7EWGBJdkffZJH3v4+/RpIb/HYE0+RkpLCo/97kKTERLp07c6s2S8C8NLcOUyeOJ6kFKuH0bJVa9569wOmT53M1MkTaNzkOtas25R+DiQsLIxVazfywP2j6NG1I7XrXMaSZcu55tpr3V6vhx99kpSUFJ567GGSEhPp2KUbz86cDcDL819k+pSJxJy2vo03b9ma1xa9x6zpU5kxbRINGzXh81Xr0+t114BBJJ4+zWsvv8T0qROJrFadUWMeYuiIUdm+v6vc+8CjnE1JYeozj5GclEjrDl0YO3kmAG+9Np+5M6fw+5HTOTpW89btmfnSQhbMnsG7b75ChfCKdO91CyPuf+TiLy5gRfH30N11ev3VV1j4mvtXyckJb1yL1WsWCrAvFr3dGPNepm29jDHL7FmsFY0xHTPsX4x1oecA+3k54DrgKqAJ0Bn4zhjTWkRmA9cYY1pleL1grarwdNos1vwsFODt8rJQQGGQl4UCvF1eFwrwdnldKEC5n7sXCuj12o48vf6TwY0A18TpbT3Iiy0Yme1+EWkGfAbsBFYCzwGHgbp2kbPZvLTofboqpVQh44UdSK9LkPkxClhrjOmXtkFERmfY/yswVETC0oZYsXqaIW6MUSmllBPeONPWa2axFoDTQEsR6SQibe0h1d5AFRGpAbwPnASWiEgLEemAdZ3MyewPqZRSyh1E8vZwpaKUIMcCPwJLsWbCnsE6D1kMeNQYcxpoBfwHLAfmYA3DbvNItEoppdL5iOTp4UpeM8Tq7ARrxm1pE3Ey7e+T4efDgLN78FTLUGYf0CnT/kW5j1YppVRR5zUJUiml1KXL+85AaoJUSinlBbxxko4mSKWUUh6Xk8XK/921iUO7Nrs+GJsmSKWUUh6Xkx5klQatqdKgtcO2PzcudVFEmiCzdfLrrMsxlajSkJKRDT0QjVJKuc4Xny9nxefLPR2G19EEmY2gG4Z6OgSllHKLbt170K17D4dtC19/1a0xeOEpSE2QSimlPE8n6SillFJO5GSSjrtpglRKKeVx3tiDLEpLzSmllFIFRnuQSimlPM77+o+aIJVSSnkBVy88nheaIJVSSnmcF+ZHTZBKKaU8zxsn6WiCzIaupKOUulToSjrOaYLMhq6ko5S6VOhKOs5pglRKKeVxOklHKaWUcsIL86MmSKWUUp6nk3QKkRIlS3g6BJVDJYsXvQWhTiad9XQISnmdv7Zv4K8dG932fpoglVJKeVxOvubWaNyGGo3bOGz7Zc3HrgkITZBKKaW8gA6xKqWUUk544+2uit7JG6WUUqoAaA9SKaWUx3ljD1ITZDbiNs3Pss2vaiP8qjX2QDRKKeU63rDUnJ6DLERKtxrh6RCUUsotvGGpOe1BKqWUUk54YQdSJ+kopZRSzmgPUimllMfpYuVKKaWUE944nKkJUimllMd5YQdSE6RSSinP88YhVm/s1SqllFIepz1IpZRSHueFHUhNkEoppTwvJwsF7P1uPb9/t971wdh0iDUbcZvmZ3kk/7U9R699sNvlbBnfgZ3TuvDcHdfi51ssz3FcGVmajx9ozt7nu7NlfAcGtamZ52PllTGGSRPGUe/yWlSvEsHwIYNITEzMtvzSJYtp1qQh4WVL06NrJw7s3++w/9dffqFzh7ZUKBNC86ZNWPXlShfXwDljDJMnjuPKurWpUbUiI4ZduF6fLF3MDdc3omL5UG7s1jlLvdJ8/NEH9O3Ty1VhX5QxhpdfmErP1tfQsUkdJjw6kqQk5/UyxrD0/Tfp06EJN1wRwa2dm/LZ4ncxxmQp+/Ou7fTv2tzV4WerKP4eekudvvh8OSOHD3V4uJuPyEUfdZu2o+cDkx0eLo3JpUcvxEq3GpHlkZN1WMd0uYyBrWsyaelu7ntjBw2qhzHnnoZOy/4558ZsH5XC/CkbVJL377uB/dGnuHnWV8z8/FceubEu/W+oWtDVvaBpUyYxf96LTJk2g4WL3mHbtu8YOOBOp2U3b9rI3Xf0p2//21m2fCWBQUG0b9uSpKQkAGJiYujYrhWRkVVZvmI1Xbp245bePdn5/ffurBIAz06dxIKX5jJ52nRef+Nttm/dyuB77nJadvOmjQy48zb69ruNTz5bQVBQEB3bt0qvV5p9v//O1EkT3BF+tl57cQYfLnqZ+x+fyKTnX+HnXTsY+6DzD7xNa1Yw7ekHufm2gbz+8Zf06ns3U54cw8plHzmUO3okitlTx7oj/GwVxd9Db6lTt+49mLfgFYeHu4nk7eHSmJx9U7yUiYgBqDh8aa5fW9xH2D6lMzOW/8p7X/8FwLXVQvnsf61o8uSXHI5Ldihfs0JglmPc27EOEaX9uGPuN9xyfVWe7FWPax9byblUq53G3lyfqyJD6fP8V7mO788Xc9+rSUlJoXb1KowdN5GBg4cAsG3rVlq3aMrv+/+hcuXKDuX79+1DyRIlefPtdwFITk6mWuVwZs+dT99+/Zk/by6zZk7nt30HKFbM6ln37N6FKlUimTv/5VzHB5Camvvf4ZSUFOrUjGTsMxO4Z5BVr+3bttKmZTP2/vE3lTLV6/Z+t1CiZEneWPROer1qREbwwosvcWvf/gBcVqsq/x48CED3Hj35cPEneaoPwG9RJ/P0upSUFLo1u4LhDz5J7/4DAPh51w7uubk9X3y9hwoRlRzKjxnUF/+AAKa++Eb6tpkTH+enndtY9Mk6AAbd0pkfdnwLQJ26V/L+ii15ig3gisrBeXpdYfg9LGp18ve1so8xxqVpKO0zd8KafXl6/dgOtQHXxKk9yAJ0WcVgyof4sX73kfRtP/wdS1zifzS/vHyW8n9Gn3J4hAWWpPll5Rj95g5SDfiV8CE55Vx6cgQ4mXwW/xJ5H7LNrT27dxMdHU2nLl3TtzVq3JiwsDA2rl+Xpfz6tWscyvr5+dG6bTvWrVlt7V+3hg4dOqX/Bwbo1Lkra9eudmEtstqzZzdHo6Pp1Pl8rA0bWfXasMFJvdatoVPnLunP/fz8aN2mHevWrknftvTTL9i28ye6db/RtcFfwJ+//8KJ40dp3qZj+rZ6VzcguHQo277ZlKW8iNDguhsctoWGlSH68KH052OfncuHX35LnzsGuS7wiyiKv4dFsU754SN5e7g0Jtce/tJSPsQPgKMJ53uKxsCRuGTKBpW84GtLFvfhhbsbMPbjnzhx8j8ANv9ylJAAXwa1qUmJ4j7UrRTM7TdU44tdhy54rIIUHW0l+/Dw8PRtPj4+RERUJPpotEPZxMREEhISqFixosP2ihUrcdQue+TIESIy769UiaPR0U7Pe7lK9BGrXhUy1Ss8oiJHo53XKyIic70cy9arV5969epTOjTUhZFf2IljRwEoU65C+jYfHx/KlY8gxt6X0fOvfcAtdwxOf54QH8uKZR9R7+rzpwWq1qhFrcuuIKxMORdGfmFF8fewKNYpPySPf1xJZ7EWoNIBviSeOUvmEb/TZ84SFljigq8d1LYmh2KSWPXj4fRtB46dZsKS3Uy//Vom3HoVAN/tO85Lq/M2FJEXMTExBAQEOHwrBQgMCuLE8eMO22JjYwEoVcpx6DgoKIjjdtnY2BhKBWbdn5SURFJSEgEBAQVdBadiY53XKygoiBMnHOsVZ9crMFPcgU7Kelp8XCx+/lnrVSowkLjYExd87c+7tjPh0VEcjT7M9JfecmWYuVYUfw+LYp3ywxtvd6U9yAIUl5hCQMniWRo6yN+X+MSUbF8X5Fec+zpfxvTPfnHY3rZeBZ7oVY9xH/9E56kbGDD/W4L9fZl1ZwNXhO9UWFgYiYmJnDt3zmF7QkI8oZl6SmnPT51yPH8WH3++bGhoGKdOZt1fokQJ/P39Czr8bIWGOq9XfHx8lh5g2vOTmeJOiI+ndGnP9RadCSkdSnJS1nqdOplAUEhpp685dTKBCY+OYkDv9vj5+/PmkjXUqH25G6LNuaL4e1gU65QfOsRaxB2Nt4ZWw0uf/2UUgQohfun7nOnbrCoHTySyY3+Mw/bBbWuy+Lt/eHX9n/z8TxxrfjrC6Dd20LdZVSrYw7muVqGCNfwTFRWVvi01NZXDUVGEh0c4lA0ICCA4OJioQ45DwFFRh9LLhoeHZ91/6BDhERFuvaN42tDq4Uz1OnL4AvWKylyvKMIjHMt6Wply1rnuY9HnRyJSU1M5Fn2YsuXDs5SPi43hrpvasvrzpTw1dQ6LPlnvdckRiubvYVGsU1FzSSRIEflSRLJM4xKRbSIyraDeZ29UAscSkmlT7/z5n2urhRLi78uWvceyfd3dLWvw4bd/Z9keULI4qamO2wzW+K27JurUq1+f8uXLszrD9VTbt20jLi6OVm3aZinfpl17Vq06XzY5OZlNG9bTtn0Ha3/b9qxdu5qzZ8+ml1m9aiXt2nVwYS2yqlevPuXKl2d1hlh3bLfq1bq1k3q1bc/qVV+mP09OTmbTxvW0bdfeLfHmVM061rnCbzaenzy058fvOZkQT+OmLbOUnzXxcU7Gx/HGkjX06nc3Pj7e+ZFQFH8Pi2Kd8kNE8vRwpUvlHOQC4F0RedQYEwcgIo2BRkDfgnqTs6mGNzfu5/Ge9Th+8gynk88yse9VfLHrEFGxSZQO8KV0qRL8dex0+msuqxhEjQqBbP416wSKD7/9m0l9r+av46fY9scJwkv781jPK/hu33GHY7iSr68vw+8dxdinHqdc+fIEBQXx4JjR9Lq5D5GRkcTExBAbE0PNWrUAuHfkaLp2ak+DBo24vmkznp81g+CQEG7q1RuAfrfdztTJE7h32BAGDx3O6lUr2bhhPZu/3uqW+jjUa8RInnn6CcqVK09gUBAPP3AfvXr3oYqTeg2/dxTdu3SgQYOGXN+0GS/Meo7gkBB63tTbrXFfjK+vL7feNZS5M8YTWrYcpUoFMn3cI7Tr0pOISlWIj4shIS6WKtVqcu7cOdauWMZN/e7Gz8+Pg3/9mX6cYsWLU7Gye6+3vZCi+HtYFOuUH954DvJSSZCfA7HAXcAce9u9wJfGmAMF+UYvrNxL8WI+PNPnSvx9i7F29xGe/vBHAAa2qclD3etSacT56+Pa1Q/nxMkz/H4463Vv73/9N+fOGQa3rcnTvesTc+o/1vx8hOeW/1qQIV/UY088RUpKCo/+70GSEhPp0rU7s2a/CMBLc+cweeJ4klKsnm3LVq15690PmD51MlMnT6Bxk+tYs25T+jmQsLAwVq3dyAP3j6JH147UrnMZS5Yt55prr3VrnQAefdyq12OPPERiUiJdunZj5vNWvebPm8OUSRM4fSY1vV6L3nmf6dOmMG3KRBo1vo5VazZ65bmdwaP/x9mzKTw/6QmSk5Jo3rYTj4yfAcAHb77MK7On8f2BeOJjYzhzJpkPF73Mh4scB1giKkXy+ZafPRF+tori72FRrFNeeeMo8CWzUICIjAVuA+oCYcC/wK3GmOWZyhkA33I5X9ItoG4HSl3R8eIFPSwvCwUUBnlZKMDb5XWhAG+X14UCVMF4/dVXWPhazlbJ2bnTWoHHXQsFPL/Z+dKNF/NAyxqAa+K8VHqQAK8BY4G2wLXAMWBFdoXL3TzDTWEppZR7DBoylEFDcrbOatpKOoWRiPgB7wAVgVeNMW9k2n8rUMMYc8E5KN55Rt4FjDFRwKfAaGAE8Iox5tyFX6WUUsodCvgyj17AFqAFMEBE0juDIuKL1Vm6eEz5rFNhMx/oCUQCr3s4FqWUUrYCXqy8IbDD7gQdxPrMTzMUWJWTmC6lIVaA9cB+4EdjzOGLFVZKKeUePhmWjdvy6Xt8/dkH+TlcCJB2gWkUEAogIkFAG2AecN3FDnJJJUhjTKqIHAT+8XQsSimlnGve8zaa97wtR2VHNa/ubHM8UBmrQ1QFiLO3Pwg8D/jm5NiXVIJUSinlnQr4Mo8dQEMR+RqoxPlO0WVYPcjSQLCIbDPGrM/uIJoglVJKeVwBLxSwDGsW663Ay8DjIrLYGHMbgIi0Bq6/UHKESzBBGmNaezoGpZRSjnwKsAtpjEkG+lxg/0Zg48WOc8klSKWUUt7HG1fS0QSZjbhN87Ns86vaCL9qjT0QjVJKuc4Xny9nxefLL17wEqMJMhulW43wdAhKKeUW3br3oFv3Hg7bFr7+qltjKMgh1oKiCVIppZTHeWF+1ASplFLK87xxWTdNkEoppTzO1Tc/zgtvTNpKKaWUx2kPUimllMd5X/9RE6RSSikvoLNYlVJKKSe8Lz1qglRKKeUFvLADqQlSKaVU4bBz8xp2bl7rtvfTBJkNXWpOKXWp8Ial5nJymUfDVh1p2Kqjw7YNn7znqpA0QWZHl5pTSl0qvGKpObe+W85oglRKKeVx3rhQgCZIpZRSHud96dE7e7VKKaWUx2kPMht/zLnJ0yEUuNDGozwdgkvEbp/r6RAK3BWVgz0dglJupUOsSimllBPeOJypCVIppZTHaQ9SKaWUcsL70qN39mqVUkopj9MepFJKKY/zwhFWTZDZGTliaJZtXbtlXW1CKaUKO29Yas7HCwdZxRjj6Ri8iogYgMT/Uj0dSoELazLa0yG4RFG8zEMpT/P3tRKWMcalmSvtM3f5z0fy9PoeV4YDrolTz0EqpZRSTugQq1JKKY8TLxxi1QSplFLK43SSjlJKKeVETibpfLdxFVs3rnZDNBZNkEoppTwuJz3Ipm060bRNJ4dtXy5+20UR6SQdpZRSyintQSqllPI4PQeplFJKOaGzWJVSSiknfLwvP+o5yOyMHDE0y+OLHCzFZIxh0oRx1K9bm+qRFRk+dBCJiYnZlv9kyWKaXdeIiHKh3NitMwf273da7uMPP+DWm3vltTr59uSwruz+9Bn2r57M/Gduw9/PN9uyQ25pzo6Pn+DoludY/8YDtGhY22F/ZEQYHzw3mH83PMuvn4/j+UdvoWQJ939XS2urepfXonqVCIYPuXBbLV2ymGZNGhJetjQ9unbK0la//vILnTu0pUKZEJo3bcKqL1e6uAbOab0shaFe3lKnLz5fzsjhQx0e7iZ5/ONKmiCzMW/+K1keOVmHddqUSSx4aS6Tp05n4Ztvs33rVgYNuMtp2c2bNnL3nbfRr/9tLFu+gsDAIDq0a0VSUpJDuX2//86USRMKpF558diQzozo14onXviEgU8tosmV1Vk46W6nZUfd1ppJ9/Vk7nsb6DRkNhu3/85n8+7lyjqVAAgMKMm6hQ9QvHgx+oxZwJhpH9GycR1em3CnO6sEWG01f96LTJk2g4WL3mHbtu8YOMB5HJs3beTuO/rTt//tLFu+ksCgINq3bZneVjExMXRs14rIyKosX7GaLl27cUvvnuz8/nt3VgnQekHhqZe31Klb9x7MW/CKw0PpWqxZ5Gct1pSUFGrXiGTsuAkMHDQEgG3bttKmRTP2/vk3lStXdih/W99bKFmyJG+89Q4AycnJVK8SwewXX+LWfv0BqFOzKv8ePAhA9x49+WjJJ3muW17WYi1e3Id9Kycx4aXPeeOTbwBoXL8qGxc9RJ0uYzl0NM6h/N/rpjLzjTXMeWd9+rZFUwdwKvEMIye+z9BbWvDEsC7U7f4MSckpANSuWp5dS57iih7P8M/h2FzHmJe1WFNSUqhdvQpjx01k4GC7rbZupXWLpvy+/58sbdW/bx9KlijJm2+/C1htVa1yOLPnzqdvv/7MnzeXWTOn89u+AxQrVgyAnt27UKVKJHPnv5zr+PJK61V46uXtdXL3Wqzrfzuep9e3vbwsoGuxer09u3dzNDqaTp27pm9r1KgxYWFhbFy/Lkv59evW0Klzl/Tnfn5+tG7TjrVr16Rv++TTL9i+8ye6db/RtcFno17NioSXDWbVlj3p23bs+YeY+ERaX3eZQ9kypUtRPiyI7T//5bB9y84/aHpNDQDq1ghn976o9OQIsO/voxyPO0Xj+tVcVo/M9uzeTXR0NJ26ZGirxhdoq7VrHMr6+fnRum071q2xLlpev24NHTp0Sv9gAujUuStr17rvombQeqUpDPUqinXKDx1iLeKio63V6MPDw9O3+fj4EBFRkaNHox3KJiYmkpCQQETFig7bK1ZyLFuvfn3q1a9PaGioCyPPXoWywQAcOZGQvs0Yw+Fj8VQIC3IoG38qiTP/pVA53DHWqhFliCgXAkB0zEkqVyjtsD+olB9lQkqll3GHC7VVdDZtVTFzW1WslN5WR44ccdKWlTgaHY07R2m0XoWnXkWxTvnhI3l7uDQm1x7+0hIbE0NAQIDDNziAwKAgThx3HD6IjbWGEksFBjqWDcxa1pPCggM4nXSG1FTH/2CnEpMpU7qUw7azZ1NZsnoXTw7rwhU1I/AtXozura9keL+WlChuTcL5ZO0uqlcuyyODOhHgV4KIciG8NuFOihcvRglf903UiclLW5VybKugoCCO22VjY2OytGVQUBBJSUlZzim7ktar8NSrKNYpP7QH6SVEpLWIjCvo44aGhZGYmMi5c+cctifEx1M6Uw8wrUd46uRJx7IJ8R7rLToTk5BIKf+S+GT6qhYc6E/syaz/6R6a/jG790Wx/aPHSdg+m6kP9OK1xVs4fCwegL0Hornz0YUM79uSE9/OYu8XE/jncAwH/j1OlF3GHcKyaysn//7pbXXKsa3i48+XDQ0Ny9KW8fHxlChRAn9//4IOP1tar8JTr6JYp6JGr4MsQBUqWEMlh6OiqFylCgCpqakcPhxFeESEQ9mAgACCg4OJOnTIYXvUoSjCwx3LelL0cWtotWK5EP6NjgNARIgoF8IRJwkt7mQSdzy6EH8/X4JL+RF94iQPDWjPP4dj0st8tuEnPtvwExXLhXA87jTnUlMZ0KsZBzOUcbW0toqKiqJKxraKyvrvn21bRR1KLxseHu6kLQ8RHhGBuHGJEK1X4alXUaxTfnhjiJdUD1JEmojIb8BbwCgR+U1ERhXU8evVr0/58uVZter8tUc7tm8jLi6O1q3bZinfpm17Vq/6Mv15cnIymzaup2279gUVUr7t+TOK6BMJdLyhXvq2xvWrUjrIn03bf89SfuJ9N3JXz+tJSk4h+oT1bbZXhwZ8vMqaat7gikjenDKAkEB/oo7F81/KWTrdcAVxCYl8+6Pza0BdIa2tVme4Tmz7NqutWrVx0lbt2ju0a3JyMps2rKdt+w7W/rbtWbt2NWfPnk0vs3rVStq16+DCWmSl9bIUhnoVxTrlh+Tx4UpFpgcpIiWBCUAfIBz4HnjCGLMlrYwxZhtwuYi0BlobY8YVZAy+vr4MGzGSZ556gnLlyhMUFMRDY+6jV+8+VImMJCYmhtiYGGrWqgXAiJGj6Na5Aw0aNuT6ps14fuZzBIeE0LNX74IMK1/Onk1lwYebmTD6Ro7FnORkYjKzHr2FT9b+wMEjsYQGBxAaEsD+g9Z5kFOJZ5j5yC0A/Lb/CINubk5IoF96gtx/8Bjtrr+cVyfcwex31hNeJphZj93KxPlfZDnP6Uq+vr4Mv3cUY596nHLlrbZ6cMxoet3ch0gnbXXvyNF07dSeBg0aWW01awbBISHcZLdVv9tuZ+rkCdw7bAiDhw5n9aqVbNywns1fb3VbnbRehateRbFO+eHjhV3IInMdpIi8BdQDngBOAPcAg4FrjDG/ZirbmmwSZNo1Odc2aJjj9x44eAiDBlsrTxhjmDj+GT54/10SExPp0rUbs154EX9/fyZNGMeUSRMcrrH8ZMlinp02hb//OkDjJtcxZ+58qlWvnuU9hg66h7i4OLdfB5nm6RHd6N+1Mf5+JVj51W4efPZjks+k8OSwrjw1vCv+11od8WLFfBg/sge3dW9CsWI+bPl+Hw9NX8yR4+dnwTa8IpLnHunDlXUqsfdANK8t3pJ+jWVe5OU6SDjfVu+/9w5JiYl06dqdWbPPt9XkieNJSjn//2PpksVMnzqZv+y2enHeAoe2+mXPHh64fxQ/7NpJ7TqX8cz4iXTo2MnJO7uW1qvw1MvddXr91VdY+FrOFgHYufP7tBjdch3kt/sufg30V+tW8vWGVQ7bPv1wEeCaOItEghSRasB+oLox5m97mwDfAMuNMVPsbdcB72Z6+RxjzJwMx8rzQgHeLj8J0pvlNUEqpbLn7oUCcpIgnWla25qk5Io4i8oQa32s4ei9mU5GlwB+S3tijNkK1HJvaEoppS7K+0ZYi0yCLA6kAg2Bc5n2nXZ/OEoppXJDb3flOr9izcgNtHuJiIgvMBNYDhz0YGxKKaUuwgvn6BSNyzyMMXuBT4G3RKSbiNwAvALcDWS9FkEppZRX8cbLPIpEgrTdAawBXgJWAdWADmmTdpRSSqncKCpDrBhjTgGj7IdSSqnCxAuHWItMglRKKVV46SQdpZRSyglvnKSjCVIppZTHeWF+1ASZnZEjhmbZ1rVbD7p17+GBaJRSynW++Hw5Kz5f7ukwvE6RWGquIOlSc4WPLjWnVMFz91Jz3/+Vt/vBNqwWArgmzqJ0mYdSSqlCSvL4x+mxRPxEZLGIfCMi92TYXkpE1ojIjyLyqlzkRpmaIJVSSnmcSN4e2egFbAFaAANEJO104s3ABuAawAA3XCgmTZBKKaWKmobADmPMOaylRiPt7fuA9411bvHIxQ6ik3SUUkp5XMbO4OL33mDpe2/m53AhQJT9cxQQCmCM+VYsd2D1Hide6CCaIJVSSnlehgzZ5/Z76HP7PdmXzeCayGBnm+OBylj3Ca4CxEH6fYKfBSoANxljUi50bB1iVUop5XEFOUkH2AE0FJFiQCXgH3t7L/vvAcaYkxeLSROkUkopjyvgSTrLsIZQtwALgcdF5AqgCdAO2CAiG0Wk+YVi0iFWpZRSRYoxJhno42TXY/YjRzRBKqWU8jhdaq4Q0aXmlFKXCq9Yas4LM6QuNZeJLjVX+OhSc0oVPHcvNbf731N5en39yoGAa+LUHqRSSimP88bbXeksVqWUUsoJ7UEqpZTyOC/sQGqCVEop5QW8MENqglRKKeVxF1gVx2M0QSqllPI4naSjlFJKFRLag1RKKeVxXtiB1ASplFLKC+QgQ65fvYINq1e4PhabJshs6FJzSqlLhTcsNZeTSTrtOnajXcduDts+eucNV4WkS81lpkvNFT661JxSBc/dS83ti07M0+trVwgAXBOnTtJRSimlnNAhVqWUUh6nk3SUUkopZ7wwQ2qCVEop5XG6ko5SSinlhK6ko5RSShUS2oNUSinlcV7YgdQEqZRSygt4YYbUBJkNXUlHKXWpKCwr6bibnoPMxrz5r2R55CQ5GmOYNGEc9evWpnpkRYYPHURiYvYrRHyyZDHNrmtERLlQbuzWmQP79zst9/GHH3Drzb3yWp18e3JYV3Z/+gz7V09m/jO34e/nm23ZIbc0Z8fHT3B0y3Osf+MBWjSs7bA/MiKMD54bzL8bnuXXz8fx/KO3ULKE+7+rpbVVvctrUb1KBMOHXLitli5ZTLMmDQkvW5oeXTtlaatff/mFzh3aUqFMCM2bNmHVlytdXAPntF6WwlAvb6lTt+49mLfgFYeH0gRZ4KZNmcSCl+Yyeep0Fr75Ntu3bmXQgLuclt28aSN333kb/frfxrLlKwgMDKJDu1YkJSU5lNv3++9MmTTBHeE79diQzozo14onXviEgU8tosmV1Vk46W6nZUfd1ppJ9/Vk7nsb6DRkNhu3/85n8+7lyjqVAAgMKMm6hQ9QvHgx+oxZwJhpH9GycR1em3CnO6sEWG01f96LTJk2g4WL3mHbtu8YOMB5HJs3beTuO/rTt//tLFu+ksCgINq3bZneVjExMXRs14rIyKosX7GaLl27cUvvnuz8/nt3VgnQekHhqVdRrFNeieTt4dKYdC1WR/lZizUlJYXaNSIZO24CAwcNAWDbtq20adGMvX/+TeXKlR3K39b3FkqWLMkbb70DQHJyMtWrRDD7xZe4tV9/AOrUrMq/Bw8C0L1HTz5a8kme65aXtViLF/dh38pJTHjpc9745BsAGtevysZFD1Gny1gOHY1zKP/3uqnMfGMNc95Zn75t0dQBnEo8w8iJ7zP0lhY8MawLdbs/Q1JyCgC1q5Zn15KnuKLHM/xzODbXMeZlLdaUlBRqV6/C2HETGTjYbqutW2ndoim/7/8nS1v179uHkiVK8ubb7wJWW1WrHM7sufPp268/8+fNZdbM6fy27wDFihUDoGf3LlSpEsnc+S/nOr680noVnnp5e53cvRbrPyeS8/T6yDJ+gK7F6vX27N7N0ehoOnXumr6tUaPGhIWFsXH9uizl169bQ6fOXdKf+/n50bpNO9auXZO+7ZNPv2D7zp/o1v1G1wafjXo1KxJeNphVW/akb9ux5x9i4hNpfd1lDmXLlC5F+bAgtv/8l8P2LTv/oOk1NQCoWyOc3fui0pMjwL6/j3I87hSN61dzWT0y27N7N9HR0XTqkqGtGl+grdaucSjr5+dH67btWLdmtbV/3Ro6dOiU/sEE0KlzV9auXe3CWmSl9bIUhnoVxTrlhzf2IDVBFqDo6CMAhIeHp2/z8fEhIqIiR49GO5RNTEwkISGBiIoVHbZXrORYtl79+tSrX5/Q0FAXRp69CmWDAThyIiF9mzGGw8fiqRAW5FA2/lQSZ/5LoXK4Y6xVI8oQUS4EgOiYk1SuUNphf1ApP8qElEov4w4XaqvobNqqYua2qlgpva2OHDnipC0rcTQ6GneO0mi9Ck+9imKd8kfy+HCdS3IWq4i0BlobY8YV5HFjY2IICAhw+AYHEBgUxInjxx3LxlpDiaUCAx3LBmYt60lhwQGcTjpDaqrjf7BTicmUKV3KYdvZs6ksWb2LJ4d1Yc8fUez7+yidml/B8H4t02eofbJ2F48P6cwjgzox990NhAT588Jjt1K8eDFK+Lrv1zEmL21VyrGtgoKCOG6XjY2NydKWQUFBJCUlkZSUREBAQEFXwSmtV+GpV1Gsk6ut+fJz1n75hdve75JMkK4SGhZGYmIi586dc/ilT4iPp3SmHmBaj/DUyZMO2xMS4j3WW3QmJiGRUv4l8fERhyQZHOhP7MmkLOUfmv4xc5/qz/aPHsfHx4c//jnKa4u30L3VVQDsPRDNnY8u5IXH+zJ+VA9SUs7x8kebOfDvcaKOxbutXmHZtZWTf//0tjrl2Fbx8efLhoaGZWnL+Ph4SpQogb+/vyuq4JTWq/DUqyjWKT9yMlzasUt3Onbp7rDtvbcWuiiiSyxBikgT4C0gAAgQkX7AXGNMgdxxt0IFa6jkcFQUlatUASA1NZXDh6MIj4hwKBsQEEBwcDBRhw45bI86FEV4uGNZT4o+bg2tViwXwr/RcQCICBHlQjjiJKHFnUzijkcX4u/nS3ApP6JPnOShAe3553BMepnPNvzEZxt+omK5EI7HneZcaioDejXjYIYyrpbWVlFRUVTJ2FZRWf/9s22rqEPpZcPDw5205SHCIyIQNy4yqfUqPPUqinXKD2+MsEicgxQRIyI3ZXgeaG9rnbGcMWabMeZy4C6sxHh5QSVHsM4Xli9fnlWrzl97tGP7NuLi4mjdum2W8m3atmf1qi/TnycnJ7Np43ratmtfUCHl254/o4g+kUDHG+qlb2tcvyqlg/zZtP33LOUn3ncjd/W8nqTkFKJPWN9me3VowMerrKnmDa6I5M0pAwgJ9CfqWDz/pZyl0w1XEJeQyLc/Or8G1BXS2mp1huvEtm+z2qpVGydt1a69Q7smJyezacN62rbvYO1v2561a1dz9uzZ9DKrV62kXbsOLqxFVlovS2GoV1GsU3544ySdS6oH6Wq+vr4MGzGSZ556gnLlyhMUFMRDY+6jV+8+VImMJCYmhtiYGGrWqgXAiJGj6Na5Aw0aNuT6ps14fuZzBIeE0LNXbw/X5LyzZ1NZ8OFmJoy+kWMxJzmZmMysR2/hk7U/cPBILKHBAYSGBLD/oHUe5FTiGWY+cgsAv+0/wqCbmxMS6JeeIPcfPEa76y/n1Ql3MPud9YSXCWbWY7cycf4XWc5zupKvry/D7x3F2Kcep1x5q60eHDOaXjf3IdJJW907cjRdO7WnQYNGVlvNmkFwSAg32W3V77bbmTp5AvcOG8LgocNZvWolGzesZ/PXW91WJ61X4apXUaxTfnjjSjpF4jpI+zqaXsaYZfbzQOAk0MYYszFDueuAdzO9fI4xZk6mY3Ftg4Y5fv+Bg4cwaLC1NJ0xhonjn+GD998lMTGRLl27MeuFF/H392fShHFMmTTB4RrLT5Ys5tlpU/j7rwM0bnIdc+bOp1r16lneY+ige4iLi3P7dZBpnh7Rjf5dG+PvV4KVX+3mwWc/JvlMCk8O68pTw7vif+0oAIoV82H8yB7c1r0JxYr5sOX7fTw0fTFHjp+fBdvwikiee6QPV9apxN4D0by2eEv6NZZ5kZfrIOF8W73/3jskJSbSpWt3Zs0+31aTJ44nKeX8/4+lSxYzfepk/rLb6sV5Cxza6pc9e3jg/lH8sGsntetcxjPjJ9KhY6c81yuvtF6Fp17urtPrr77CwtdytkrOzp3fp8XolusgD8f9l6fXR5QuAbgmzksqQebiWHlaKMDb5SdBerO8JkilVPbcvVDA4fg8JsgQ1yXIojTEKtn8rJRSyst544d2kZikY2ue4eeWHotCKaVUrukkHdcaJiKbgcPADE8Ho5RSKue8cZJOUepBvg/MBL4APvdwLEoppQq5otSD/MIYMyTD80c8FolSSqnc8b4OZJFKkEoppQopL8yPmiCVUkp5njeuhlckEqSrr9NRSinlWjpJRymllCokikQP0hVGjhiaZVvXbj3o1r2HB6JRSinX+eLz5az4fLlHY8jJEOuXK5bz5Qr33Q+ySCw1V5B0qbnCR5eaU6rguXupuZjTZy9W1KmwUlY/T5eaU0opVSTpJB2llFLKCZ2ko7J4PYe3nilszh7f4+kQXOL1V4tmexXFehXFOkHRrZc30gTpYQtfe9XTIbjEuRNFM0Hm9F56hU1RrFdRrBMU3XrpYuVKKaWUE943wKoJUimllDfwwgypQ6xKKaWUE9qDVEop5XE6i7WQ+yIXK03kpqyn3/9c/IECLZdbuTluTst6uq1yc1xXxVqYfgcv9Xp5+nfQlTHklDdO0tEEmY2RI4ZmeSyYn/MVW1Z8UfC/bLk5Zm7Kpib8VaDlcis3x81p2dwsm+WqJbZyelxXxeqKenlDrEWxXp7+Hfzi8+U88dgjjBw+NP3hbpLHhyvpEGs25s3POpXa2fqsSilV2HXr3oMVny9n3oLzn3sLX3fzJWjeN8KqPUillFJFi4j4ichiEflGRO652PbsaIJUSinlcZLHP9noBWwBWgADRKT4RbY7j0nv5uEobWV5pZRS7rubR35ljFNEngOWGWO2iMg7wFhjzP7stmd3TO1BKqWUKmpCgCj75ygg9CLbndJJOpm4+tuSUkqp81z0mRsPVAb2A1WAuItsd0p7kEoppYqaHUBDESkGVAL+uch2p/QcpFJKqSJFRPyAd7CS4MtAJLAYq+eYvt0Y8+YFj6MJUimllMpKh1iVUkopJzRBKqWUUk5oglQeJeLq5YaVUipvNEEqtxORoSIyG8AYYwp7khSRW0WkrafjUI4K+++V8jxNkMqtRCQAqAZ0EZGJUHiTpFjCgBeAR0WkuYdDyhURaSMiHT0dh6vYv1dF5lpvEYnM8HNZT8ZyqdAEqdzKGJMIzAEWAT1FZJq9vdAlSWOJAdoB4cBjItLKw2FdlJ3YA4HZQGsPh1Pg7B79rfZU/0dFpIqnY8ovuw7DReQGEXkK6C8i+vntYvoP7KVEpIGI3CEiD4vI9Z6OpyDYH8zFjDFHgBXASmCwiDwGhS9J2vXxNcb8CtwN1AZGe3tP0k7sp4CXgLtFpI6nYyooIuIP7AXqAbsBf2PMQc9GVSCOAZ8Bk4G7jDEvGmNSRSTYw3EVaZogvZCI9ALWAEOBgcDzIjLFs1Hln/3BfE5EbgKeB24AArF6XoVyuNUYkyIiNwJ9gRJAb2CiiDTzbGTZy9Dz2AgcAhrY24t5KqaCYoxJwlpjsydwENgE6ee9a3sytvwwxiQD9wClgcUiUsv+nBgtIiU8GlwRpgnSy4jI5cAMYJIxpiXQH2gCxIlIOY8GVwBEpAGwAFgK3AXUB94FbhGRCVB4kqQdZ2uslTmOAg8CNwFXABNE5AaPBeeEiNQVkarGmFQAY8xvwB/A4/bzc56MryDYyX8k8DVWQrlSRLYC3Ywx+zwaXD6ISGXgP6AL8DrwHDATWGGM+c+TsRVpxhh9eNED6ArstH+ujrVW4OtABPAoUNXTMeazfsOAn4GwDNuqYCXNeOART8eYy/pMB74EfDNsqwscBjYAzT0dox3TlcCPWD3GMUBLe3t9YBfQ034uno61AOpaNsPPq4C3MjwvlPUDhPMrn92HNXxc337u4+n4iupDe5DepyRwVESqY30LXmuMGQScwUqQLT0ZXAHwBQKw/sNjn5M8CMwFigFPiMgkD8aXI/b5RwEux/rgSrG3lzTWOcn7sYaQ/+cNE3eMMT9jDdnPwPqS8pqIfIyVIIOB6+xyhX7tSWPMcQARGYn1xeUu+3mxwlo/YxORMsD1wEBjzG4R8TH2iIAqeJogvYCIFMtw/udPoK399zJgsL09EGuh3UNuDzCP0oZJ7fql/a79jNUz7g0Ow3o+WF8IhgKvuDnUHMk47Jv2gQWsBhqnDacaY87YRfyAPcBp4Hd3x5pRWtzGmK3GmBewhumGYS3gPACrPUZ783nT7FxkJucyY0xbu1wxU4iGkNPq5eRUQwww2BizTZOj6xWZa4QKKxFpD/QDQkRkvDHmJxEZhJUkDgMV7eQ5BiiPhz9sc0pExP7G2xboBVQSkceMMZtEZDww175GbQVwDut8ZHFgjTEm1nORO5ehPtdhnRP+D+s86lKshPOMiEww1p3KA7GGNJcDU401ccRjMveajDF/AX/ZCfE64DusL2IjRCTJGLPL/VHmXsYEISJ1gXLAv8BRY8wpY8yhDOUKTXIEMNYM1QZAbxGZboxJsLcbIDGtjCdjvBTo3Tw8SEQ6Ax8D+7DOMfoD7Y0xO0RkBNZMz2jgJNbwY39jzA8eCjfX7Nmq72LNlozE+gDrbIz5wb60YxLW5JYErKHlm4wxP3om2uxlSI69gFeB41jtsQtrlnE9rIku7YBfAIPVK2tvD216ncy9DxG5FWtYOAqYZoz53mPB5UBam9g/T8KaRRyG9aVyMzDLGPOHB0PMN/v/z1JgGtYXrZOejegS5OmToJfyAxgCPAyEYn2gfoh1h+tG9v4rgO5AByDC0/Hmsm5XYn3YPmw/rw+kAn8D19rbGmH1nm8Bqng65ovUpynWF5WHsL7IPAIkAe8DIVgfzndiTTZ6HKjt6ZhzWC+fDD/fjpX0r/Z0XLmIf4T9e3az/XwR1heY64ESno4vl3XJMoEIa/TlHPAsEOjpGC+1h/YgPUBEGmKdX/wR66ady+3tFbBWmemC1fvY5rEg80lEumBdiN4dq4f8LFaCrAK0whpSjTHGbPdYkLkgIo9jzTBuhzXB6GWsIW8/rIu4HwWijYeHU/MiU28s3FgLOXg1+9ycD9YXlB+NMZNFpCvWTXFHYZ3rDjXGrPZgmLkmIpWAZGPMiQzb+mB9eX4OmGzs4VblejpJxzOOATuwPnCDIH3IKxprCvdnwHcicq3nQsy30sAJrOHTVsBZrKTyONZQ63vAJyJSpjBc84g1SaoM1vBpd6xzXQOAL7B6wFuBd0QksJDUJ50xDtedRns0mBwy57/ZlwG+FpF2wBLgMWPMQqANMNVeWcfriYiPiIRizTGYYM9WBcAYsxi4DWu0aZRY6/8qN9BJOh5gjPnHPsdYHJgjIruMdWkAxphoEXkY67KORE/GmRP2BKJU+0M2DGtafbQx5n0R+QfrA/dFrN7yPqAj1rfhV4E/M35T9gb2xKFzdn38jLWCCVjngbZgXaJyK7DJGHNURGKwJuMsB74y1hJuhU5awsmQeLyesVZlOoG1UEMYMNoY85q9+ywQizWZyusZ63xwrP258BpwWkRmGGOO2UVWYC3qMAkoaU/o00k6LqZDrG5iz0i7Gusb78/GmFUiEoI1SecarAu3f8tQ3qunpYtIj7ShYft5d2Aq1nJrXxtjBtrbr8S6YP4KO6HMBUoZY+7xRNzZEZEGxpidGZ53BB7AmkS0whjzob39RuBNoIYxJk5EXgb+NcZM9EDYl6wME6eqYU0ECzfG1BSRklijMp8C+4wxAzwYZo5luBTHiEhfrKHjZ4EX7JElROR97JEYY8wWjwV7CdEE6Qb2bLRXsXpRl2FN7thhjLnNTpIfYU1i6WiM2eOxQHNIRGpi9QbfN8bcbg8Fb8D6UDqNtTzel8aY/nb5HUAtrBmelwGtjDG7PRK8EyLSDev8zlxjzDwRaQqsB77FOmcajbUayysichVWW0ZhnVO9AWhhCvEyZoWZ3eNvA8zHGpH5F2uGcXGgqTHmbMZzrN4sU5Lsh3UaYhWQNhdhDNDYGPO7XgPpHpogXUysOyWsAOZhzXAsAXTC6m1tM8b0F5HyWNO5ywP1jL0qi7eyL2LuitWT+gR4Ayu5jxMRX6xZt+8Aq40x/USkBta5VbC+/f7qgbCzZbfRE1h343gP68O1uDFmplhrYE4GqgKLjDFviMgzWImxGDDGeOmlHJcS+/zdSKwJVDHAAnsItrgx5qxno8u5TEnyZqwRJoC3gdeMMV9pcnQfTZAuZl/rOBtrVupBe5s/1t0GxgNDjDGb7dlrPqaQ3JrH/o/cFSsRlgReNcbcb+8rgXWu8R2s1UwG2Nu97sMqw1BdDeBJrMttwoEJxpgP7DLVsdqqBjDHGPORvT3AWPe3VF7I209TZCdTkuyBNTIzG/ift/3/Kep0FqvrpWJN7CgN6R/ISVhDeOWw1vLEGHOoMCTHjP95sRbpvh1rVm59u2eJse4usApr5t1dIrLQfrlXfljZbbIfa+HxA0A1oGHafmPMAeBprGHlp0RkgL2r0F3ScSkpjMkRzk+Usn8vl2OdshgFzLBHm5SbaIJ0AREJEetu5mB9qPoDA0TEP8O5kHPAr1jJxeuJvV6s/a22pojUAioYY1ZgDW01whpyBaz7JGLd07IL1mQDr5shmeHcVIjdG9wLjMMaZu0qIvemlTXG/A1MBL7BOt/qdfVRRUemJPkh1g2578c6DaDcRIdYC5h9gfyDWKurzDXGvCUiPbHOMb4IfIC1Ws5QrMsFmhlj/vFQuBcl1rqw+40xG+znN2FNiCiGleCfss+LdMe+ttEYc7en4s2pDEOrae1VGmv49G0RiQSewVrJ6C1jzPwMr/O6YWJVdGUabq2tk8HcSxNkARKRFliTVrZhLR/XGHjIGDPbPpfwMlavPRarB3mH8eK1VUWkCta9KCtiLWb9M9Z96N7FGl5shDWx6LEMSXIR1jWCvT0Tdc45aa9GWEvjzRaRqsBYoA5W0p/luUjVpSzDl7m0v3WSjptogiwgIhKAdc4tAphhjEkWkaexJnc8YH/oVsQ6vyVY12gd9VjAOSQiLbGGUGthJb+aWEkkxU4w92PVOS1J9gZeAK43xkR5KOyLukh7jTHGzLF7ks9h9S5vNcbEeSpepZT76Uo6BUBEymHdv3EX8IGxV18xxky0J648LyLngIXGmG88GGqOpc0AtGfYpmBdg/UEsCvtMpS0KedYEwgmibW6x1IRWW28eEWZHLTXCyKSaoyZKyIPAWc1OSp16dEEWQCMMcdE5BZgJfCLiAQZ+9Y0xpjxdnKcA5wRkdcKyeSOtPvsVcG6KH4L1t0sbhORlsaYzQDGur+jwbpE4hER+dabkyPkvL1EJMUY87JHg1VKeYwOsRYgsW5+/CXW+boPTYY7O4jII8Byb7tI3pkM5zp6Yt2T0mANCy/FuhawJjDSZFjuSkRuAP42xvzriZjzoqi0l1LKNTRBFjB7YYBPgWFk+tAtTESkCdZdRV4A9mItnL4V+yJ6rOnmo00hXxOyqLSXUqrg6RBrATPGfGn3vD4G/ETkTXP+jhCFSUOstVPnZFwtRkQisGbifoN1e6e+xpitHoox34pQeymlCpguFOACxpi0FWbGYS3DVmikXXeFtUh3+lJqYi0KDVbPsTnWNZCbKSQLHVxIYW4vpZTr6BCrC4lIoLdPWMmOiLTFWi7uDnslj7TtfbCuD2wLJNjLyhUJhbm9lFIFT3uQLlTIP2y3AHOxLt+4FdLvmNACa5GD1KKUHKHQt5dSqoBpD1Jly14Y+SlgOOcn6kQAPYwxP3oyNqWUcjVNkOqCxLpD+/VY5x0PYS0jd8CzUSmllOtpglRKKaWc0HOQSimllBOaIJVSSiknNEEqpZRSTmiCVEoppZzQBKmUUko5oQlSKaWUckITpFJKKeWEJkillNcRkTdFxGR6RIvIRyJS1QXvN05EfrB/HmC/X7WCfp8LvP9GEXkhm30DRCQuF8f6S0TG5DMeIyI35ecYRYEmSKWUt9qGdfeY2sDlwL1AU2CZiBRz4fsutd/z0MUKikh3EdHVVooovR+kUspbJRlj/sjwfK+IBAMLgRrAPle8qTEmAUhwxbFV4aI9SKVUYXLS/tsP0ocCO4vIlyKy095WUkSeFZE/ReS0iGwWkeZpB7D3zxCRv0XkqIi8DgRk2N/aPm5p+3moPeR7WERiROQDEakgIq2B5RniGGD/XFFEPrSHhGPsYeHKGY4fISKLReSEiPwhIg8DafdhvSgRuUxEVopIrIgkishOEemeqZi/iCwSkeMickBEpohIiUzHWGHHd1REXk2rrzpPE6RSyuuJpTbwOPA38FuG3fOAFcCd9vNXgfZYQ7KtgJ+BdSJS194/ExgMPA10AwzwQHbvC3yONcTbDxgEXI3Vi90KDLOL1gaW2ov7b8a6JVxPoDdQDlgvIn720PAq4DKgv328m4FmufjnWASE2a/vDOwBPrbfO83TQIy9fzowBphk16ks8B3WHXo6AncDTYBPM9wwXaFDrEop79VSRJLtn4thfV7tBfoYY1IylFtsjJkDYE+suQOoboz52972PdAA6CUiB7Fu3zbcGPNWhv3XZRcD1t1sahpj/rLLnwKeBM4CUQBpQ8Eicrf9ujuNMefsbTuBY3YMQUB9oK4xZq+9vw+QmzvkfAx8aYzZY7/+nF3nCOAvu8wOY0xa0t8hIiHAWBF5HBgJ7MqwHxHZj/WlIyKtTkoTpFLKe+0A7srwPB44YrLeguiHDD/Xxxqu3JupM1QCKwHUxkq269J2GGNSRWQj1s3AM7sSOJiWHO3ya4A1AE46XFdinR897eT9a2P1/A6mJUf7eIdEZC85Nxu4TkRGANdi9RIz25Dp+WpgKhBpx9gqw5cPOD/EWxtNkOk0QSqlvFWiMea3ixcjKcPPxYFUoCHWMGdGp4HS2RwjNZvtvhfY50xxsib2NMeA23L5/g5ExA/4EqiMNdv2C/vvlZmKZv4SkXY67Ywd42dYw9WZaXLMQM9BKqWKkl+xPtcCjTG/2Qn2T6zzkZcD+4EUoF3aC+zzbi2zOd4vQGSmSTbNRGS3iIRlU74GVi8x7f0TgcewJgL9ClQRkToZjlfBji0nWmL1dK82xjxijPkU5xN8Wmd63hE4ARy2Y6wD/J4hxgDgUbJ+qbikaYJUShUZ9tDlp8BbItJNRG4AXsGaiPK7MeY0MB+YISJ3ikgjrEk+1bM55FpgN/ChiLQSka7AHOC4MSYGO6GISEP7EpT3sHppH9qJtAvwAXAV1nWV64EfgcUi0kFEWmCdU0zK/MbZOI31uX2fiDQWkfvt+AFuEJG0UcGGIvKcHdcI4Clgmj08PQ+oBrxmH6MX8D5Q2v73UTZNkEqpouYOrHOEL2HNGK0GdEibtAP8DytpTsIargwAHnF2IHuiTSfgIFYiexv4A2sGKVgzVr8FtgBdjTGnsHp4qViXgLyNNQGnmzEm1RiTinXOcC9WUlqENUy6JId1+wZ4BmtW6ho7tu52bM8BgXa50UBVu8yD9mtm2nU6BDS3/13W2v9O67C+RKgMJOv5bqWUUkppD1IppZRyQhOkUkop5YQmSKWUUsoJTZBKKaWUE5oglVJKKSc0QSqllFJOaIJUSimlnNAEqZRSSjnxf5WKh9zzyeGZAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "conf_matrix = evaluate_mlpf(model_native, with_VICReg=False)\n", + "plot_conf_matrix(conf_matrix, \"native MLPF\")" + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Num of ssl mlpf model paramaters: 202652\n" + ] + } + ], + "source": [ + "print(\n", + " \"Num of ssl mlpf model paramaters: \",\n", + " sum(p.numel() for p in model_ssl.parameters() if p.requires_grad),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Num of native mlpf model paramaters: 198920\n" + ] + } + ], + "source": [ + "print(\n", + " \"Num of native mlpf model paramaters: \",\n", + " sum(p.numel() for p in model_native.parameters() if p.requires_grad),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test from script" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "test_loader = torch_geometric.loader.DataLoader(data[5000:], batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "# downstream model\n", + "NUM_CLASSES = 6\n", + "\n", + "\n", + "class MLPF(nn.Module):\n", + " def __init__(\n", + " self,\n", + " input_dim=34,\n", + " width=126,\n", + " num_convs=2,\n", + " k=8,\n", + " ):\n", + " super(MLPF, self).__init__()\n", + "\n", + " self.act = nn.ELU\n", + "\n", + " # GNN that uses the embeddings learnt by VICReg as the input features\n", + " self.conv = nn.ModuleList()\n", + " for i in range(num_convs):\n", + " self.conv.append(\n", + " GravNetConv(\n", + " input_dim,\n", + " input_dim,\n", + " space_dimensions=4,\n", + " propagate_dimensions=22,\n", + " k=k,\n", + " )\n", + " )\n", + "\n", + " # DNN that acts on the node level to predict the PID\n", + " self.nn = nn.Sequential(\n", + " nn.Linear(input_dim, width),\n", + " self.act(),\n", + " nn.Linear(width, width),\n", + " self.act(),\n", + " nn.Linear(width, NUM_CLASSES),\n", + " )\n", + "\n", + " def forward(self, batch):\n", + "\n", + " # unfold the Batch object\n", + " input_ = batch.x.float()\n", + " batch = batch.batch\n", + "\n", + " # perform a series of graph convolutions\n", + " for num, conv in enumerate(self.conv):\n", + " embedding = conv(input_, batch)\n", + "\n", + " # predict the PIDs\n", + " preds_id = self.nn(embedding)\n", + "\n", + " return preds_id\n", + "\n", + "\n", + "class ENCODER(nn.Module):\n", + " def __init__(\n", + " self,\n", + " width=126,\n", + " embedding_dim=34,\n", + " num_convs=2,\n", + " space_dim=4,\n", + " propagate_dim=22,\n", + " k=8,\n", + " ):\n", + " super(ENCODER, self).__init__()\n", + "\n", + " self.act = nn.ELU\n", + "\n", + " ### 1. different embedding of tracks/clusters\n", + " self.nn1 = nn.Sequential(\n", + " nn.Linear(TRACKS_X, width),\n", + " self.act(),\n", + " nn.Linear(width, width),\n", + " self.act(),\n", + " nn.Linear(width, embedding_dim),\n", + " )\n", + " self.nn2 = nn.Sequential(\n", + " nn.Linear(CLUSTERS_X, width),\n", + " self.act(),\n", + " nn.Linear(width, width),\n", + " self.act(),\n", + " nn.Linear(width, embedding_dim),\n", + " )\n", + "\n", + " ### 2. same GNN for tracks/clusters\n", + " self.conv = nn.ModuleList()\n", + " for i in range(num_convs):\n", + " self.conv.append(\n", + " GravNetConv(\n", + " embedding_dim,\n", + " embedding_dim,\n", + " space_dimensions=space_dim,\n", + " propagate_dimensions=propagate_dim,\n", + " k=k,\n", + " )\n", + " )\n", + "\n", + " def forward(self, tracks, clusters):\n", + "\n", + " embedding_tracks = self.nn1(tracks.x.float())\n", + " embedding_clusters = self.nn2(clusters.x.float())\n", + "\n", + " # perform a series of graph convolutions\n", + " for num, conv in enumerate(self.conv):\n", + " embedding_tracks = conv(embedding_tracks, tracks.batch)\n", + " embedding_clusters = conv(embedding_clusters, clusters.batch)\n", + "\n", + " return embedding_tracks, embedding_clusters\n", + "\n", + "\n", + "# define the decoder that expands the latent representations of tracks and clusters\n", + "class DECODER(nn.Module):\n", + " def __init__(\n", + " self,\n", + " input_dim=34,\n", + " width=126,\n", + " output_dim=200,\n", + " ):\n", + " super(DECODER, self).__init__()\n", + "\n", + " self.act = nn.ELU\n", + "\n", + " ############################ DECODER\n", + " self.expander = nn.Sequential(\n", + " nn.Linear(input_dim, width),\n", + " self.act(),\n", + " nn.Linear(width, width),\n", + " self.act(),\n", + " nn.Linear(width, output_dim),\n", + " )\n", + "\n", + " def forward(self, out_tracks, out_clusters):\n", + "\n", + " return self.expander(out_tracks), self.expander(out_clusters)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "encoder_state_dict = torch.load(\n", + " f\"ssl/encoder_best_epoch_weights.pth\", map_location=\"cpu\"\n", + ")\n", + "with open(f\"ssl/encoder_model_kwargs.pkl\", \"rb\") as f:\n", + " encoder_model_kwargs = pkl.load(f)\n", + "encoder = ENCODER(**encoder_model_kwargs)\n", + "encoder.load_state_dict(encoder_state_dict)\n", + "\n", + "decoder_state_dict = torch.load(\n", + " f\"ssl/decoder_best_epoch_weights.pth\", map_location=\"cpu\"\n", + ")\n", + "with open(f\"ssl/decoder_model_kwargs.pkl\", \"rb\") as f:\n", + " decoder_model_kwargs = pkl.load(f)\n", + "decoder = DECODER(**decoder_model_kwargs)\n", + "decoder.load_state_dict(decoder_state_dict)\n", + "\n", + "mlpf_state_dict = torch.load(\n", + " f\"ssl/mlpf_best_epoch_weights.pth\", map_location=\"cpu\"\n", + ")\n", + "with open(f\"ssl/mlpf_model_kwargs.pkl\", \"rb\") as f:\n", + " mlpf_model_kwargs = pkl.load(f)\n", + "mlpf = MLPF(**mlpf_model_kwargs)\n", + "mlpf.load_state_dict(mlpf_state_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4c86dd9535da42e586eeaee82efc1e65", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Time taken is 93.74s\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcgAAAGoCAYAAADCVXWwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAAsTAAALEwEAmpwYAACKGklEQVR4nOzdeVwVVRvA8d8BQUA2URFQUFPL0hb33HFX1FKz3FrMTM2l/W01M3ctNVOzsszKdjXLXXFNK5e0Rc0y90RAZVMBRTjvH3NBLlz0gncDn2+f+cQ9c2bu8zhwz50zZ84orTVCCCGEMOfm7ACEEEIIVyQNpBBCCGGBNJBCCCGEBdJACiGEEBZIAymEEEJYIA2kEEIIYYE0kEIUklIqUimllVJHC7HNGNM2Wil1WCmlrNxuQ67tFuRZt8BUvqmQ8eeOxdJyVim1VSk1WSnlX8A+jl5jH3mXAYWJUQhXIA2kEI5XDbjrWpWUUuWBVnaPJr8goBnwInBAKVXXCTEI4XTSQArhHL2sqHMP9v0bPaa1VrkXwB2oDjwLnAdCgXlKKfcC9vFG3n0UsCywYx5C2IU0kEI41mHT/++zopu1Z55t7E5rnaW1Pqy1ngEMNBXXBzo4KgYhXIU0kEI41m/AEeAWoHZBlUzX/toDqcAah0SW3yLgjOnnAmMVoqSSBlKUOEqp0kqp4UqpbUqpWKXUOaXU70qpN5VS5QrYpo5SaqFS6i+lVJpS6oRSaq1SqpO1A2qspIHFpp+v1s3aBfAEVmE0kg6njYmaj5pe3uSMGIRwJmkgRYmilPIGtgKzgaZARcAXuAN4HvhNKRWUZ5sHgD+A/kAtwAuojHEGtwp4zsZhLjL9/76r1MnuXl18lTp2ZfpiUNX08riz4hDCWaSBFCXNq0ADIBHoA1QA/IHOQCxGwzcqu7JSKgD4GFDA10AdwAdjpOkHpmoTlVIVbBjjTuA/oI5S6pa8K02NfBRwCVhhw/ctrPuA8qaf/3BiHEI4hTSQoqSJNP1/tNb6a631Ga31Oa31amCCaV2jXPUbYjSIp4H+Wut9Wus0rfVRYJip3AO401YBaq2zgCWml5a6WTuYYlqntU6x1ftaQynlppSqppR6GuOLA8AeCr4O+roV90BWdUTsQthaKWcHIISNlTH939vCug8xujczLdQHo2v1QvYLrXWmUuo2jL+TZBvHuQh4EuMsbUKedY7qXq2ilLrWA2HjgEFa68xr1BOixJEzSFHSbDD9f4JS6mOlVNPse/i01ula61it9elc9XdgNIoVgJ2mwT05A3lMZ6CxWus0G8f5E0bjU1cpVT27UCnliXH/Yybwg43f01qZwF7gPeB2rfXuq9S15j7Iow6JWggbkwZSlDSvA8sxukUHANuAs0qplUqpwXlHsWqtTwEPYVyzvBVjcM8ZpdRepdRMpVQzG49izX7fTK50s+YerBMJBAKbtNZnbf2+eeSbKMC0lNJa3661fiLPlwkhbijSQIoSRWt9HuMMrDXwLsboywCMQTrvA4eUUr3zbPMdcDMwEuMM9BLGfX9PYoyIXaGUCrRDuNldqLkbSKePXhVCGKSBFCWONmzSWg/HuE2hGjAC2I/RWH6ilKqSZ5szWuvZWuu2GGdwbYAFwEWMxnWiHULdDJwFGimlIkxdwd0x7pVcaof3E0IUgjSQosRQSpVRSj1vWrwgp7E8qrWeA9TFOKMsjTEZN0qp9qb6bbL3YxrFulFr/SjwlKm4na3j1Vpf5kpD2BNognHf5k+mrl8hhBPJKFZRkqQD4zEawP3AyjzrL+f6OXtUakOMUaQ7lFJ3m2aPyS0jT31bWww8hnG7R5VcZUIIJ5MGUpQYptsydgLNgfeUUoMxRotmADWBl4EIjKnbfjZttt30/0bATKXUDOAkRldsB+BN0/poO4W9HqPxbYZxHRSuDN6xlpcV9xqmaq3jC7lfIW5o0kCKkmYYxkw14RjTxOWVBTyotU4A0FqvV0p9jjHN3EjTktfPwBv2CFZrfUkp9QPGSNoKwK9a62OF3E1jjAnQr+Z7jOubQggryTVIUaJorf/EuF1jHrAPSDEtf2JMHVfLNGo1t0cwGqgfMaaAu4RxrXKTaV0rrXW6HcNelOtn6V4VwkWo/JdchBBCCCFnkEIIIYQF0kAKIYQQFkgDKYQQQlggo1jzsOLpBkIIccPQWtt8LuLcbPWZa4845QxSCCGEsEDOIAvgVe/JfGUZx9bjUaWtVdtbW/figa8oXauPTfdZUN0/V022WHfUc8MZP23ONfdpbT2AHh2a8d3abVbVLcx+LdWtHJT/0Y/Dhw5mznsfWLXPwtRt1rgB27bvsul+LdVLu2T58YtPjxjK27Pfs+r9C1O3TfPGbNi6/Zr1rvf9vT3dLda1x/Gyx7EqqG5BdwMMf2Iwc+Za8TtgZT2AZnc3ZNsvO62qW5j95q3r4+nY8yevu4YXabv036z77CgKaSCFEEI4n7p2g5yZfISs5GvNiWE7BTaQSqk9GE8VuCatdT2bRSSEEOLGY8VjV90Db8I98Cazssyz++wV0VXPIJfa7V2FEEKI3Kw4g3S0AhtIrbVd5p4UQggh8rHiDNLRrL4GqZRqDfQGagCDMCZI/ldr/audYnOqjGPr8xeW8rF6e7eAajaMpvD7LEzd1h2ibFqvsAqzX2vrRnXtZvU+C1O3MKzdb2Hev1NUF7vUdfb7O/t42ev9o7pY+TtgZb3Csna/K5Yv4/jx4wx/YrBd4iiurJqLVSl1D8Ykyt8C9wO1gQeBF4EuWmt7PQrI4bLvybE0itUeCjOK9XoVNIrVHgozivV6WRrFai+FGRl5PQoaxWov1o5ivV4FjWK1B0cdKyh4FKs9FGYU6/XIHsXqqPsgvRo+V6Tt03dOA5x7H+RoYKzWuh+mh85qrUcDMzEeUCuEEEIUnVJFW+zI2gbyVuAHC+XfAnVsF44QQogbknIr2mJH1l6DPAZUAX7PU14ViLFlQEIIIW5AVpwNZib+S1bivw4IxmBtA/kxME0plWB6HaaUuhN4G+PBtEIIIYRduZetgXvZGmZlmfF5z9tsx9oGchrgB6wFSgMbMK5FzkGuQV4X9/Ils4e694MDnR2CXQwcVDJH+T08cJCzQ7C5knqsBg563Nkh2IcL3gdp1SjWnMpKlca4zaMUxi0eF+wVmLM4ehSrIzlyFKsjOXIUq6M4ehSrozhyFKsjOXIUq6M4fBRr01eKtH36TxMB+8RZmPsgKwADgVrAReAPpdQnJbGRFEII4WAueAZpVURKqXrAYeB5IAioBIwFDiqlbrratkIIIYQjKaW8lFKLlFI/KaUezVVeVim1USn1u1LqmteBrG2yZwDbgKpa63u11t2AasA+YHZREhBCCCFy2PY+yB7AVqAFMEApld1b2h/4BKgLXPMitbUN5F0YEwXkdKdqrc8BbwBNrNxHsZJxbH2+JTPpsFXbvjq4E3uXjuLwmrHMfa0v3l4eBdYd3rcVfyx5lfgtU1jz/gjq1qpcYN354x7im2mPFTqX66W15p03x9O2cR2a3l6Nl58eSlpq6jW3e+1/I/n4/fzfn/75ax8DHuhG/ZvDaH5XDV7730iSkxLtEfpVaa0ZP3YMtWvVoFp4KEMff4zUq+S1ZPEimjaqT0j5QLpFdeTIYfPfh7/276dT+zZULBdA8yaNWLN6lZ0zsExrzeQJb1D/9lu49abKjHzi8avm9f13i2ndrBFVw8px3z2dOXqk4N/zIQMf5sHePe0R9jWVxOOVnVOdW2tSLSKMoYOvntN3ixfRtHEDQiuU5Z4unSzm1LlDW0LKB9KiaWOrc1qxfBnDnxhstjicbe+DrA/s0lpnAieACFN5KkYvaBmseFqVtQ3kYSDAQrk/8J+V+yhWPKq0zbfkfcyKJS8N6sATvVvyyswfGDhqIY1ur8L8sQ9ZrPtojyaMGdaFSR+uocPgWRw4Gsfyd4cRHOSXr27vTvXpG9XguvMqijkzJvPZR+/x4usTeGvOfH7bvZPnRxTcUGut+XFTNEu//SLfutQLFxjY5x68vb356MulTJz+Lrt3/MKTjz/k8IEOkyeOZ+6cWUyc/CbzP1nIjh2/MHCA5WO1ZfMmHnmwL7379mfpslX4+vnRrk1L0tLSAEhISKBD21ZERFRh2cq1dI7qwv0972X3r46fqvitKROY994c3pgwhfc++oRdO7czdNAjFutu3bKJxwf0p1fvvnzz3XJ8/fzo0qF1Tl65Lfr6S779Ov8xdZSSeLwmTxzPe+/OZsKkqcxf8Bk7t2/nsQEPW6y7ZfMmHnmoH3369mPpspX4+vrRvm0rs5w6toskPKIKP6xcQ6fOUTxwX3d27752Tl26dmPO3A/MFofL1ehdjt3NxT3zrFoKEMCVe/RjgLKmn5cArwBHgeXXDMnKuVg7YXSzDsU4bdVAc+BDYJTW+ptr7qSYuJ5RrKVKuXFwxRuMfW8lH3/3MwAN61Rh08dPc3OXMZyMTzarv27eSPb89R8vTP8OADc3xZE14xj1zjI+W3ZlXsyI0CB+/vx54s6e49/j8Tzw3EdFyq0oo1gzMjJoWbcmT784mt4PGV32v/26gwe6tGbz7r8JDTM/4933xx4eui+KcylGrq+MncqjQ0bkrF+9fCnPDXuUn/88gn9AIAB/7fuDe9rczeqte6he85ZCx1iUUawZGRnUrBbO6DHjcobN79i+ncgWTfjn8HEqVzbPq2/vXpT2LM2Czz4HID09naqVQ5g5ey69+/Rl7pzZTJ82lQMHj+DubozUvLdrZ8LDI5g99/1Cx1fUUawZGRncfnNVXh79Bo88aty6sWvndjq2bs4ffx+hUiXzvB7p/wClPUvzwcef5eRV66ZKTHt7Dvc9cGWO4OPHjhLZrCHBFUOoUaMmC79eUqT4ijqK1dWPV1G+3GVkZFDzpghGjxnLwMdMOe3YTusWTfn70LF8OfXrfT+lS5fm408X5uRULTyUmbPe5YE+fZn77mxmTHuTv/45fCWnblGEh4cz+93C5+TwUayRY4u0ffqm0YB5nEqpt4AftNZblFJfYrRTh5RSc4EvgR3AMqCv1vpMQfsu8AxSKZWolEowTQ7wBcbtHRuAdNOyEWOwzktFyqoEql09lJDy/qzZuj+nbNe+4ySkpBLZ6OZ89b1Le3IuNT3ndVaWJjXtEj65umTd3BTzxz3Ih4t/Yte+Y/ZNwIJ/DuzjzOl4WrXrmFN2R90GBJYN4ucfN+WrX636zXz1QzQrNu0kuGJIvvXnz6XQ8O7mOY0jQFC58gCcinFcZ8S+vXuJi4ujY+crTwdp0LAhQUFBbNqQ/0kuG6LXmdX18vIisk1b1q9ba6xfv4727TvmfDABdOwURXT0Wjtmkd9f+/YSHx9H+46dc8rq1W9I2aAgtmzckK/+pg3RZnW9vLxoGdmGDevX5ZRlZmYydNAjPPrYYOrXb2jfBApQEo/Xvr17iY+Lo2OnXDk1uEpO69fRsZP5sYps3ZboaONYbYiOpl37DmY5derUmfXR6/Lt6wawC6ivlHLHaKeOm8qDgHiMOzEuYPSCFuhqXaxPA8/kWgZh3OYxCHjc9PMwjAnLBVCxnPFvHXs2JadMa82p0ylUtNBtunTD7zzavQl33FwJj1LuPNG7BeUCyxD9y985df73aHu8vTwZ/75zrmediY8DoELwlcbOzc2N4IqhnDkdn6++T5ky3HxrbW6+tTYenp751vfq+zALvlmW81przWcfvoeHpye33na7HTKwLC4uFoCQEPO8QkPDiDPlnC01NZWUlBTCwsLMysPCKhFvqhsbG0to3vWVKhEfF+fQruPsvCpWNM8rJCSM0xbyOpeSQkhoqFl5WFgls7oz3ppMWmoaL776uh0jv7qSeLyullN8ATnlj/lK3bi4WEJDzdeHhjn+d7DIbHsNcinQDKPHcz7wslLqNmAC8BmwB9intb7qwJKrPTD5k2vmo1Qo8Oi16t0oggJ8uJB2kaws81/G8xfSKRfom6/+WwuiiWpRm+1fvpBTNuSNLzh04jRgdM8+83AbWj4ynYzLzrlxPCkxEW9vH7NvpQBlfH1JTDh7XftOTDjLW+Nf45vPF/Dsy2MoVyH4uvZXGAkJCfj45M/L18+Ps2fMe1wSE40BRGXKmB9DPz8/zpjqJiYmUMY3//q0tDTS0tLw8bH+WaLXIykxsYC8fDl71jyvJNPAqLxx+/r6kmCqu2vndma9PY11m37C08IXHkcpiccrsSg55TtWV+omJibg6wK/g0VmwydzaK3TgV4FrLa6G8SqiQKUUt7Ac0DeUSo1MW73mGjtG5ZkCcmplPEujZubMmsk/X29STyXf2TavDf641vGi17PzCMmPonm9Wow6el7iT2TwpZf/+Xj8Q8x6p1l/HM0/5maowSWLUtaWiqZmZlmf8jnUlIIyNVNWhhaa35Y/DXjXn2OzMxMJkybw/39B9gmYCsFBQWRmpo/r5SUZMqWLWtWN/v1+fPnzMqTk6/ULVs2iPPn8q/39PTE29txM/0Eli1rOa/kFAIDzfPKfp037pSUFAICy5KWlsaQgQ8zZtwkbr6llv2Dv4qSeLzKFpRTcjKBBeWU71iZ53Qu73on/A4WmQtOFGDtTDqzge7Ar0BbYAUQDNQDOha82Y0lztS1GlYhgP/ikgBQShFawZ/YMylmdcMqBPBg10Y07juVP/45CcCeA/9RqWIgw/u24sCROKqHV+Ct53vy5nM9APAo5Y5SkPjTW3QdPpdtew7ZPafywRUBiI89RahpgEdWVhbxcaeoYOEaozWmjH2Vj959m249e/Pq2CkOPXPMlt0FGRMTQ3h4OGDkdSomhpAQ8y5HHx8f/P39iTl50qw8JuZkTt2QkJD860+eJCQ0FGXnZ9bllp3XqVMxVK58Ja/Y2BgqWsjLz9+fUzHmD+Q5ZcrrdHwcRw4f4uX/PcMrLzwLGANLtNaEBpVh8Q+raNq8pQOyKpnHK+dYxcRQOXdOp2LydXsXmNPJK/lXrBhCTIyFnB38O1hkLhijtU12FDBSa90BY8LysVrru4GvAOfce+CC9h06RdzZc3RoemtOWcPaEQT6ebN550Gzut5eRndV3u5YrTW+PqWJOZ3EnfdNoFHfKTTuN5XG/aayYsteNu/6l8b9prL7r+M4ws21alOufDCb16/JKft99y5SkpNo0jyy0Pv7+cdNfPTu27wwegLT3p3vlMYRoHadOgQHB7M2131iO3fsICkpiVat2+Sr37ptO9asuVI3PT2dzRs30KZde2N9m3ZER6/l8uXLOXXWrllF27bt7ZhFfrfWrkOFCsFEr12dU/brrh0kJyXRMrJ1vvqtWrc1q5uens6WzRuJbNOO0LBK/LJ7L1t+2c3mn39l88+/0rlLN1q0jGTzz79yVz3H/emXxOOVnVPuOHftNHKKjLSQU5t2rF1jfqw2b9pAm7btAGjTti3ro9eZ5bRmzeqc9SVB5un9ZPy1xGyxJ2vPIMsCB0w//4LpJkyM4bJvAW9eTxCmYb5tgEeALsAl4C2t9QzTejdgBDAE47mUB4HJWuuvTesjgdXAbcAHQGOMZ1g+rbWONtVxB57FGFxUBfgLo6H//npiz+3y5Sze+2YLY0d043Tiec5duMj0F+7ju/W/cyI2kbL+PpT19+Hwf2c4dOI02/YcYt4b/Rk9exmxZ1JoVrc6Qx9owdNTFnH5cla+rtXk82kohUO7XD08PHjwsSFMm/g65cpXoIyvH2NffY5O3XoQVjmcpMQEkpMSqVKtulX7W7H0WyqGhtGuU1eOHzW/Pl4xJAwvB3UFeXh4MHTYCEaPepkKwcH4+fnx7NMj6XFfLyIiIkhISCAxIYHqNYxH6wwbPpKoju2oV68Bdzdpyozpb+IfEED3HsZN83369WfShLEMG/I4gwYPZe2aVWzauIEt27ZfLQy75DVo6DDGvf4qFSoE4+vry4vPP809Pe6jcngEiQkJJCYmcFN1I6/BQ4fTo2tH7qpXn0Z3N2HW29Pw9w+g67098PDwyNe16u8fgNba4V2uJfF4eXh4MOSJ4bw+6hUqVDByeu7pJ+nRsxfhFnJ6YvgIunRqT7369Y2cpr2Ff0AA95py6t23P5MmjGPY0MEMGjyEtatXsXnjBjZv/cVhOV0XK7pY3YPr4B5s/gSkzJM77BWR1Q3kf0AksBv4A3gIeB9jVoIqNoplJsZ9le8DTwHTlVLrtNZ7gZEY1zmfw7h/pQPwuVIqTWv9g2n7UhjPrZwBTAUmA18opSpqYwjX6xjTDD2PcZPoPcASpVQHrXW+MdUXD3xldeDu5etQyvTYqskfrsWjlDtTnumOt5cnq7bu49mpiwEY1qclo4Z0xrv+UwA88NyHvDY0ijmj+lAusAwHj8Xz5KRvze6BdAXDn3mJyxkZTHz9JdLTUmndvjOjJ04H4NMP32XWWxM5GHftmXUAYk+dJO5UDB2a3plv3cIlq2nczDFddgAvvTKKjIwMXvzfs6SlptI5qivTZ84C4N3Z7zBh3BukZRhn+C1bRfLp518xddIEJk0YS8NGjVm3fnPOtZ2goCDWRG/imadG0C2qAzVvvoXFS5dxV926Dssn2/MvvkpGRgajXnqetNRUOnTuwpRpxmDz9+fOYurEcSRcMM4ymreM5MNPvmD61Em8OXk89Rs0YvmaDS55zaokHq/snF564TlSU1PpHNWF6W9fyWni+LGkXsq6ktPCL5kyeSKTJoyjYaPGrI3eZJbT6nUbefbpkdwT1ZGaN9/Cou9+MMvpow8/YP6HLvoIXxfsYrV2ooD/AVMwzuBWAnuBuRhdryla6+v6VDOdQc7WWo80vQ4G4oAeWuulSqmTwPta67G5tvkIuFlr3cJ0BrkR6KW1Xmxa/wDwNcbZbzqQCHTUWm/JtY8vgXNa68G5yuRxV8WMPO6q+JDHXRUfDp8ooOO0Im2fvuY5wImPu9Jav6mU+gNI1FqfNM2C/jzGDZe2akl+zvV+8dkXlZVSAUAYsCVP/c3AvQXtwxRbthqAF7Au+2CYeFjYrxBCCEdzwTNIq58HqbVek+vn74DvbByLdX10V2SRP/6C9pFdLwo4mWfdpUK+rxBCiBtAgQ2kUsrqM0Ot9Tu2CcfivpOVUqcwHluyKdeqVsB+ixvl9y9wGSiffb1RGaeob2AM1rHuMR1CCCHso5jdB/mMlfvQgN0aSJM3gXFKqThgJ8YgnUeB+63ZWGt9Xik1B3jb1MV6FGOWhWeApnaJWAghhPWKUxer1rqaIwO5hpkYDfEzGM/1+hd4yNTVa60XgBSM0bBhGAONumqtd9o4ViGEEIVlxRlkZtyfZMXtdUAwBquvQdqTpdFHucu01lnA26bF0vabAHW1Mq31JWC0aRFCCOFKrLkPMuRO3EPMbxHLPPGTvSKyeiYdIYQQ4obiEmeQQgghbnDF6RqkEEII4TDFbBRrPkqpEIxHXO0BMrXWaXaJygVkHMv/RG+3gGq4B+Z94pcQQhRvK5YvY+WKZdeuaE/F9QzSNJvNUox7D7MwJgV/SSnlDwzQWp+3W4RO4lGlrbNDEEIIh+jStRtdunYzK/v4ow8dG4QLnkFaG9GbQCBwK8YN92A8I7IOMMn2YQkhhBDOZW0X673Aw1rrv7PnMtVa71ZKvYzRUI60V4BCCCFuAFZ0sWae+o2s2D8cEIzB2gayFOaTf2c7AfjYLhwhhBA3ImVFA1kqrC6EmT+SLO2o/Z43YW0X6y8Yz1LMlv1EjCeAX20akRBCiBuOUqpIiz1Zewb5LPCzUuoujEdETVZK3QxUBZrbJzQhhBDCeaw6g9Ra/w3cDPyI8ZgrN+AH4Bat9e/2C08IIcQNQRVxsaPCPA/yDMbjoYQQQgibsnd3aVFYex/k9Kut11o/a5twhBBC3IiKbQMJ1M3z2geoBaQCq2wakRBCiBuONQ3k5ZN7uByzxwHRGKxqILXWrfOWKaX8gI+AbbYOyhXIVHNCiBuFS0w1Z4VSlepSqpL5+VrG4c12ez+ltb52rYI2VqoWsEhrXcd2ITlX9kQIXvWedHYoNvfnqsnODsEuKgd5OzsEm0u7lOnsEOzC29Pd2SHYxfV8jroqH09jDKel5/XaUvZnrn+fT4u0fcpXDwP2ifN6n+ZRE6hii0CEEELcwFzvEuR1DdIJAroDW20ZkBBCiBtPSRqkk20RMNpGsQghhLhB2bKBVEp5AQuBMGCe1vpjU/lTQA9TNX9gnNb6uwL3Y03fuVIqCEjSWmddb+CuLrs/fO9/55wdis016PqSs0Owi8Sds50dgrjByTXIosv+zA3sv7BI2yd9/iBgHqdSqi9QEZgFbADaaq0v595OKfUJ8JTWOqmgfVs7F+shoGGhohZCCCGsZOO5WOsDu7TWmRgP1YjI815NgH1XaxzB+i7Wz4A+wHYr6wshhBBWy93Ypf+znosHN1zP7gKAGNPPMUDZPOtHAI9fayfWNpD/AU8rpe4GfgPScq+UmXSEEEJcl1wng163tMXrlrZWbZbwaT9LxclAZeAwEA4k5byNUmHAJa116rX2bW0D2Rn42/RzrTzrSl7nuxBCCIey8SjWXUB9pdQ2oBJwPNe6TsBGa3ZS5Jl0hBBCCBe1FGMU6wPA+8DLSqlFWuv9QBTwjDU7KbCBVEplAjW11oevP9biZ8wLI/OVRbbvTGT7KCdEI4QQ9uMKU83Z8gxSa50O9CpgncVyizEVNDxZKZUF1LjRGki5zaP4kds8hLPJbR5Fl/2ZW+HRr4u0/emPewOuOdWcEEIIcf1cbyKdazaQPZVS8dfaida6aLPMCiGEEBTPqeamWrEPDUgDKYQQwq4uHt/FxeO/Ouz9rtVA3nDXIIUQQjieNWeQXlUa4lXFfFK39L/zP7vXVuQapBBCCKcrjl2sQgghhN0VtwbyE6Dk3esghBDC9bhe+1hwA6m1ftSRgQghhBCuRLpYCyAz6QghbhQlbSYdW7H2eZA3nDFTZ+VbrtU4aq2ZM20inZvdSWS9Grz23DDS0q45YTxvvPQUn304x1ah28WrQ6LY+/3rHF47gbmv98Pby6PAusP7RvLH0tHEb32LNfOeou6t4Q6M1Hpaa8aPHUPtWjWoFh7K0McfIzW14OO1ZPEimjaqT0j5QLpFdeTIYfMB3n/t30+n9m2oWC6A5k0asWb1KjtnYJnkZSgOeWXnVOfWmlSLCGPo4Kvn9N3iRTRt3IDQCmW5p0snizl17tCWkPKBtGja2OqcunTtxpy5H5gtjmbNsx/Tj+0iafNcs8WepIG0ofdnTuWLj9/juVHjmPzOh/yxZycvP1nwI8e01mzbvJ5li760uH7gA12oV7282TLwgS72Cr9ALz3eiSf6tOKVt79j4KhPaHR7NeaPf8Ri3Ud7NGXMiG5MmreKDoPe5sCRWJbPHUFwkF9OnX0/vE7iLzPMlvfHPOiodHJMnjieuXNmMXHym8z/ZCE7dvzCwAEPWay7ZfMmHnmwL7379mfpslX4+vnRrk1L0tKMJ78lJCTQoW0rIiKqsGzlWjpHdeH+nvey+1fH3bOVTfIqPnlNnjie996dzYRJU5m/4DN2bt/OYwMetlh3y+ZNPPJQP/r07cfSZSvx9fWjfdtWZjl1bBdJeEQVfli5hk6do3jgvu7s3u34Y1UU1jSQPtUaEdR6uNli15hK4hyC16Ooc7FmZGTQrlEtRj4/il79jcu3f+zeSf9727Ju+1+EhFUyq7//z994rHdXzqUkA/DimMk8NMj8YDe/vQojnh9Fo2Ytc8q8vLwJq2z2cGyrFWUu1lKl3Di4ajxj313Ox9/9BEDDOlXY9Mlz3Nx5NCfjk8zqr/voafbsP84L05YA4OamOLJuIqNmfs9nP/yCr09pTm+bxv3PvM8/R69M0nTuQjqnTicXKa+izMWakZFBzWrhjB4zjoGDjC8xO7ZvJ7JFE/45fJzKlSub1e/buxelPUuz4LPPAUhPT6dq5RBmzp5L7z59mTtnNtOnTeXAwSO4u7sDcG/XzoSHRzB77vtFyqsoJC/n5FWUz9GMjAxq3hTB6DFjGfiYKacd22ndoil/HzqWL6d+ve+ndOnSfPzpwpycqoWHMnPWuzzQpy9z353NjGlv8tc/h6/k1C2K8PBwZr9b+JwcPRdr5WFLi7T9f+92B+wTp5xB2si/f+/n7Ol4WrTpmFNW5676BASWZfu2TfnqV61ek0+XrOW76O1UqBiSb31iwhmSEhO4u0UkN9W4JWcpauNYVLWrhxFS3p81W/fllO3ad5yE5FQiG9+Sr753aQ/OpV7MeZ2VpUlNu4iPlycAN1etSFZWFtE/H+Cfo3E5S1Ebx6Lat3cvcXFxdOx8pdu8QcOGBAUFsWlD/huPN0SvM6vr5eVFZJu2rF+31li/fh3t23fM+WAC6NgpiujotXbMIj/Jy1Ac8tq3dy/xcXF07JQrpwZXyWn9Ojp26pzz2svLi8jWbYmOXmesj46mXfsOZjl16tSZ9ab1Lk8VcbEjaSBt5Ex8HADlgyvmlLm5uVGhYihnT+efztbHpww1a91GzVq34eHhmW/9kX//AeC9t6fS8q6b6NTsDiaOep6U5CT7JFCAiuX9AYg9m5JTprXm1OlkKubqNs22dP1vPNqjKXfcXAmPUu480acV5QJ9if7lL8BoIJPPp7NgwiOc2DCZ35aM4rUnulDa07HjxeLiYgEICbny5cTNzY3Q0DDiTMcyW2pqKikpKYSFhZmVh4VVIt5UNzY2ltC86ytVIj4uzqFPepC8ik9eV8spvoCc8sd8pW5cXCyhoebrQ8Mcf6yKypouVkuLPbnEKFal1Bigu9b6rgLWR2I8Abqs1jrJUe9bGMlJiXh7+5h9ewMo4+tLYsLZQu/v8L//oJSicngV5nz8DTEnTzBj4miOHvmX9xd+57ARX0H+PlxIu0hWlvkf2PnUdMoFlslX/62P1xHVsg7bv345p2zImIUcOn4agFuqVsS7tAfb9hzizflrua1GGBOevpfgID9GTvjKvsnkkpCQgI9P/uPl6+fH2TNnzMoSExMBKFPG16zcz8+PM6a6iYkJlPHNvz4tLY20tDR8fHxsnYJFklfxySuxKDnlidnX90rdxMQEfF3gWBWVK45idYkGsiQICCxLWloqmZmZZr/w58+l4B8YWOj9derWk9YdulCufAUAbq/bgHLlKzCgV2eOHfmXqjfVtFXoV5WQkkoZ79K4uSmzRtLf15vEc2n56s8b+xC+Pl70evp9YuKTaF6vBpOe6UHsmRTWbtvPzM/WM+vzjSQkXwDg1/3HyczK4oMxD/LCtMWkpWc4JK+goCBSU/Mfr5SUZMqWLWtWN/v1+fPm16WTk6/ULVs2iPPn8q/39PTE29vbHilYJHkVn7zKFpRTcjKBBeWUJ+bc+ZctG8S5vOudcKyKyhUbSOlitZHsrtX4uFM5ZVlZWcTHxlIhOP81xmvx9fPPaRyz3Xb7XQCcOvlf0QMtpLgzRtdqWIWAnDKlFKEVAojNc90wrEIAD3ZrzKDXPmXF5j/Z89cJZn2+kc9++IXhfSMBSDqXltM4ZvvtrxO4u7sRmus97K2i6bpvTExMTllWVhanYmIICQk1q+vj44O/vz8xJ0+alcfEnMypGxISkn/9yZOEhIY69A9f8io+eWXndCpvTqdiCAm1MqeTV/KvWDGEmBgLOTv4WJUkDm0glVJllVILlFKnlFIJSqmvlFIVc62/Tym1Vyl1Xim1TilVOc8ublVKbTWt/1Mp1SrXtrcopVYppRKVUqlKqd1Kqa651pdWSr2plDqmlIpXSn0E2KzPocYttxFUvgJbN1y5yP/nnl2cS0micbNWV9nSsomjnueF4QPNrh0c+ucAANWq33z9AVtp36EY4s6m0KFZ7ZyyhnWqEOjnzead/5jV9fY2rqVm5bneoTX4+pQG4Ku3BjHx6e5m62+9KYS09Esci0mwQwaW1a5Th+DgYNbmuk9s544dJCUl0ap1m3z1W7dtx5o1V+qmp6ezeeMG2rRrb6xv047o6LVcvnw5p87aNato27a9HbPIT/IyFIe8snPKHeeunUZOkZEWcmrTjrVrVue8Tk9PZ/OmDbRp2w6ANm3bsj56nVlOa9aszlnv6qy53ph2ZAdnN8w2W+zJYV2syvgKsxxwB/oAQcBEYD6wE6gBDAWGAFWBd4A3gMdy7eYD4FXgIjDBtG1107pPMMY09QVSgceBb5VSgVrri8A0oD/wFPCX6X2eAfZhAx4eHvQbMIS3J48hqHwFyvj6MfG15+nQpTuhlcJJTkwgOSmRiGrVr70zoFX7zgzp350KISF0iOpOYuJZpo0fxb3398t3y4g9Xb6cxXtfb2HsyHs4nXCOc6npTH/xfr6L/o0TsYmU9fehbIAPh0+c4dDx02zb/S/zxj7E6Fk/EHsmhWZ1qzO0d0uenvwNAGu27Wf2qD4kpKSy4ZcD3FS5PFOe68m0BdFkZmY5LC8PDw+GDhvB6FEvUyE4GD8/P559eiQ97utFREQECQkJJCYkUL1GDQCGDR9JVMd21KvXgLubNGXG9DfxDwige4+eAPTp159JE8YybMjjDBo8lLVrVrFp4wa2bNvusJwkr+KVl4eHB0OeGM7ro16hQgUjp+eefpIePXsRbiGnJ4aPoEun9tSrX9/Iadpb+AcEcK8pp959+zNpwjiGDR3MoMFDWLt6FZs3bmDz1l8cltN1seIk16d6Y3yqNzYrO793jZ0CcuB9kKazvQ1Ada31UVNZe4wGbxvwPyBcax1nWjcXuFVrHZlrkE5PrfV3pvW9ga+y731RSj0HrNZa7zO9bgZsBaoBZ4AkYKjW+kPTejfgdyAz9yCd7HtybrujrtW53d9vAPc/aJztzXlrAsu/+5q0tFRate3EK+PewsvbmznTJjJ3xiSL91d2uLs2Dw0alu8+yBXffcO82W9x4ugRgkPCiOrei8Ej/0dpLy+rY8utKPdBZnvtiS70jWqIt5cnq37cy7NTviX9YgavDoli1NAovOuOACAooAyvPdGFLi3rUC7Ql4PH45nzxSY+++HKH+mIfpEM7d2KShUDOXryLB8t3sq7X23ONxDIWkW5DxKM0bjj3nidL79YSFpqKp2jujJ95iy8vb0ZP3YME8a9QVrGlZiWLF7E1EkTOHr0CA0bNWbWnPeoWq1azvr9+/bxzFMj+G3PbmrefAuvvzGO9h06Wnhn+5K8HJ9XUT9Hs3P66svPSU1NpXNUF6a/fSWniePHknrpyhfH7xYvYsrkiRwz5fTO7Ln5cnr26ZE5OY0eM9Ysp48+/ID5H86zKrY9pgkGHHUfZLVnVhRp+yMzjMlT7BGnIxvIEcDzWuuqFtaNAfpprW/OVfYW0CBPA1lJax1jWt8VWJargSwFNAbuAOoCnYBwjAayLLAbuElrfSTXe8wCWlhqIAs7UUBxcD0NpCsragMphK0Uh9soCsvREwXc9OzKIm1/eLpxH2lxnyjAA7haH9q1Jy0toI5SyguIxuhmrQasAAbnqnLZ0nbXiEcIIcQNzJEN5H4gIvfAG6VUU6XUXozrkdejJdACuFNr/YLW+nvMe7QPAxlA21zvrUzbCSGEcDKlirbYkyPvg4wG9gJfK6VeAcoAYzGuD17v8MULGI39k0qpaKApxmAcgGbA18Bc4E2l1EWMQToDMc42j17newshhLhOrngrisMaSK11plKqIzAT+BZjNOs6jJGkQ65z9z8BrwNPAy+aXncFxgBvYXS5/g9IB8ZjNM7LgReAYdf53kIIIa6TNe3j+X9/4cIhx400lqd55CGDdIofGaQjnK0kfo46epDOzS+svlZVi/6Z2gko/oN0hBBCiGJD5mIVQgjhdC54CVIaSCGEEM7n5uZ6LaR0sQohhHA6W97moZTyUkotUkr9pJR6NFe5u1LqPaXUHqXUk9eKSc4gCzDmhZH5yiLbdyayfZSF2kIIUXytWL6MlSuWOTUGG9/m0QNjqtFZwAal1Gda68tAOyAFqA98r5Sao7XOLGgn0kAWYMzUWc4OQQghHKJL12506drNrOzjjz50UjQ2UR9Yarq98AQQgTFhTCvgR611llLqwWvtRBpIIYQQTpf7BDJh9woS9hRtblaTACD7QZsxGPNxA5QHepgmq1mutZ5wtZ1IAymEEMLpcnexlqvflXL1u16l9hV/jrP4DM9koDLGWWM4xtOcANIwHnE4BFiilLpZa/2PpR2ADNIRQgjhAqx5YLKlpQC7gPpKKXegEnDcVP4bkKS1zgDOcY02UBpIIYQQTmfjycqXYszDvRWYD7yslLoN+AZ4WCm1GziltT5wtZiki1UIIUSJorVOB3oVsNq6vlukgRRCCOECbuineQghhBAFccH2URpIIYQQzidnkEIIIYQF1rSPSQd+Ivnvn+0fjIk0kAWQqeaEEDcKV5hqzhqBtZoSWKupWdmZXSvs9n7ywOQ85IHJxY88MFk4W0n8HHX0A5MbjN9YpO13jWoN2CdOOYMUQgjhdC54CVIaSCGEEM7nioN0ZCYdIYQQwgI5gyxAg+6jnR2Czf2zfpqzQxBWOnvuorNDsItyfqWdHYJduOLZT3Hjiv+E0kAKIYRwOmu+ZCTu30bigZ8cEI1BGkghhBBOZ80ZZFDtZgTVbmZWFr9juZ0ikgZSCCGEC3DFbmoZpCOEEEJYIGeQBcg4ui5fmVvgTbgHVndCNEIIYT8rli9j5XLnzqTjgieQ0kAWxKNqe2eHIIQQDtGlaze6dO1mVjb/o3kOjcEVu1ilgRRCCOF00kAKIYQQFrhg+ygNpBBCiOLh7L6tnN23zWHvJw2kEEIIp7Omi7V8nRaUr9PCrCz2F/sNLpIGUgghhNNJF6sQQghhgQzSEUIIISxwwfZRZtIRQgghLJEzSCGEEE7n5oKnkNJAFkCmmhNC3ChkqjnLpIu1AB5V2+dbrG0cX328I3uXvMLhVWOYO6o33qU9Cqw7vE8L/lj0EvGbJrLmvWHUrVXZbP3D3Rqx55sXOfvjZHZ++T8e6FD3uvIqCq0106eMo0WD26h/W1X+9+QQ0lJTC6y/4oclRLVpQp1qFXmwV1eOHT1stv6XbVvo0TmSWyPK06FFAxbMm4vW2t5p5KO1ZvzYMdSuVYNq4aEMffwxUq+S15LFi2jaqD4h5QPpFtWRI4fN8/pr/346tW9DxXIBNG/SiDWrV9k5A8u01kybPI5m9W+j3q1VeW7k1Y/X8u+X0Ll1E26rWpH+9+U/Xls2RtOlXXNuDi9H22b1Wbr4a3unYFFJPF6uklOXrt2Y894HZoujKaWuuZzeu5X9X042W+xJGkgbe+mx9jzxQHNeeWcZA0d/TqPbqzB/bH+LdR/tfjdjnohi0kfr6DBkDgeOxLF89lCCg3wBaN2oJu+++gAfLNpG68fe4atVv/LJhIdoVvcmR6bEO9MmsWDeXF4dM4mZc+eze9cOnn5ioMW6P2/dzMjHH6ZHrz588s33lPH14/6u7UlPSwPgj99206d7J+5u2oKvf1jLY0NH8NbEMSz82LHzPgJMnjieuXNmMXHym8z/ZCE7dvzCwAEPWay7ZfMmHnmwL7379mfpslX4+vnRrk1L0kx5JSQk0KFtKyIiqrBs5Vo6R3Xh/p73svvXXx2ZEgAz35rEx/PmMuqNSbzz3nz27NrBk0MtH6+ftm5mhOl4fWY6Xvd1aZ+T199/7eeRPj2IbNOeJSvW89Cjj/PU0IFs2RjtyJSAknm8SmJOReWmrr2E3NGC2/u/bLbYk3LGN3dXppTSAF4Nnin0tqXc3Ti4YjRj31vNx0t/AaBh7Qg2zX+Sm7uN42R8sln9de8PZ8+B/3hhxvcAuLkpjqwaw6jZy/ls2U7mj+2Pr7cnD/zv45xtoj8Ywe9/n+S5ad8VOr5/1kwq9DYZGRncfUcNnnt5NP0efgyAPbt20L1TK375/SChlczPeIcM6Evp0p688/4nAKSnp9Pg1ipMeOsd7r2vN08M7M/Fi+nM/3xxzjYLF8xj3rsz2bT9zyIN9a7gX7pIedWsFs7oMeMYOOhxAHZs305kiyb8c/g4lSub59W3dy9Ke5ZmwWef5+RVtXIIM2fPpXefvsydM5vp06Zy4OAR3N3dAbi3a2fCwyOYPff9Qsd39tzFQm+TnVej22vw/Muj6f+Icbx279rBvR1bsf2Pg4TlOV6DH+mLZ2lPZn9w5XjVq1WFidPeoft9vXlz4htEr1nJms3bc7Z5/OE+lPEtw9vvflTo+Mr5Ff5YZeflyserJObk7WH8LWqt7dr5mf2Z2+ndX4q0/ephdwP2iVPOIG2odvVQQsr5s2bbXzllu/afICEllciGNfPV9/by4FzqlQ/CrCxNatolfLw8AYhPOMeqXPsCOJ14Hi+vgrtsbe3vv/ZxOj6ONu065ZTdWa8BgWWD2LplY776Wzetp3Wuul5eXjRr2TrnjOPg339Rt15Ds20aN2nBkUP/cvbMaTtlkd++vXuJi4ujY+eonLIGDRsSFBTEpg3r89XfEL3OrK6XlxeRbdqyft1aY/36dbRv3zHngwmgY6cooqPX2jGL/HKOV/srx+Au0/HaZuF4/bhpvVndnOO1wThe6enp+Pr6mm3j5+dHWmqanTKwrCQer5KY0/WwpovV0mJP0kDaUMXyfgDEnk3JKdNac+p0ChWD/PLVX7rhDx69tzF33ByGRyl3nnigOeUCyxD9y98AvPT2Dzlnop4e7kQ2rElkw5qs/HGfA7IxnI6PA6BCxZCcMjc3NyqGhHLmdLxZ3bTUVM6dS6FiSKhZecXQsJy65SsEExPzn9n6/04cAyAu9pTN4y9IXFwsACEh5nmFhoYRZ8o5W2pqKikpKYSFhZmVh4VVIt5UNzY2ltC86ytVIj4uzqHXV+PjjHiCLRyv01Yer5CwsJy6HTp3Yc+vO1n+/RIyMjL45acfWbHsO6K6dbdvInmUxONVEnO6HkoVbbGnYjOKVSm1AAjUWne38X6PAm9rrd++3n0F+ftwIe0iWVnmv4znUy9SLrBMvvpvfbKBqBa12f758zllQ8Z+xaETZ8zqNbmjKhs+ehKA977ZyootjmsgkxIT8PbxMftWClDG15eEhLNmZclJica6MuZnHL6+viScNep279Wb0S89S7uOXWgR2ZZ/Duxn3KgXAMi4dMleaeSTkJCAj4W8fP38OHvG/N8/MdFyXn5+fpwx1U1MTKCMpTOttDTS0tLw8fGxdQoWJSVZPl65j8GVulc5XqZj27hJcwYPf4qhj/bLWX9f737ce98D9gi/QCXxeJXEnK6HwnatnVLKC1gIhAHztNYfm8pvBr4BkoBkrfW9V9tPsWkgi4OElFTKeJfGzU2ZNZL+vl4knsvfJTXv9b74+pSm13MfEROfTPO6NzHpqXuIPZvC2p8O5NT77e+TNOjzJrfeVJGJT3YjISWVce+vdkhOgWWDSEtNJTMz0+wP+VxKCgEBgWZ1AwLLAnD+/Dmz8nMpKQQEGnUf6PcIMSf/Y+ijfbmYno6fnz/Dn3mByWNH5TuTsaegoCBSLeSVkpJM2bJlzepmv86bV3LylbplywZx/lz+9Z6ennh7e9sjBYsCAws+XoGmY3ClrhH7hTx5paSkEGg6tl98Op8vPv2YqW+/y+131uXwvweZNHYU0yaP47mXXrNrLrmVxONVEnO6Hm62PRvsAWwFZgEblFKfaa0vAzWAOVprq0YFSgNpQ3FnjF/OsAoB/BeXBBj96qHlA4g9k2JWN6xCAA92bUjj/m/xxz8xAOw58B+VKgYyvE9L1v50gEZ1qvDvidMkJKey79Ap9h06hY+XJ6OHdHJYA1khuKKRW2wMYZXCAcjKyiIu9hQVc3XjAXj7+ODn50/sqRiz8thTMTldfm5ubjz74ms8+dzLxMedIiS0Er/v3oWnpyflTe/lCNmxx8TEEB5+Ja9TMTGE5GmofXx88Pf3J+bkSbPymJiTOXVDQkLyrz95kpDQUIfOMRlc0XS8TsUQVvlKXrGxp8y6XeHK8Tpl6XiZuv3mzprOk8++SL+HjVGwt99ZF58yZRg5eAAjnnmB0qWLNuimsEri8SqJOdnKsS3fcezHwg9EzKU+sFRrnamUOgFEAIeBakA3pdQA4D2t9WdX24nLXINUSmmlVGul1AKl1Gml1EmlVL6hpEqpYUqpg0qpc0qpxUop/1zrHlFK7VdKpSulzpjWh+Raf7NSao1SKlkptU8pZXk8dRHtO3SKuLPn6NC0Vk5Zw9oRBPp5sXnnQbO63qaBNnm7Y7XW+Hobg3S+nDKAvp3rm633K1Oay5lZtgz7qm65tTblKwSzMXpNTtlvv+4kJTmJpi0j89Vv3qoNm9ZfqZuens5PP26iRWRbABZ9tZDxo1/C3d2dsErhuLm5sfz7xUTd0wMPD8cNPqpdpw7BwcGszXWf2M4dO0hKSqJV6zb56rdu2441a67UTU9PZ/PGDbRp195Y36Yd0dFruXz5ck6dtWtW0bZteztmkV/28dqQ63jtMR2vZgUcr9zHNj09nZ+2XDleqampKDfzjwmlFOnpaWRlZtonCQtK4vEqiTldj9wDb6q26kmrUZ9ZtRQgAMj+5hcDZJ+SHwJGA92Ap5VSgVeLydXOIGcCHwLvA08B05VS67TWe03rWwMXgQeBesDbpnrjlFK3AR8D04BvgXDgTWAiMFAp5QdsAQ4AXYEypu3Nx1Jfh8uZWbz37VbGDuvC6YTznEu9yPTne/Dd+j84EZdEWX8fyvr7cPi/Mxw6cYZtew4zb0xfRs9ZSeyZFJrddRND72/O01OXALB8y16ee6QtMfHJ/HPsNLffHMZLj7Vnzlc/2irka/Lw8OCRQUOZMm405csHU8bXl9EvPUvUPT2pVDmCpMQEkhITqXqTMYnCgMefoF/PKO64qz71G97N+7Nn4OfvT+eu3QEIj6jCs8MH4eHhQceoe/jlpx/58tP5fPX9mqtEYZ+8hg4bwehRL1MhOBg/Pz+efXokPe7rRUREBAkJCSQmJFC9Rg0Ahg0fSVTHdtSr14C7mzRlxvQ38Q8IoHuPngD06defSRPGMmzI4wwaPJS1a1axaeMGtmzbfrUw7JLXgEFDmTxuNOUrBOPr68trLz5LF9PxSjQdr2qm4/Xo4Cfo2yOKO++qT/1Gd/PeLPPj1bvfQ8yYOoGAgABq33EXh/89yITXX+aenvfj7cBrWiXxeJXEnK6HjU9ykzE+2w9jtAVJAFrrnK43pdRPQJXsdRZjcpXRTaZ7YWZrrUeaXgcDcUAPrfVS0yCdzkAVrXW6qc4qIE5rPUApVQuj4Zuhtc40rZ8H1NRaRyqlhgBvmbZPMK1vAOwEnskepJN9T47yCbY6dvcKt1Oqwh05r18b0om+nevj7eXBqh/38+xb35F+MYNXH+/IqMEd8W74LABBAT68NqQTXVrUplxgGQ4eP82cr7bw2bKdgHG2OHZYF7q1qkNQQBmOxpxl/tJfePfrH/OdeVqjKPdBgmkmncljWfLtl6SnpdG2Q2fGTp6Bl7c306eM4+2pEzh+Nj2n/oofljB7+hROHDvKXfUbMnHaLCKqVMtZv+SbL5j51kTiY2O5s259nnvldRo2blqk2KBo90Fm5zXujdf58ouFpKWm0jmqK9NnzsLb25vxY8cwYdwbpGVc+XdesngRUydN4OjRIzRs1JhZc96jarUree3ft49nnhrBb3t2U/PmW3j9jXG079CxSLEV9T7I7LzemjSW7779kjTT8Ro3ZQbe3t5MmzyOGVMn8F/CleO1/PslzDIdr7r1GzJp+pXjdfnyZT56bzZffvYxJ04cIyQklHvve4CRz7xYpAayqPdBZuflqseruOT00bwPmP+hdbPk7N79a3aMDrkPsseHu4q0/XeDGgDmcSql+gChwDvARqCt1jpDKTUOWA1sB9YB3bXWyfl2mr0fF2sg+2utv8hTlruBDNNad8i1fhFwXms9wPS6AtAYuANoBHQCfjE1kDOBu7TWrXJtr4AU4LW8DWRRJgpwdUVtIF1dURtIV3Y9DaQru54GUjiWoycKsHEDmT2KtRJGj2QEsAjjbPEHwBN4V2v93tX27WpdrAVPQniN9UqpphiJ7wZWYZwtngJuNVW5XMCmjruQIoQQwiJbdrGaehl7FbC6gbX7cbUG8nqMAKK11n2yC5RSI3Ot/wsYrJQKyu5ixTjTDHBgjEIIISxwxZG2LjOK1QYuAC2VUh2VUm1MXao9gXCl1E3Al8A5YLFSqoVSqj3GKfi5gncphBDCEWQmHfsajTFqaQnG4J5FGNchvwNe1FoPUUq1AmYDyzC6XycDNr3VQwghROFZ88DkmD2bifltiwOiMbhMA2npQnDusuyBOHnW98r18ymMUa55Vc1V5yCQd5jaJ4WPVgghhKOF1W1FWN1WZmWHN1/XhAJX5TINpBBCiBuX612BlAZSCCGEC3DFQTrSQAohhHA6G09WbhPSQAohhHA6OYMsRjKOrstX5hZ4E+6B1Z0QjRBC2M+K5ctYuXyZs8NwOdJAFsCjavGYAV8IIa5Xl67d6NK1m1nZ/I+semSizbjgCaQ0kEIIIZzPmi7WE7s38d/uG/A+SCGEEDcuawbpVKkfSZX6kWZl/25cbJ+AkAZSCCGEC3DFQTolaS5WIYQQwmbkDFIIIYTTud75ozSQQgghXIA1k5U7mjSQQgghnM4F20dpIIUQQjifNYN0ju3axPFfN9k/GBNpIAsgM+kIIW4UxWUmnSoNIqnSINKs7MD6RXZ7P2kgCyAz6QghbhQyk45l0kAKIYRwOhmkI4QQQljggu2jNJBCCCGczxVn0pEGsiA6y9kR2Jx2dgDCaq74YSHEjUYaSCGEEE7nivOeSgMphBDC6azpNTmycyNHdm50QDQGaSCFEEI4nTWPu6reqDXVG7U2K9u37ls7ReSaZ7VCCCGE08kZpBBCCKez5gzS0aSBLEDGsfX5ytwCquEeeJMTohFCCPtxhanmXHHktjSQBfCo0tbZIQghhEO4wlRztjyDVEp5AQuBMGCe1vrjPOsfAG7SWk++aky2C0kIIYQoGqWKthSgB7AVaAEMUEqVuvI+ygMYbU1McgYphBDCpfy++mt+X/3N9eyiPrBUa52plDoBRACHTesGA2us2Yk0kEIIIZwu92TldTv3oW7nPlZtN6XLLZaKA4AY088xQFkApZQf0BqYAzS+ZkxWRSCEEELYkVsRlwIkA5VNP4cDSaafnwVmYOXMm3IGKYQQwulsPIh1F1BfKbUNqAQcN5XfgnEGGQj4K6V2aK03FLQTaSCFEEI4nY2fB7kUYxTrA8D7wMtKqUVa634ASqlI4O6rNY4gDaQQQogSRmudDvS6yvpNwKZr7UcaSCGEEE7ngvMESAMphBDC+VxxqjkZxVqAjGPr8y2ZSYevvSHw6uBO7P3uVQ6vfoO5r/XBu7RHgXWH92nJH4tfIX7zZNa8P4K6tSqbrX/4nsbs+fZlzm6dys6vXuCBjvWuK6+i0FozY8o4Wja4jQa3VeV/Tw4hLTW1wPorflhClzZNuL1aRR7q1ZXjR83/3X7ZtoWenSO5LaI8HVs0YMG8uWjt+Mc5a60ZP3YMtWvVoFp4KEMff4zUq+S1ZPEimjaqT0j5QLpFdeTIYfO8/tq/n07t21CxXADNmzRizepVds7AMq010yaPo1m9W6lbqwrPjbz68Vr+/RI6Rd7NrVWC6XdfF44dte733NFK4vFylZxWLF/G8KGDzRZHc1PqmsvBXzaw/O3XzBa7xmTXvRdjHlXa5lusmYf1pcc68MQDLXhl5g8MfG0hjepUYf64By3WfbT73YwZ1oVJH62lw+DZHDgSx/I5TxAc5AtA60Y38+6rvflg0VZaD5zJV6t+5ZMJD9OsrmPng31n2iQWzJvLK2Mm8fbc+ezZtYOnnxhose7PWzfz5OMP071XHxZ88z1lfP24v2t70tPSAPjzt9307d6Ju5u24Osf1jJw6AimTRzDwo8dO60VwOSJ45k7ZxYTJ7/J/E8WsmPHLwwc8JDFuls2b+KRB/vSu29/li5bha+fH+3atCTNlFdCQgId2rYiIqIKy1aupXNUF+7veS+7f/3VkSkB8PZbk5j/wbuMGjuZd97/mN27tvPk0Ect1v1p62aGD3qIHvf34bNvf6CMrx89u7TLyatvzyhuCvG3uCz66nNHplUij5er5NSlazfmvPeB2eJo1syaU6tpW+59drzZYteYnPHN3ZUppTSAV/2nCr1tKXc3Dq4Yw9j3VvLx0l8AaFi7Cps+foqbu77Byfhks/rrPhjJngMneGH6UgDc3BRHVo9l1KxlfLZsB/PHPoivT2keeP6jnG2i543k979P8txbSwod399rpxR6m4yMDJrcUYPnXh5N34cfA2DPrh306NSKn38/SGgl8zPeoQP64lnak3fe/wSA9PR0Gt5ahfFvvcO99/Vm2MD+XLyYzkefL87Z5vMF85j37kw2bv+zSBMWB/uXLlJeNauFM3rMOAYOehyAHdu3E9miCf8cPk7lyuZ59e3di9KepVnw2ec5eVWtHMLM2XPp3acvc+fMZvq0qRw4eAR3d3cA7u3amfDwCGbPfb/Q8SWcv1TobbLzalinOv975XX6P2Icr927dnBPh5bs+PNfwvIcr8cf6UNpz9LMnnfleNW9JYJJ02bRvVdvTp44Tlqa+RnNT1u38PabE1mzeTsVgisWKr4gX88i5+XKx6sk5uTtYfwtaq3t2vmZ/Zk7dt3BIm0/un1NwD5xyhmkDdWuEUpIeX/WbPsrp2zX/uMkpKQS2fDmfPW9S3tw7sLFnNdZWZrUtEv4eBkfIvEJ51i1dZ/ZNqcTz+N1lS5bW/v7r32cjo+jdbtOOWV31mtAYNkgtm3J/2TvrZvWm9X18vKiacvW/LgxGoCDf//FXfUamm3TqEkLjhz6l7NnTtspi/z27d1LXFwcHTtH5ZQ1aNiQoKAgNm3I/ySXDdHrzOp6eXkR2aYt69etNdavX0f79h1zPpgAOnaKIjp6rR2zyC/7eLVpf+UY3GU6XlstHK8fN643q+vl5UXzVq3ZbDpelcIjqHFzrZwlqFx5Zs94kzkfflroxvF6lMTjVRJzuh5uqmiLXWOy7+5vLBXL+QMQezYlp0xrzanTKVQs55ev/tINv/No97u54+ZKeJRy54neLSgXWIboXw4A8NLb3+eciXp6uBPZsCaRDW9m5Y97HZCN4XR8HAAVKobklLm5uVExJJQzp+PN6qalpnLuXAoVQ0LNykNCwzhtqlu+QjCnYv4zW//fiWMAxMWesnn8BYmLizViCzHPKzQ0jDhTztlSU1NJSUkhLCzMrDwsrBLxprqxsbGE5l1fqRLxcXEOvb4ab8orOM/xCgkJ5UyevHKOV2j+45W3brbRLz1Hp6730KRZSxtHfnUl8XiVxJyuhyrif/Yko1htKMjfhwtpF8nKMv9lPJ+aTrnAMvnqv/XJeqJa1mb7F//LKRsy9ksOnThjVq/JndXY8JHR5fveNz+yYov5WaU9JSUm4O3jY/atFKCMry8JCWfNypKTEo11ZXzz1U08a9S9t1dvXn/pWdp27EKLyLb8c2A/40a9AEDGpaJ1KxZFQkICPhby8vXz4+wZ83//xETLefn5+XHGVDcxMYEyvvnXp6WlkZaWho+Pj61TsCgpMbGA4+VHwlnz45VU4PHyy3dsAX7duZ1NG9ax9VfH/f5lK4nHqyTmdD1ccRSrNJA2lJCSShnv0ri5KbNG0t/Xm8SUtHz1543ph6+PF72enUdMfDLN61Vn0lP3EHsmhbU/Xemm/e3AfzToPYVbbwph4lP3kJCcyrj3HTPiLrBsEGmpqWRmZpr9IZ9LSSEgINCsbkBgWQDOnz9nVn4uJYWAQKPuA/0e4dTJ/3ji0b5cTE/Hz8+f4c+8wOSxo/KdedpTUFAQqRbySklJpmzZsmZ1s1/nzSs5+UrdsmWDOH8u/3pPT0+8vb3tkYJFgWXLFnC8knOOT07dAo9Xcs7xym3SG6MYPOypnO0cqSQer5KY0/VwxQZSulhtKM7UtRpWISCnTClFaHl/Ys+kmNUNqxDAg10bMej1z1mxZR97DvzHrC8289myHQzvY3RfNapThaAAH9IuZrDv0CkWrdvD+A9W88g915yE3mayrzPFxcbklGVlZREXe8qsGw/A28cHPz9/4k7FmJXHnYrJqevm5sYzL77G/mNn+PmPg/xxOJa7m7XE09OT8g68plXRFE9MjHlep2JiCMnTUPv4+ODv70/MyZNm5TExJ3PqhoSE5F9/8iQhoaEOfVJ69r9z7CkLxyvE8vGKjTE/XrGnYgiuaP5vsPfP39m5/Sf6PPiInSK/upJ4vEpiTiXNDdFAKqVWK6XyDeNSSu1QSl31idKFse/fU8SdPUeHprfmlDWsHUGgnzebd/5jVtfbyxhok5WVZVautcbXxxiV+eXUR+nbuYHZej+f0lzONN/Gnm65tTblKwSzKfrK49N++3UnKclJNG0Zma9+s1Zt2Lj+St309HR++nETzSPbArD4q4WMH/0S7u7uhFUKx83NjeXfL6bzPT3w8HDc4KPadeoQHBzM2lz3ie3csYOkpCRatW6Tr37rtu1Ys+ZK3fT0dDZv3ECbdu2N9W3aER29lsuXL+fUWbtmFW3btrdjFvllH6+NuY7Xnl93kpycRPMWkfnqt4hsY1Y3PT2dbVs20dJ0vLJ9+tF7tO3QOd+XIkcpicerJOZ0PZRS11z++nkDi6e9arbY043Sxfoe8LlS6kWtdRKAUqoh0ADobas3uZyZxXvf/MjY4V05nXCec6npTP/ffXy3/ndOxCVR1t+Hsv4+HP7vDIdOnGHbnkPMe6M/o2cvJ/ZMCs3q3sTQB1rw9NRFACzfso/nBrQl5nQy/xyN5/abw3hpUAfmfLnFViFfk4eHBw8PGsqUcaMpVz6YMr6+vP7Ss0Td05NKlSNISkwgKTGRqjdVB2DA40/Qv2cUd9xVn/oN7+b92TPw8/enc9fuAFSOqMKzwwfh6eFBh6h7+OWnH/nq0/l89b1Vzy+1aV5Dh41g9KiXqRAcjJ+fH88+PZIe9/UiIiKChIQEEhMSqF6jBgDDho8kqmM76tVrwN1NmjJj+pv4BwTQvUdPAPr068+kCWMZNuRxBg0eyto1q9i0cQNbtm13eF4DHn+CSWNfo3yFCpTx9eO1F5+hy709qRQeQWJiAkmJCVS7ycjr0cHD6NO9M3fUrUeDRk14b9Z0/PwD6Nyte84+s7KyWL1iGc++YN8Po6spicerJOZ0PazpYq3TrC11mpl/edux4ms7RXSD3AeplCoFHAWmaq3fMZV9DFTUWkflqVvk+yCzvTakM30718fby5NVW/fx7JtLSL+YwauDOzFqcCe8GzwNQFCAD68N6UyXlnUoF1iGg8dOM+cro5sVwK9MacYO70q3VrcTFODD0ZgE5n/3M+9+vSXfQCBrFOU+SDDOaqdPHst3335JeloabTp0ZuzkGXh5ezNjyjjenjqBY2fTc+qv+GEJs6dP4b9jR7mrfkMmTJtFRJVqOeuXfPMF77w1kfjYWO6oW5/nXnmdho2bFik2KNp9kNl5jXvjdb78YiFpqal0jurK9Jmz8Pb2ZvzYMUwY9wZpGVf+nZcsXsTUSRM4evQIDRs1Ztac96ha7Upe+/ft45mnRvDbnt3UvPkWXn9jHO07dCxSbEW9DzI7r7cmjWXJN1+QlpZGu45RjJsyA29vb6ZNHsf0KeM5mXjl9qLl3y/hnWmTOXHsKHUbNGTy9Nlmx2v3rh10a9+C9dt2U+u22kWOC4p+H2R2Xq56vEpiTo6+D3La5kNF2v65VsaXc3vEeUM0kABKqdFAP+BWIAj4D3hAa70sTz0NoHyCrd63e/k6lKpwu+2CtZOiNpCurqgNpCu7ngbSlV1PAymu30fzPmD+h9bNkrN7tzEDj6MayBlbijbF4TMtjZnF7BHnjdLFCvAhMBpoA9QFTgMrC6pc+ta+DgpLCCEc47HHB/PY49bNs5p9Bnkju2EaSK11jFLqe2AkcDvwgdY608lhCSGEwDVv87hhGkiTucA64DLw0TXqCiGEcBBXvBPlRmsgNwCHgd+11o6b10wIIcRVuVkxbdyfW6P5c1v+eWrt5YZqILXWWUqpE8BxZ8cihBCicG5v3o7bm7czK/tp2Vd2e78bqoEUQgjhmqSLVQghhLBABum4AK11pLNjEEIIYc7NBU8hb7gGUgghhOtxwfZRGsiCZBzLP1LKLaAa7oE3OSEaIYSwnxXLl7Fy+bJrV7zBSANZAI8qba9dSQghSoAuXbvRpWs3s7L5H81zaAzSxSqEEEJYYE37+NuP0fz+Y7T9gzGRBlIIIYTTWfNw4not2lGvhfl9kFuWfmmfgJAGUgghhAtQLtjFak2jLYQQQtxw5AxSCCGE07ne+aOcQQohhHABbkoVabFEKeWllFqklPpJKfVorvIIpdROpdSfSqnHrhmTDfMTQgghikQVcSlAD2Ar0AIYoJTK7i0dDIwF6gJDrxWTNJBCCCGcTqmiLQWoD+zSWmcCJ4AIU/kPwEbAG0i7VkxyDVIIIYRL2bDkczYs+fx6dhEAxJh+jgHKAmitdyilagI7ganX2ok0kAWQqeaEEDcKV5hqLvdtHm3ve5C29z1o1Xb96lW2VJwMVAYOA+FAkuk9grTWB5VSIUC0UuptrXVqQfuWBrIAMtWcEOJG4RJTzdl2d7uA+kqpbUAl4LipfJ5S6lXgH655GVOuQQohhHABSqkiLQVYCjTDGKgzH3hZKXUbMAP4DtgNfK21vnC1mOQMUgghhNPZ8j5IrXU60KuA1bdaux85gxRCCCEskDPIAvR/YZCzQ7C5F5btd3YIdrGgf11nh2BzQb6ezg5BFMKZcxedHUKx54pzsUoDKYQQwums6c7cuXktuzavs3ss2aSBFEII4XTWnEE2iuxIo8iOZmXR13e/5FVJAymEEMLpXK+DVQbpCCGEEBbJGaQQQginc8ExOtJAFmTrB2PylUXUjySifqTDYxFCCHtat3oF0atXODUGNxfsZFVaa2fH4FKUUhrgsa//dHYoNnc+LcPZIdhFSbzNQxQvJfE2j/AgLwC01nZtubI/c5f9GVuk7bvdHgLYJ065BimEEEJYIF2sQgghnE5Z0cW6fdMadmxa64BoDNJACiGEcDprBunc3bojd7c2vw9yzeKFdopIGkghhBAuwBUH6UgDKYQQwulc8TYPGaQjhBBCWCBnkEIIIZzOFc8gpYEUQgjhdNaMYnU0aSCFEEI4nZsV7ePPG9fw88Y19g/GRK5BFmDrB2PyLcd/3XTN7bTW7P72Xb55MoovhrTmx/dGc/li2jW3y0hPZdHT3di74rMC62ya9RLr3nyyMGnYhNaavd+9x4oX7uH7pzqw46M3rMrp8sU0Vr7Ug7/X2O9xNNdDa834sWOoXasG1cJDGfr4Y6SmphZYf8niRTRtVJ+Q8oF0i+rIkcOHzdb/tX8/ndq3oWK5AJo3acSa1avsnIFlkpehOOSltWb65HE0r38b9W+tyvMjh5B2lZxWfL+EqNZNqF21Iv3v68qxo+Y5/bxtCz06RVIrojztmzdgwby5WDNb2rrVK3jx6WFmi6MpK/5r2roTz42dYbbYkzSQBWg+eEy+xZp5WH9b8j77V39BowefI3LEJOIP/sHm2a9cc7vtn04l+dTRAtcf2rqCQ1udM1fi/h8+5GD019zZ+2nuHjKOs4f+ZPsHr11zuz1fvMW52GNmZZveHMa3g+62uBzdttxeKVg0eeJ45s6ZxcTJbzL/k4Xs2PELAwc8ZLHuls2beOTBvvTu25+ly1bh6+dHuzYtSUszvigkJCTQoW0rIiKqsGzlWjpHdeH+nvey+9dfHZkSIHlB8clr5luT+HjeXF59YxIz35vPnl07eGroQIt1f966mRGPP0z3Xn349Jvv8fX1o1eX9jk5/fHbbvrc24nGzVrwzQ9reWzoCN6cMIbPPp53zTjad+rClLffNVuEzMWaz/XMxZp1OYOvhrWnXu8R1GrbC4D4g3+w7LUH6TNnLWXKhVjc7uiO9fz00XiyLmdwV88h1Oli/kd/Lv4kS196AJ/A8viHVqH9/94pdGxQtLlYsy5nsOzZKOr0fILqkT0BOHvoT6LHD6DbtJX4BFW0uN1/v27g108nk3U5g9vuGcQtHfsDcOHsKTIvppvVjT/wK/t++ICOb3yJV0C5QsdYlLlYMzIyqFktnNFjxjFw0OMA7Ni+ncgWTfjn8HEqV65sVr9v716U9izNgs+Ms+H09HSqVg5h5uy59O7Tl7lzZjN92lQOHDyCu7s7APd27Ux4eASz575f6PiKSvJyTl5FmYs1IyODxrfX4PmXR9PvkccA2LNrB/d2bMX2Pw4SWsk8pyGP9MWztCezPvgkJ6f6taowYdo7dL+vN0882p/0i+l8/MXinG0WLpjHB3NmsnnHn1Y9kDg3R8/FuuHAmSJt36ZWeUDmYnV5CSf+JS35LOF1W+SUVaheh9K+AcTs3W5xmwsJ8Wz74A1aPjEOD+8y+dZnZWWyec4r3Nr+ASrUqGO32AuSfPIQ6SlnCb2zeU5ZULXaeJYJIG7/DovbpCWeZteC8TR67HVKefmYrStTLhT/sGo5S2m/QP5aMZ8mQycVqXEsqn179xIXF0fHzlE5ZQ0aNiQoKIhNG9bnq78hep1ZXS8vLyLbtGX9OmPaqw3r19G+fcecD1uAjp2iiI523LRYIHllKw55/f3XPk7Hx9G6faecsjvrNSCwbBBbt2zMV//HTetpk6uul5cXzVq2ZsuGaAD++fsv6tZvaLZN4yYtOHLoX86eOW2nLGzHmi5WS//ZkzSQNpSWZHwD8g4sn1Om3NzwKVuBtKSz+errrCy2vPsqNzXrTOW7mudbD/D7dx9y+WIade93/DUBgPRkI+7cjZdyc8M7sDzpKZZz2v7haCIadyL0jmbX3P/uz9+iUr3WBNeqb7ugrRAXZzw5ICTkylm9m5sboaFhxMXHmdVNTU0lJSWFsLAws/KwsErEm+rGxsYSmnd9pUrEx8VZdQ3IViSv4pPX6TgjluCK5jlVDAnlzOl4s7ppqamcO5dCxZBQs/KQsLCcuhWCgzl18j+z9f8dNy5xxMWesnn8tuamirbYNSb77v7GcvF8MqVKe+Pm5m5W7uFVhvRzSfnq7135GamJp2nY/xmL+4s/+Ad/LltA5MjJuJfysEfI13TxfDLunl75cirlVYZLFnL6Z+0XpCWe5o4Hrj2Y6My/fxC792fqdB9iq3CtlpCQgI+Pj9kZBICvnx9nz5h39SQmJgJQpoyvWbmfnx9nTHUTExMo45t/fVpaWs41IkeQvIpPXklJCXhbysnXl4SzZ/PUNXLyyZNTGV9fEhKMut3v682irz9n/dpVXLp0ib1//MbYUS8AcOnSJXulYTNyBukilFKRSqkxtt5vad8ALl9MIysr06z8Uup5SpfxNytLOH6QPYvmEjlyCqU8vfLt6/KldDbNeomG/Z8hsNJNtg7VaqV9A8i8lJ4vp4y083jkySnpv3/Zu/R97h460WJOef3x7Sxu6dgfzzz7cYSgoCBSU1PJzDTPKyUlmbJly5qVZb8+f/6cWXly8pW6ZcsGcf5c/vWenp54e3vbOvwCSV7FJ6/AwCDSLOaUQkBgYJ66RtwX8uR0LiWFgACj7gP9H+GJkc8yZEBfqof480C3DtzfzxjPEJLnzFNYR+6DtKHsrtXUhNP4ljc9xDMri9TEeLzLljere+bwPjLSLrBsVL+cssyMS+xYOI1dX77NfdN/4FzcCX5ZMJntn0wBIOvyZTSaBQ/Wp+Mr7xN6WwO755TdtZqeeBqfcldySk86jXeAeU6JR/ZzOf0C0WMfzinLunyJ379+mz++ncX9H/5ype6xvznz7+80eWKS3XOwpKKpWysmJobw8HAj1qwsTsXE5Psw8fHxwd/fn5iTJ83KY2JO5tQNCQnJv/7kSUJCQws9OOJ6SF7FJ68KFY0BbnGnYgirfCWnuNhTZt2uAN4+Pvj5+RN7KsasPPZUDMGmbmc3Nzeefek1nnz+ZeJjTxESVonfd+/C09OT8sGWB9O5Emv+2bdtWM22DavtH4zJDXUGqZRqpJQ6AHwKjFBKHVBKjbDV/oPCa+AVEMR/v/2YU3b63z+5lHqOsDqNzepWbdSW+6Z/T/cp3+YsPmWDub3bALpP+ZYyQcHcN/17ekxdlLM+okEkobUb0X3Kt1SoXttWYV9VQKXqlPYP4tSf23LKzh7ey6XUcwTfZj4goFL91nSeuJiOY7/MWbwDK1Cr88N0HPulWd1/N3xD2B3Nza7XOlLtOnUIDg5mba5733bu2EFSUhKtWrfJV79123asWXOlbnp6Ops3bqBNu/bG+jbtiI5ey+XLl3PqrF2zirZt29sxi/wkL0NxyOuWW2tTvkIwG6Ov3Pj+2687SUlOolnLyHz1m7dqY1Y3PT2dn7ZsomVkWwAWfbWQca+9hLu7O2GVw3Fzc2PZ0sVE3dMDDw/nXKIpDGXF0rxNJ14c/7bZYk8l5gxSKVUaGAv0AkKAX4FXtNZbs+torXcAtZRSkUCk1nqMLWNwK+XBbR37suvLmXgFBOHpVYafP55Etcbt8S0fysXzyVw8n4x/SASePn54+vjl2b4U3gHlcrpU83atevr4orV2aJerWykParbtzR/fzqa0XxAe3j7sXjiV8AbtKFPOyOnShWT8KlrOSbmXorR/EP5h1XLKdFYWJ3dvova9gx2WR14eHh4MHTaC0aNepkJwMH5+fjz79Eh63NeLiIgIEhISSExIoHqNGgAMGz6SqI7tqFevAXc3acqM6W/iHxBA9x7GrS99+vVn0oSxDBvyOIMGD2XtmlVs2riBLdssj16WvCQvDw8PHhk0lMnjRlOuQjC+vr6MfvFZutzTk0qVI0hMTCApMZFqN1UH4NHBT9C3RxR33FWfBo3u5r1ZM/Dz96dT1+4AVI6owjPDBuHh6UHHqHvYvu1Hvvx0Pl//4LiZZ66HmwtOxlpiGkhgHlAbGAacBR4F1iul7tJa/1XYnX3/cm+r697Sthe12t0PwF09h5CVeZkdn77J5YvphNdrRZOBLwOwb9Xn7Fk0t0j3WDrTbfcMIivzMr99NZ3Mi+mE3tWCeg8aF/8PrvuSfd9/QO8Fu63eX8KRfVw8l0gFB49czeulV0aRkZHBi/97lrTUVDpHdWX6zFkAvDv7HSaMe4O0DGNEY8tWkXz6+VdMnTSBSRPG0rBRY9at35xzvSooKIg10Zt45qkRdIvqQM2bb2Hx0mXcVbfw92hKXjdOXk89/zKXMzIYN+oF0tLSaNuhM2OnGLPDfPz+HGZMncCJBOO+4SbNWzHnw8+YNX0KM9+cyF31G/Lt8nU5Od3dtAUz35vP229OZMG8udxZtz6ffvM9d9xVL+f9Pl/wIZ9/Ot+hOVrL9ZrHEjJRgFKqKnAYqKa1PmYqU8BPwDKt9URTWWMg77xn72it38m1ryJPFODqijJRQHFQlIkChLClokwU4OocPVHAzwcTi7R9k5rGACZ7xFlSziDrYHwB+TvPBXZP4ED2C631dqCGY0MTQghxTTZs3pRSXsBCIAyYp7X+2FReBlgKBAM7gMH6KmeJJWWQTikgC6gP3JVruQ0Y7ayghBBCWMfG90H2ALYCLYABSqnsk8H7gI0Y7YMGrjqbSUlpIP/CyMVXa31Aa30AOIRxPbKWUyMTQghxTUoVbSlAfWCX1joTOAFEmMoPAl+azhpjrxVTiehi1Vr/rZT6HvhUKfUskAQMAnoC05wZmxBCiGvL3dZ99+UCvvtqwfXsLgDIvmk0BigLoLX+WRkexDh7HHe1nZSIBtLkQWAy8C5QDtgJtM8etCOEEKJ46NF3AD36DrCqbqPqgZaKk4HKGIM3wzFOmrIHb04BKgLdtdZXHblYYhpIrfV5YIRpEUIIUZzYdgzqLqC+UmobUAk4birvYfr/gKsNzslWUq5BCiGEKMZsPEhnKUYX6lZgPvCyUuo2oBHQFtiolNqklLL8GCWTEnMGKYQQoviy5UQ6Wut0jFnV8nrJtFhFGkghhBBO54oz6UgDWYCtH4zJVxZRP5KI+pEOj0UIIexp3eoVRK9e4ewwXE6JmGrOlmSqueJHppoTziZTzRVd9mfur0eTr1l3S/QqtqxfZVb23ZefADLVnBBCiBLqKgNucrRqF0WrdlFmZdkNpD1IAymEEMLpXPBpV3KbhxBCCGGJnEEKIYRwOhc8gZQGUgghhAtwwRZSGkghhBBOZ80gHUeTBlIIIYTTueIgHWkghRBCFAub1q1kc/Sqa1e0EWkghRBCOJ01J5Ct20fRur35fZCLv1hgl3hAGsgCyVRzQogbhUtMNeeCXawy1VweMtVc8SNTzQlnk6nmii77M3fvf+eLtH2dyr6ATDUnhBCihHLFQToyk44QQghhgZxBCiGEcDoXPIGUBrIg/5254OwQbK5u1UBnhyBEiZQlQzmunwu2kNJACiGEcDprZtLZuHYlG9etdEA0BmkghRBCOJ01g3TadIyiTUfz+yC//fxjO0Ukg3SEEEIIi+QMUgghhNO54CVIaSCFEEK4ABdsIaWBLMC+LyblK6twe3OCb2/hhGiEEMJ+olevIHqNc6eac8XHXclUc3lkT3vUcc4vzg7F5krqbR6vd7jF2SGIG1x8Ssmbaq5KOcdONXcwLrVI29es6APIVHNCCCFuYOvXrGDjWrnNQwghxA3EmtO/dh270K5jF7Oyrxfa7zYPaSCFEEI4n+tdgpQGUgghhPO54iAdaSCFEEI4nTzuSgghhLAzpZSXUmqRUuonpdSjeda5KaWWWrMfaSCFEEI4nSriUoAewFagBTBAKVUKQClVBfgTqGVNTNJACiGEcD7btpD1gV1a60zgBBBhKj8O3AkcsyYkuQZZAJlJRwhxo3C1mXS++ORDvvx0/vXsLgCIMf0cA5QF0MbMOJezJye4FjmDLEDtfi/nW67VOGqt+XfFPLa83ouNL3dh78IJZF5Kv+5Yko7s5aeJD133fopKa82Pn8/ivUEdmPVgC1a+/SoZ6WkW616+dJENH07h3QFtmN6rPl+89DCxB/ea1Tn+5w4+e74v03vV46Ph9/DrsoU4Y0YnrTXjx46hdq0aVAsPZejjj5GaWvBsHksWL6Jpo/qElA+kW1RHjhw+bLb+r/376dS+DRXLBdC8SSPWrF5l5wwsk7wMxSEvrTUzpoyjZYPbaHBbVf735BDSrpLTih+W0KVNE26vVpGHenXl+FHznH7ZtoWenSO5LaI8HVs0YMG8uVb9bbXr1IXJM941W5yp3yODWLb+J6uWAiQDlU0/hwNJRYlDGkgbOrz6Y45v/pZbeozgjkfGkHRkL398MsZi3dTT/7HuqZb5lpM/Lzerl54Uzz/fzXZA9AX76au5/LpsIa0H/o9uz08l5u/fWTbthQLr7tu0nHZDXqHfpE/wDarA16MfJ+1cEgCxB/fy5SsDiLi9If0mfUqDex9hy2cz2bPyKwdmZJg8cTxz58xi4uQ3mf/JQnbs+IWBAyx/EdmyeROPPNiX3n37s3TZKnz9/GjXpiVpacYXhYSEBDq0bUVERBWWrVxL56gu3N/zXnb/+qsjUwIkLyg+eb0zbRIL5s3llTGTeHvufPbs2sHTTwy0WPfnrZt58vGH6d6rDwu++Z4yvn7c37U96aac/vxtN327d+Lupi34+oe1DBw6gmkTx7Dw43mOTKnIlCraUoBdQH2llDtQCaNrtfAxyVys5oo6F2tW5mU2v3oPNboNJrxZd8A489s+7XFajfser7LBZvXj/9zKnwtep/H/PjIrL+1fDg8fPwC2Tx9C0qHfAfCrVJOmr3xWpJyyFWUu1szLGbz7SGtaPPQkd3V6AICYA7/z6fN9GLZgI/7lQ8zqz3m4FY16DqRh90cAyEhPY0bvRtz7wjRuadaBpZOe5vKli/R6fW7ONntWfcWOJR8z+IPVqCKM9S7KXKwZGRnUrBbO6DHjGDjocQB2bN9OZIsm/HP4OJUrVzar37d3L0p7lmbBZ58DkJ6eTtXKIcycPZfeffoyd85spk+byoGDR3B3dwfg3q6dCQ+PYPbc9wsdX1FJXs7JqyhzsWZkZNDkjho89/Jo+j78GAB7du2gR6dW/Pz7QUIrmec0dEBfPEt78s77n+Tk1PDWKox/6x3uva83wwb25+LFdD76fHHONp8vmMe8d2eycfufhf7bcvRcrMfPFq23LcJCnEopL2AhRuP4PsY1yEVa6/2m9au11p2utW85g7SR8zGHuHQugQq1m+aUBVS5DQ8ff87+vTNf/Qtxx/ANvQnfkKpmS3bjCFCn/6s0ffVzwlv0dEgOlpw+dpALSWeo3qBVTlnozbfj7RfAsd9+NqublZlJuYjqVLq1bk6Zh5c3nt4+XL5kfICcOXGIsFvuNNsuvHZDEmOOkZqcYMdMzO3bu5e4uDg6dr7ydPIGDRsSFBTEpg3r89XfEL3OrK6XlxeRbdqyft1aY/36dbRv3zHnwxagY6cooqPX2jGL/CQvQ3HI6++/9nE6Po7W7a58Tt9ZrwGBZYPYtmVjvvpbN603q+vl5UXTlq35cWM0AAf//ou76jU026ZRkxYcOfQvZ8+ctlMWtmPLM0itdbrWupfWuonWeoHWemx242haf83GEaSBtJmLKWcB4wwwm3Jzo3RAeS6dy//BfyHuKJmXL7L9rcdZ/78O/DxlADE7VpldLyhTMQK/sOp4+pW1fwIFuJB4xoilbPmcMuXmhm9QMBeSzprVdXN3p8/4+YTdcgdaay6mnmfXD5+RdTmDiNsbGfsJLEfKmVNm2yXHnwTg/Nl4e6ZiJi4uFoCQkCtnwG5uboSGhhEXH2dWNzU1lZSUFMLCwszKw8IqEW+qGxsbS2je9ZUqER8X59Drq5JX8cnrtCmWChXNc6oYEsqZ0+Z/C2mpqZw7l0LFkFCz8pDQME6b6pavEMypmP/M1v93whisGRdr/jfnmmx8o4cN3JANpFIqUik1xpb7zLiQgrunF8rN3ay8lJcPl84n5at/IfYYGeeSqNKmD/WHz6DC7S3489NxxP66zpZhXbf0c8l4lPbGzd08L0/vMqSlJBa43c9fv8+M+xsQ/f4Eop6eiF/5igDc1qorezd8z787NpGZcYnYQ/vZMG8yYHTnOkpCQgI+Pj5mZxAAvn5+nD1zxqwsMdHIs0wZX7NyPz8/zpjqJiYmUMY3//q0tLSc616OIHkVn7ySEhPwtpBTGV9fEhLMv3wmJ1nOqYyvL4lnjbr39urN4q8/Z/3aVVy6dIm9f/zGuFHGWIGMS5fslUaJJrd52IhHGX8yL6WjszLNGsnLaRfw8PHPV/+uQZNwL+1NKS/jWWaBVWuTfvYUxzd9S2iDDg6L+1q8/ALIuJhGVmamWSN5MfU8Xr7588p2V+fe3NSgBSf27mLl26/g4eVD9QYtuaN9T1LOnGLppKe4fOkipX18afLAEDYtmIZvueAC92drQUFBpKamkpmZafYBlZKSTNmy5mfs2a/Pnz9nVp6cfKVu2bJBnD+Xf72npyfe3t72SMEiyav45BVYNog0CzmdS0khICDQrG5AoOWczqWkEBBo1H2g3yOcOvkfTzzal4vp6fj5+TP8mReYPHZUvjNPV+SKU83dUA2kUqoR8CngA/gopfoAs7XW1z1MNLtrNT35DN5ljbMlnZVFevIZs27XnPoB+cv8wm/hzIHt1xuKTWV3rZ5PiMe/gvFHprOyOH82njJlK5jVvZh6njPH/yWkRm18AsriE1CWkBq1iT20n/0bl1G9QUuUmxst+o+kWZ8nOJ9wGr9yFTn1z5+4l/KgTGD+fxN7qWjq1oqJiSE8PByArKwsTsXEEJLnw8THxwd/f39iTp40K4+JOZlTNyQkJP/6kycJCQ0t0sCjopK8ik9eFYKNz4m42BjCKl3JKS72FMEVzQe/efv44OfnT9ypGLPyuFMxOXXd3Nx45sXXGPncy8THnSIktBK/796Fp6cn5U3v5cqs+Vdfu2q5Q+/XLBFdrEoprZTqnuu1r6ksMnc9rfUOrXUt4GGMhrGWLRpHAF/TtcIz+64MXEk+tp/LaecIuqWBWd3LaRfYOrY3sbs3mJVfiD2Cb0hVW4RjMxWq1MQnsByHd23JKYv55w/SL6RQ5c67zepeSDzDZ8/14czxQ2bll9IuoEzfkP9cv5QNH05BubnjXyEU5ebGX1tXcUvzjriX8rB/Qia169QhODiYtbnufdu5YwdJSUm0at0mX/3WbduxZs2Vuunp6WzeuIE27dob69u0Izp6LZcvX86ps3bNKtq2bW/HLPKTvAzFIa9bbq1N+QrBbIpek1P22687SUlOomnLyHz1m7Vqw8b1V+qmp6fz04+baB7ZFoDFXy1k/OiXcHd3J6xSOG5ubiz/fjGd7+mBh4fj/raKypoBOR2juvLmzLlmiz3dUGeQ9uTmXoqIVvdz8Id38fQrS6nSPvz17TQq3tUG76AQLl1IJuNCCmWCwynlXQb/8Fs4sGg6l9Mv4B9+Mwl/7+K/n36g/vC3nZ2KGfdSHtTv2p9Nn0zHJ7Acnt5lWPfeeGo160hAcBhp55JIP5dM2bAqBIaEUz6iBmvnjKHlI8/gVcafQ7s28+/2jfSZYMyKEVCxEiumv4S7hwc1727Hib07+X31t/Sb9IlD8/Lw8GDosBGMHvUyFYKD8fPz49mnR9Ljvl5ERESQkJBAYkIC1WvUAGDY8JFEdWxHvXoNuLtJU2ZMfxP/gAC69zBGGPfp159JE8YybMjjDBo8lLVrVrFp4wa2bHNsj4DkVXzy8vDw4OFBQ5kybjTlygdTxteX1196lqh7elKpcgRJiQkkJSZS9abqAAx4/An694zijrvqU7/h3bw/ewZ+/v507todgMoRVXh2+CA8PTzoEHUPv/z0I199Op+vvl9zlShchys+7qpE3Adpuo+mh9Z6qem1L3AOaK213pSrXmPg8zybv6O1fifPvvCPsGouWwAqN+tOePPuOTPpnNqxmsxL6VS4vRm33v8c7p5e/LtiHodWfpRzf+Xl9FT++X4Ocb9tIjM9Fd+wm6jR5XHK33Z3vv3/u2Ie8b9vccp9kGCaSWfhLPZt/IHLF9Op3iiS9kNH4VHaix8/n8W2L+bw0ooDACScPMqGD6dw8sAeMjMyqFClJk37DqN6g5Y5+9u74Qe2fTmHC4mnCal5Oy0feorKt9Urcl5FuQ8yO69xb7zOl18sJC01lc5RXZk+cxbe3t6MHzuGCePeIC3jyt/HksWLmDppAkePHqFho8bMmvMeVatVy1m/f98+nnlqBL/t2U3Nm2/h9TfG0b5DxyLnVVSSl+PzKsp9kNk5TZ88lu++/ZL0tDTadOjM2Mkz8PL2ZsaUcbw9dQLHct0fuOKHJcyePoX/jh3lrvoNmTBtFhFVruS05JsveOeticTHxnJH3fo898rrNGx85dazLz75kC8+sW4Ktz9/350do0PugzyVVLSBRKGBnoB94ryhGshC7KvQEwUUB0VtIF1dURtIIWylqA2kK3P0RAGnkovYQAbYr4EsSV2sqoCfhRBCuDhX/NAuEYN0TJrn+rllgbWEEEK4HBvPxWoTJekMcohSagtwCnjT2cEIIYSwnjWDdNasWs7aVcuvWc9WSlID+SUwDeM5YB8Dtzo3HCGEELbUsXNXOnbualb22YKPCqh9/UpSA7lCa/14rteWn8ckhBDC9bjgRciS1EAKIYQoplywfZQGUgghhPPJXKx2Yu/7dIQQQtiXK86kU5Ju8xBCCCFspkScQdrDvi8m5SurcHtzgm9v4YRohBDCfqJXr3DoUzIskS7WYqR2v5edHYIQQjhEu05daNepi1nZl59aN2erI61euYzVKx3XkEsDKYQQwumsOYPs3KUbnbt0Myv79OMP7RSRNJBCCCFcgAzSEfmc2LrU2SHYxW+rvnZ2CHbx0bwPnB2CXZTEvEpiTmA8sko4hjSQTvbftqXODsEuflv9jbNDsIv5H5bMD92SmFdJzAmw+nmOxY1MVi6EEEJY4HodrNJACiGEcAUu2EJKAymEEKJYWLViGatWyOOuhBBC3ECsGcUa1eUeorrcY1a2YL79Bi3JIJ1CiP/zR7vUdfb7H9y+wab1Cqsw+7W27orly6zeZ2HqFoa1+7VXrPbIyxViLU55Ra+27qZ2a+sVVmH2a68YrOWKg3SkgSzAvi8m5VuOb15k9fan/9xq85gKs8/C1P13+0ab1iuswuzX2rorC/EhVpi6hWHtfu0Vqz3ycoVYi1Ne1k7fZq9p3qx+/9UrmDjmZV56ZljO4miqiIs9SRdrASxNNWdpflYhhCju2nXqQvSaFUye8W5OmcOnmnPBQTpyBimEEKJEUUp5KaUWKaV+Uko9eq3ygkgDKYQQwulUEf8rQA9gK9ACGKCUKnWNcssxaa1tlF7JoJSSfxAhhDCx9wPpbfWZmztOpdRbwFKt9Val1EJgtNb6cEHlBe1TziCFEEKUNAFAjOnnGKDsNcotkkE6edj725IQQogr7PSZmwxUBg4D4UDSNcotkjNIIYQQJc0uoL5Syh2oBBy/RrlFcg1SCCFEiaKU8gIWYjSC7wMRwCKMM8eccq31gqvuRxpIIYQQIj/pYhVCCCEskAZSCCGEsEAaSOFUStl7umEhhCgaaSCFwymlBiulZgJorXVxbySVUg8opdo4Ow5hrrj/XgnnkwZSOJRSygeoCnRWSo2D4ttIKkMQ8DbwolKquZNDKhSlVGulVAdnx2Evpt+rEnOvt1IqItfP5Z0Zy41CGkjhUFrrVOAd4BPgXqXUZFN5sWsktSEBaAuEAC8ppVo5OaxrMjXsvsBMINLJ4dic6Yz+AdNQ/xeVUuHOjul6mXIYqpRqppQaBfRVSsnnt53JP7CLUkrVU0o9qJR6Xil1t7PjsQXTB7O71joWWAmsAgYppV6C4tdImvLx0Fr/BTwC1ARGuvqZpKlhPw+8CzyilLrZ2THZilLKG/gbqA3sBby11iecG5VNnAZ+ACYAD2utZ2mts5RS/k6Oq0STBtIFKaV6AOuAwcBAYIZSaqJzo7p+pg/mTKVUd2AG0AzwxTjzKpbdrVrrDKXUPUBvwBPoCYxTSjV1bmQFy3XmsQk4CdQzlbs7KyZb0VqnYcyxeS9wAtgMOde9azoztuuhtU4HHgUCgUVKqRqmz4mRSilPpwZXgkkD6WKUUrWAN4HxWuuWQF+gEZCklKrg1OBsQClVD3gPWAI8DNQBPgfuV0qNheLTSJrijMSYmSMeeBboDtwGjFVKNXNacBYopW5VSlXRWmcBaK0PAP8CL5teZzozPlswNf7DgW0YDcrtSqntQBet9UGnBncdlFKVgUtAZ+Aj4C1gGrBSa33JmbGVaFprWVxoAaKA3aafq2HMFfgREAq8CFRxdozXmd8Q4E8gKFdZOEajmQy84OwYC5nPVGA14JGr7FbgFLARaO7sGE0x3Q78jnHG+DTQ0lReB9gD3Gt6rZwdqw1yLZ/r5zXAp7leF8v8AMWVmc+exOg+rmN67ebs+ErqImeQrqc0EK+UqobxLThaa/0YcBGjgWzpzOBswAPwwfiDx3RN8gQwG3AHXlFKjXdifFYxXX9UQC2MD64MU3lpbVyTfAqjC/l/rjBwR2v9J0aX/ZsYX1I+VEp9i9FA+gONTfWK/dyTWuszAEqp4RhfXB42vXYvrvlpE6VUOeBuYKDWeq9Syk2begSE7UkD6QKUUu65rv8cAtqY/r8UGGQq98WYaPekwwMsouxuUlN+2b9rf2KcGfcEs249N4wvBIOBDxwcqlVyd/tmf2ABa4GG2d2pWuuLpipewD7gAvCPo2PNLTturfV2rfXbGN10QzAmcB6AcTxGuvJ104JcYyTnUq11G1M9d12MupCz87JwqSEBGKS13iGNo/2VmHuEiiulVDugDxCglHpDa/2HUuoxjEbiFBBmajyfBoJx8oettZRSyvSNtw3QA6iklHpJa71ZKfUGMNt0j9pKIBPjemQpYJ3WOtF5kVuWK5/GGNeEL2FcR12C0eC8rpQaq40nlftidGkuAyZpY+CI0+Q9a9JaHwWOmhrExsAvGF/EnlBKpWmt9zg+ysLL3UAopW4FKgD/AfFa6/Na65O56hWbxhFAGyNU6wE9lVJTtdYppnINpGbXcWaMNwJ5mocTKaU6Ad8CBzGuMXoD7bTWu5RST2CM9IwDzmF0P/bVWv/mpHALzTRa9XOM0ZIRGB9gnbTWv5lu7RiPMbglBaNrubvW+nfnRFuwXI1jD2AecAbjeOzBGGVcG2OgS1tgP6Axzsrambo2XU7esw+l1AMY3cIxwGSt9a9OC84K2cfE9PN4jFHEQRhfKrcA07XW/zoxxOtm+vtZAkzG+KJ1zrkR3YCcfRH0Rl6Ax4HngbIYH6hfYzzhuoFp/W1AV6A9EOrseAuZ2+0YH7bPm17XAbKAY0BdU1kDjLPn+4FwZ8d8jXyaYHxReQ7ji8wLQBrwJRCA8eH8EMZgo5eBms6O2cq83HL93B+j0b/T2XEVIv4nTL9n95lef4LxBeZuwNPZ8RUyl3wDiDB6XzKBKYCvs2O80RY5g3QCpVR9jOuLv2M8tHOZqbwixiwznTHOPnY4LcjrpJTqjHEjeleMM+QpGA1kONAKo0s1QWu902lBFoJS6mWMEcZtMQYYvY/R5e2FcRP3i0CcdnJ3alHkORsL0cZEDi7NdG3ODeMLyu9a6wlKqSiMh+KOwLjWXVZrvdaJYRaaUqoSkK61PpurrBfGl+e3gAna1N0q7E8G6TjHaWAXxgeuH+R0ecVhDOH+AfhFKVXXeSFet0DgLEb3aSvgMkaj8jJGV+sXwHdKqXLF4Z5HjEFS5TC6T7tiXOsaAKzAOAPeDixUSvkWk3xyaG1232mcU4Oxkr7yzb4csE0p1RZYDLyktZ4PtAYmmWbWcXlKKTelVFmMMQZjTaNVAdBaLwL6YfQ2jVDG/L/CAWSQjhNorY+brjGWAt5RSu3Rxq0BaK3jlFLPY9zWkerMOK1hGkCUZfqQDcIYVh+ntf5SKXUc4wN3FsbZ8kGgA8a34XnAodzflF2BaeBQpikfL23MYALGdaCtGLeoPABs1lrHK6USMAbjLAN+1MYUbsVOdoOTq+FxedqYleksxkQNQcBIrfWHptWXgUSMwVQuTxvXgxNNnwsfAheUUm9qrU+bqqzEmNRhPFDaNKBPBunYmXSxOohpRNqdGN94/9Rar1FKBWAM0rkL48btA7nqu/SwdKVUt+yuYdPrrsAkjOnWtmmtB5rKb8e4Yf42U4MyGyijtX7UGXEXRClVT2u9O9frDsAzGIOIVmqtvzaV3wMsAG7SWicppd4H/tNaj3NC2DesXAOnqmIMBAvRWldXSpXG6JX5HjiotR7gxDCtlutWHK2U6o3RdTwFeNvUs4RS6ktMPTFa661OC/YGIg2kA5hGo83DOIu6BWNwxy6tdT9TI/kNxiCWDlrrfU4L1EpKqeoYZ4Nfaq37m7qCN2J8KF3AmB5vtda6r6n+LqAGxgjPW4BWWuu9TgneAqVUF4zrO7O11nOUUk2ADcDPGNdM4zBmY/lAKXUHxrGMwbim2gxooYvxNGbFmemMvzUwF6NH5j+MEcalgCZa68u5r7G6sjyNZB+MyxBrgOyxCE8DDbXW/8g9kI4hDaSdKeNJCSuBORgjHD2BjhhnWzu01n2VUsEYw7mDgdraNCuLqzLdxByFcSb1HfAxRuM+RinlgTHqdiGwVmvdRyl1E8a1VTC+/f7lhLALZDpGr2A8jeMLjA/XUlrracqYA3MCUAX4RGv9sVLqdYyG0R14WrvorRw3EtP1u+EYA6gSgPdMXbCltNaXnRud9fI0kvdh9DABfAZ8qLX+URpHx5EG0s5M9zrOxBiVesJU5o3xtIE3gMe11ltMo9fcdDF5NI/pDzkKoyEsDczTWj9lWueJca1xIcZsJgNM5S73YZWrq+4m4FWM221CgLFa669MdaphHKubgHe01t+Yyn208XxL4YJc/TJFQfI0kt0wemZmAv9ztb+fkk5GsdpfFsbAjkDI+UBOw+jCq4Axlyda65PFoXHM/ceLMUl3f4xRuXVMZ5Zo4+kCazBG3j2slJpv2twlP6xMx+QwxsTjR4CqQP3s9VrrI8BrGN3Ko5RSA0yrit0tHTeS4tg4wpWBUqbfy2UYlyxGAG+aepuEg0gDaQdKqQBlPM0cjA9Vb2CAUso717WQTOAvjMbF5SnTfLGmb7XVlVI1gIpa65UYXVsNMLpcAeM5iRjPtOyMMdjA5UZI5ro2FWA6G/wbGIPRzRqllBqWXVdrfQwYB/yEcb3V5fIRJUeeRvJrjAdyP4VxGUA4iHSx2pjpBvlnMWZXma21/lQpdS/GNcZZwFcYs+UMxrhdoKnW+riTwr0mZcwLe1hrvdH0ujvGgAh3jAZ+lOm6SFdM9zZqrR9xVrzWytW1mn28AjG6Tz9TSkUAr2PMZPSp1npuru1crptYlFx5ultrymAwx5IG0oaUUi0wBq3swJg+riHwnNZ6pulawvsYZ+2JGGeQD2oXnltVKRWO8SzKMIzJrP/EeA7d5xjdiw0wBha9lKuR/ATjHsGezonaehaOVwOMqfFmKqWqAKOBmzEa/enOi1TcyHJ9mcv+vwzScRBpIG1EKeWDcc0tFHhTa52ulHoNY3DHM6YP3TCM61sK4x6teKcFbCWlVEuMLtQaGI1fdYxGJMPUwDyFkXN2I9kTeBu4W2sd46Swr+kax+tprfU7pjPJtzDOLh/QWic5K14hhOPJTDo2oJSqgPH8xj3AV9o0+4rWepxp4MoMpVQmMF9r/ZMTQ7Va9ghA0wjbDIx7sF4B9mTfhpI95BxjAMF4ZczusUQptVa78IwyVhyvt5VSWVrr2Uqp54DL0jgKceORBtIGtNanlVL3A6uA/UopP216NI3W+g1T4/gOcFEp9WExGdyR/Zy9cIyb4rdiPM2in1KqpdZ6C4A2nu+oMW6ReEEp9bMrN45g/fFSSmVord93arBCCKeRLlYbUsbDj1djXK/7Wud6soNS6gVgmavdJG9Jrmsd92I8k1JjdAsvwbgXsDowXOea7kop1Qw4prX+zxkxF0VJOV5CCPuQBtLGTBMDfA8MIc+HbnGilGqE8VSRt4G/MSZO347pJnqM4eYjdTGfE7KkHC8hhO1JF6uNaa1Xm868vgW8lFIL9JUnQhQn9THmTn0n92wxSqlQjJG4P2E83qm31nq7k2K8biXoeAkhbEwmCrADrXX2DDNjMKZhKzay77vCmKQ7Zyo1ZUwKDcaZY3OMeyC3UEwmOria4ny8hBD2I12sdqSU8nX1ASsFUUq1wZgu7kHTTB7Z5b0w7g9sA6SYppUrEYrz8RJC2J6cQdpRMf+w3QrMxrh94wHIeWJCC4xJDrJKUuMIxf54CSFsTM4gRYFMEyOPAoZyZaBOKNBNa/27M2MTQgh7kwZSXJUyntB+N8Z1x5MY08gdcW5UQghhf9JACiGEEBbINUghhBDCAmkghRBCCAukgRRCCCEskAZSCCGEsEAaSCGEEMICaSCFEEIIC6SBFEIIISyQBlII4XKUUguUUjrPEqeU+kYpVcUO7zdGKfWb6ecBpverauv3ucr7b1JKvV3AugFKqaRC7OuoUurp64xHK6W6X88+SgJpIIUQrmoHxtNjagK1gGFAE2CpUsrdju+7xPSeJ69VUSnVVSkls62UUPI8SCGEq0rTWv+b6/XfSil/YD5wE3DQHm+qtU4BUuyxb1G8yBmkEKI4OWf6vxfkdAV2UkqtVkrtNpWVVkpNUUodUkpdUEptUUo1z96Baf2bSqljSql4pdRHgE+u9ZGm/QaaXpc1dfmeUkolKKW+UkpVVEpFAstyxTHA9HOYUuprU5dwgqlbuHKu/YcqpRYppc4qpf5VSj0PZD+H9ZqUUrcopVYppRKVUqlKqd1Kqa55qnkrpT5RSp1RSh1RSk1USnnm2cdKU3zxSql52fmKK6SBFEK4PGWoCbwMHAMO5Fo9B1gJPGR6PQ9oh9El2wr4E1ivlLrVtH4aMAh4DegCaOCZgt4XWI7RxdsHeAy4E+MsdjswxFS1JrDENLn/FoxHwt0L9AQqABuUUl6mruE1wC1AX9P+7gOaFuKf4xMgyLR9J2Af8K3pvbO9BiSY1k8FngbGm3IqD/yC8YSeDsAjQCPg+1wPTBdIF6sQwnW1VEqlm352x/i8+hvopbXOyFVvkdb6HQDTwJoHgWpa62Omsl+BekAPpdQJjMe3DdVaf5prfeOCYsB4mk11rfVRU/3zwKvAZSAGILsrWCn1iGm7h7TWmaay3cBpUwx+QB3gVq3136b1vYDCPCHnW2C11nqfaftMU86hwP/bu58Qq8o4jOPfB0xEBMVNCOafYAYXWcQUglOOIBbNuHEb1V6igqCpQHQjJDguCjQINzlQCbWYQBRnxlxEbQYqGNIZagiGQaE/EDRKCPfn4veeOp0516bdMD2fzdwz973vee9d3Oe87/s73J9Km6mIqEJ/StJG4Jikt4GXgW9qzyNpjrzo2FK9J3NAmtnKNQW8VDv+HbgVS3+C6Nva40fI5cqZxmRoLRkAPWTYTlZPRERH0jXyx8CbdgPzVTiW9uPAOEDLhGs3uT+62HL+HnLmN1+FY+lvQdIMy/cusEfSEeBxcpbY9EXj+ArwDrCtjHGgdvEBfy/x9uCA/IsD0sxWqtsRcePfm3Gn9ngN0AH6yGXOukVgU5c+Ol3+/8B9nmuzhqXBXvkZeP4/nv8fJK0DLgNbyWrbi+XvpUbT5kVEtZ32Zxnj5+RydZPDscZ7kGa2mlwnv9c2RMSNErA/kvuRu4A54C5woHpB2Xfb16W/74FtjSKbvZKmJW3u0v5hcpZYnf828BZZCHQdeEhSb62/B8vYlmMfOdN9LCKGI2KM9gKf/Y3jZ4BfgZtljL3AbG2M64E3WXpR8b/mgDSzVaMsXY4B5yUNSeoHPiALUWYjYhF4Hzgl6UVJT5BFPju7dDkBTAMXJA1IGgTeA36JiN8ogSKpr9yC8hE5S7tQgvQ54BPgUfK+yqvAd8Cnkg5KeprcU7zTPHEXi+T39quSnpT0Whk/QL+kalWwT9JIGdcR4ChwsixPnwF2AOdKH4eBj4FN5fOxwgFpZqvNC+Qe4VmyYnQHcLAq2gHeIEPzBLlcuR4YbuuoFNo8C8yTQTYK/EBWkEJWrH4NfAkMRsQf5AyvQ94CMkoW4AxFRCciOuSe4QwZSh+Sy6SfLfO9fQUcJ6tSx8vYDpWxjQAbSrtXgO2lzevlNafLe1oAniqfy0T5nCbJiwir0dL9bjMzM/MM0szMrIUD0szMrIUD0szMrIUD0szMrIUD0szMrIUD0szMrIUD0szMrIUD0szMrMU9UaKWRlHrBHcAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "batch_size = 1000\n", + "conf_matrix_ssl = evaluate_mlpf(mlpf, with_VICReg=True)\n", + "plot_conf_matrix(conf_matrix_ssl, \"ssl MLPF\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.3" + }, + "vscode": { + "interpreter": { + "hash": "bd710be4164d8116e60481776a482d6ed163c0c31d42101b2cd55e4bfc6d2c5e" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}