diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..c6ac140 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,134 @@ +version: 2 + +images: + python: &python + - image: circleci/buildpack-deps:stretch-browsers + +############################################################################### +utils: + prepare_container: &prepare_container + name: Prepare build container + command: | + sudo apt-get update + sudo apt-get install curl pandoc + sudo apt-get clean + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh + bash miniconda.sh -b -p /home/circleci/miniconda + sudo rm -rf ~/.pyenv/ /opt/circleci/.pyenv/ + source /home/circleci/miniconda/etc/profile.d/conda.sh + conda create --name=causalnex_env python=${PYTHON_VERSION} -y + conda activate causalnex_env + conda install -y virtualenv + pip install -U pip setuptools wheel + activate_conda: &activate_conda + name: Activate conda environment + command: | + echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV + echo "conda deactivate; conda activate causalnex_env" >> $BASH_ENV + + setup_requirements: &setup_requirements + name: Install PIP dependencies + command: | + echo "Python version: $(python --version 2>&1)" + pip install -r requirements.txt -U + pip install -r test_requirements.txt -U + conda install -y virtualenv + setup_pre_commit: &setup_pre_commit + name: Install pre-commit hooks + command: | + pre-commit install --install-hooks + pre-commit install --hook-type pre-push + linters: &linters + name: Run pylint and flake8 + command: make lint + + unit_tests: &unit_tests + name: Run tests + command: make test + + build_docs: &build_docs + # NOTE: doesn't work on python 3.5 + name: Build documentation + command: make build-docs + + install_package: &install_package + name: Install the package + command: make install + + unit_test_steps: &unit_test_steps + steps: + - checkout + - run: *prepare_container + - run: *activate_conda + - run: *setup_requirements + - run: *unit_tests + +############################################################################### +jobs: + unit_tests_35: + docker: *python + environment: + PYTHON_VERSION: '3.5' + <<: *unit_test_steps + + unit_tests_36: + docker: *python + environment: + PYTHON_VERSION: '3.6' + <<: *unit_test_steps + + unit_tests_37: + environment: + PYTHON_VERSION: '3.7' + docker: *python + <<: *unit_test_steps + + linters_37: + docker: *python + environment: + PYTHON_VERSION: '3.7' + steps: + - checkout + - run: *prepare_container + - run: *activate_conda + - run: *setup_requirements + - run: *setup_pre_commit + - run: *linters + - run: *install_package + + docs: + docker: *python + environment: + PYTHON_VERSION: '3.7' + steps: + - checkout + - run: *prepare_container + - run: *activate_conda + - run: *setup_requirements + - run: *build_docs + + all_circleci_checks_succeeded: + docker: + - image: circleci/python # any light-weight image + steps: + - run: + name: Success! + command: echo "All checks passed" + +############################################################################### +workflows: + version: 2 + regular: + jobs: + - unit_tests_35 + - unit_tests_36 + - unit_tests_37 + - linters_37 + - docs + - all_circleci_checks_succeeded: + requires: + - unit_tests_35 + - unit_tests_36 + - unit_tests_37 + - linters_37 + - docs diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..5a0426a --- /dev/null +++ b/.coveragerc @@ -0,0 +1,6 @@ +[report] +fail_under=100 +show_missing=True +exclude_lines = + pragma: no cover + raise NotImplementedError diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..8783059 --- /dev/null +++ b/.flake8 @@ -0,0 +1,8 @@ +# copied from black + +[flake8] +ignore = E203, E266, E501, W503 +exclude = causalnex/bbn +max-line-length = 80 +max-complexity = 18 +select = B,C,E,F,W,T4,B9 diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 0000000..ba480fe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,40 @@ +--- +name: Bug report +about: If something isn't working +title: '' +labels: 'Issue: Bug Report' +assignees: '' + +--- + +## Description +Short description of the problem here. + +## Context +How has this bug affected you? What were you trying to accomplish? + +## Steps to Reproduce +1. [First Step] +2. [Second Step] +3. [And so on...] + +## Expected Result +Tell us what should happen. + +## Actual Result +Tell us what happens instead. + +``` +-- If you received an error, place it here. +``` + +``` +-- Separate them if you have more than one. +``` + +## Your Environment +Include as many relevant details about the environment in which you experienced the bug: + +* CausalNex version used (`pip show causalnex`): +* Python version used (`python -V`): +* Operating system and version: diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..a7911c2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Let us know if you have a feature request or enhancement +title: '<Title>' +labels: 'Issue: Feature Request' +assignees: '' + +--- + +## Description +Is your feature request related to a problem? A clear and concise description of what the problem is: "I'm always frustrated when ..." + +## Context +Why is this change important to you? How would you use it? How can it benefit other users? + +## Possible Implementation +(Optional) Suggest an idea for implementing the addition or change. + +## Possible Alternatives +(Optional) Describe any alternative solutions or features you've considered. diff --git a/.github/ISSUE_TEMPLATE/thank-you.md b/.github/ISSUE_TEMPLATE/thank-you.md new file mode 100644 index 0000000..b959a8c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/thank-you.md @@ -0,0 +1,21 @@ +--- +name: Say thank you +about: Tell us how you use CausalNex and help us grow a community +title: '<Title>' +labels: 'Issue: Thank You' +assignees: '' + +--- + +## Let us know +If you (or your company) are using CausalNex - please let us know. We'd love to hear from you! + +## Making CausalNex even better +If you would like to help CausalNex - any of the following is greatly appreciated. + +- [ ] Give the repository a star +- [ ] Help out with issues +- [ ] Review pull requests +- [ ] Blog about CausalNex +- [ ] Make tutorials +- [ ] Give talks diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..40e82b4 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,22 @@ +## Notice + +- [ ] I acknowledge and agree that, by checking this box and clicking "Submit Pull Request": + +- I submit this contribution under the [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0.txt) and represent that I am entitled to do so on behalf of myself, my employer, or relevant third parties, as applicable. +- I certify that (a) this contribution is my original creation and / or (b) to the extent it is not my original creation, I am authorised to submit this contribution on behalf of the original creator(s) or their licensees. +- I certify that the use of this contribution as authorised by the Apache 2.0 license does not violate the intellectual property rights of anyone else. + +## Motivation and Context +Why was this PR created? + +## How has this been tested? +What testing strategies have you used? + +## Checklist + +- [ ] Read the [contributing](/CONTRIBUTING.md) guidelines +- [ ] Opened this PR as a 'Draft Pull Request' if it is work-in-progress +- [ ] Updated the documentation to reflect the code changes +- [ ] Added a description of this change and added my name to the list of supporting contributions in the [`RELEASE.md`](/RELEASE.md) file +- [ ] Added tests to cover my changes +- [ ] Assigned myself to the PR diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2890d45 --- /dev/null +++ b/.gitignore @@ -0,0 +1,132 @@ +########################## +# Common files + +# IntelliJ +.idea/ +*.iml +out/ +.idea_modules/ + +### macOS +*.DS_Store +.AppleDouble +.LSOverride +.Trashes + +# Vim +*~ +.*.swo +.*.swp + +# emacs +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc + +# vscode +.vscode/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# C extensions +*.so + +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +.static_storage/ +.media/ +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/tmp-build-artifacts +docs/build + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..d8abb4f --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,9 @@ +# copied from black +[settings] +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=88 +known_first_party=causalnex,tests +default_section=THIRDPARTY diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4100c8c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,78 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks + +default_stages: [commit, manual] + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.2.3 + hooks: + - id: trailing-whitespace + stages: [commit, manual] + - id: end-of-file-fixer + stages: [commit, manual] + - id: check-yaml # Checks yaml files for parseable syntax. +# exclude: + - id: check-json # Checks json files for parseable syntax. + - id: check-added-large-files + - id: check-case-conflict # Check for files that would conflict in case-insensitive filesystems + - id: check-merge-conflict # Check for files that contain merge conflict strings. + - id: debug-statements # Check for debugger imports and py37+ `breakpoint()` calls in python source. +# exclude: + - id: detect-private-key # Detects the presence of private keys + - id: requirements-txt-fixer # Sorts entries in requirements.txt + - id: flake8 + exclude: ^causalnex/ebaybbn + +- repo: https://github.com/pre-commit/mirrors-isort + rev: v4.3.21 + hooks: + - id: isort + exclude: ^causalnex/ebaybbn + +- repo: local + hooks: + # It's impossible to specify per-directory configuration, so we just run it many times. + # https://github.com/PyCQA/pylint/issues/618 + # The first set of pylint checks if for local pre-commit, it only runs on the files changed. + - id: pylint-quick-causalnex + name: "Quick PyLint on causalnex/*" + language: system + types: [file, python] + files: ^causalnex/ + exclude: ^causalnex/ebaybbn + entry: pylint -j0 --disable=unnecessary-pass,cyclic-import --ignore=ebaybbn + stages: [commit] + - id: pylint-quick-tests + name: "Quick PyLint on tests/*" + language: system + types: [file, python] + files: ^tests/ + entry: pylint -j0 --disable=missing-docstring,redefined-outer-name,duplicate-code,no-self-use,invalid-name,cyclic-import + stages: [commit] + + # The same pylint checks, but running on all files. It's for manual run with `make lint` + - id: pylint-causalnex + name: "PyLint on causalnex/*" + language: system + pass_filenames: false + stages: [manual] + entry: pylint -j0 --disable=unnecessary-pass,cyclic-import --ignore=ebaybbn causalnex + exclude: ^causalnex/ebaybbn + - id: pylint-tests + name: "PyLint on tests/*" + language: system + pass_filenames: false + stages: [manual] + entry: pylint -j0 --disable=missing-docstring,redefined-outer-name,duplicate-code,no-self-use,invalid-name,cyclic-import tests + # We need to make some exceptions for 3.5, that's why it's a custom runner. + - id: black + name: "Black" + language: system + pass_filenames: false + entry: python -m tools.min_version 3.6 "pip install black" "black causalnex tests" + - id: legal + name: "Licence check" + language: system + pass_filenames: false + entry: make legal diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..bfb1093 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,425 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist=numpy + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +# ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins=pylint.extensions.docparams + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=ungrouped-imports,bad-continuation,c-extension-no-member + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=useless-suppression + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio).You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[BASIC] + +# Naming hint for argument names +argument-name-hint=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + +# Regular expression matching correct argument names +argument-rgx=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + +# Naming hint for attribute names +attr-name-hint=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + +# Regular expression matching correct attribute names +attr-rgx=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-zX_][A-Za-zX0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-zX_][A-Za-zX0-9_]{2,30}|(__.*__))$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zXA-Z0-9]+$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zXA-Z0-9]+$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming hint for function names +function-name-hint=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + +# Regular expression matching correct function names +function-rgx=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + +# Good variable names which should always be accepted, separated by a comma +good-names=ex,Run,_,io,df,ds,bn,sm,ax,X,W,E,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-zX_][A-Za-zX0-9_]*$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-zX_][A-Za-zX0-9_]*$ + +# Naming hint for method names +method-name-hint=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + +# Regular expression matching correct method names +method-rgx=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + +# Naming hint for module names +module-name-hint=(([a-zX_][a-zX0-9_]*)|([A-Z][a-zXA-Z0-9]+))$ + +# Regular expression matching correct module names +module-rgx=(([a-zX_][a-zX0-9_]*)|([A-Z][a-zXA-Z0-9]+))$ + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming hint for variable names +variable-name-hint=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + +# Regular expression matching correct variable names +variable-rgx=(([a-zX][a-zX0-9_]{2,30})|(_[a-zX0-9_]*))$ + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )?<?https?://\S+>?$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=20 + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zXA-Z0-9_]*[a-zXA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp,fit,_init + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=8 + +# Maximum number of attributes for a class (see R0902). +max-attributes=10 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=20 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=1 + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..25d1e46 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,31 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: html + configuration: docs/conf.py + fail_on_warning: true + +# Build documentation with MkDocs +# mkdocs: +# configuration: mkdocs.yml + +# Optionally build your docs in additional formats such as PDF and ePub +#formats: all + +# Optionally set the version of Python and requirements required to build your docs +python: + version: 3.6 + install: + - method: pip + path: . + extra_requirements: + - docs + - requirements: + - test_requirements.txt + - doc_requirements.txt diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..01e82f6 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,78 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behaviour that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behaviour by participants include: + +* The use of sexualised language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behaviour and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behaviour. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviours that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behaviour may be +reported by contacting the project team at causalnex@quantumblack.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +**Investigation Timeline:** The project team will make all reasonable efforts to initiate and conclude the investigation in a timely fashion. Depending on the type of investigation the steps and timeline for each investigation will vary. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8b175ca --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,174 @@ +# Introduction + +Thank you for considering contributing to CausalNex! CausalNex would not be possible without the generous sharing from leading researchers in causal inference and we hope to maintain the spirit of open discourse by welcoming contributions in the form of pull requests (PRs), issues or code reviews. You can add to code, [documentation](https://causalnex.readthedocs.io), or simply send us spelling and grammar fixes or extra tests. Contribute anything that you think improves the community for us all! + +The following sections describe our vision and contribution process. + +## Vision + +Identifying causation from data remains a field of active research and CausalNex aims to become the leading library for causal reasoning and counterfactual analysis using Bayesian Networks. + +## Code of conduct + +The CausalNex team pledges to foster and maintain a welcoming and friendly community in all of our spaces. All members of our community are expected to follow our [Code of Conduct](/CODE_OF_CONDUCT.md) and we will do our best to enforce those principles and build a happy environment where everyone is treated with respect and dignity. + +# Get started + +We use [GitHub Issues](https://github.com/quantumblacklabs/causalnex/issues) to keep track of known bugs. We keep a close eye on them and try to make it clear when we have an internal fix in progress. Before reporting a new issue, please do your best to ensure your problem hasn't already been reported. If so, it's often better to just leave a comment on an existing issue, rather than create a new one. Old issues also can often include helpful tips and solutions to common problems. + +If you are looking for help with your code in our documentation haven't helped you, please consider posting a question on [Stack Overflow](https://stackoverflow.com/questions/tagged/causalnex). If you tag it `causalnex` and `python`, more people will see it and may be able to help. We are unable to provide individual support via email. In the interest of community engagement we also believe that help is much more valuable if it's shared publicly, so that more people can benefit from it. + +If you're over on Stack Overflow and want to boost your points, take a look at the `causalnex` tag and see if you can help others out by sharing your knowledge. It's another great way to contribute. + +If you have already checked the existing issues in [GitHub issues](https://github.com/quantumblacklabs/causalnex/issues) and are still convinced that you have found odd or erroneous behaviour then please file an [issue](https://github.com/quantumblacklabs/causalnex). We have a template that helps you provide the necessary information we'll need in order to address your query. + +## Feature requests + +### Suggest a new feature + +If you have new ideas for CausalNex functionality then please open a [GitHub issue](https://github.com/quantumblacklabs/causalnex/issues) with the label `Type: Enhancement`. You can submit an issue [here](https://github.com/quantumblacklabs/causalnex/issues) which describes the feature you would like to see, why you need it, and how it should work. + +### Contribute a new feature + +If you're unsure where to begin contributing to CausalNex, please start by looking through the `good first issues` on [GitHub](https://github.com/quantumblacklabs/causalnex/issues). + +We focus on two areas for contribution: `core` and [`contrib`](/causalnex/contrib/): +- `core` refers to the primary CausalNex library +- [`contrib`](/causalNex/contrib/) refers to features that could be added to `core` that do not introduce too many dependencies e.g. adding a new type of causal network to network module. + +Typically, we only accept small contributions for the `core` CausalNex library but accept new features as `plugins` or additions to the [`contrib`](/causalnex/contrib/) module. We regularly review [`contrib`](/causalnex/contrib/) and may migrate modules to `core` if they prove to be essential for the functioning of the framework or if we believe that they are used by most projects. + +## Your first contribution + +Working on your first pull request? You can learn how from these resources: +* [First timers only](https://www.firsttimersonly.com/) +* [How to contribute to an open source project on GitHub](https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github) + + +### Guidelines + + - Aim for cross-platform compatibility on Windows, macOS and Linux + - We use [Anaconda](https://www.anaconda.com/distribution/) as a preferred virtual environment + - We use [SemVer](https://semver.org/) for versioning + +Our code is designed to be compatible with Python 3.5 onwards and our style guidelines are (in cascading order): + +* [PEP 8 conventions](https://www.python.org/dev/peps/pep-0008/) for all Python code +* [Google docstrings](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) for code comments +* [PEP 484 type hints](https://www.python.org/dev/peps/pep-0484/) for all user-facing functions / class methods e.g. + +``` +def count_truthy(elements: List[Any]) -> int: + return sum(1 for elem in elements if element) +``` + +> *Note:* We only accept contributions under the Apache 2.0 license and you should have permission to share the submitted code. + +Please note that each code file should have a licence header, include the content of [`legal_header.txt`](https://github.com/quantumblacklabs/causalnex/blob/master/legal_header.txt). +There is an automated check to verify that it exists. The check will highlight any issues and suggest a solution. + +### Branching conventions +We use a branching model that helps us keep track of branches in a logical, consistent way. All branches should have the hyphen-separated convention of: `<type-of-change>/<short-description-of-change>` e.g. `contrib/structure` + +| Types of changes | Description | +| ---------------- | ---------------------------------------------------------------------------- | +| `contrib` | Changes under `contrib/` and has no side-effects to other `contrib/` modules | +| `docs` | Changes to the documentation under `docs/source/` | +| `feature` | Non-breaking change which adds functionality | +| `fix` | Non-breaking change which fixes an issue | +| `tests` | Changes to project unit `tests/` and / or integration `features/` tests | + +## `core` contribution process + +Small contributions are accepted for the `core` library: + + 1. Fork the project by clicking **Fork** in the top-right corner of the [CausalNex GitHub repository](https://github.com/quantumblacklabs/causalnex) and then choosing the target account the repository will be forked to. + 2. Create a feature branch on your forked repository and push all your local changes to that feature branch. + 3. Before submitting a pull request (PR), please ensure that unit tests and linting are passing for your changes by running `make test` and `make lint` locally, have a look at the section [Running checks locally](/CONTRIBUTING.md#running-checks-locally) below. + 4. Open a PR against the `quantumblacklabs:develop` branch from your feature branch. + 5. Update the PR according to the reviewer's comments. + 6. Your PR will be merged by the CausalNex team once all the comments are addressed. + + > _Note:_ We will work with you to complete your contribution but we reserve the right to takeover abandoned PRs. + +## `contrib` contribution process + +You can also add new work to `contrib`: + + 1. Create an [issue](https://github.com/quantumblacklabs/causalnex/issues) describing your contribution. + 2. Fork the project by clicking **Fork** in the top-right corner of the [CausalNex GitHub repository](https://github.com/quantumblacklabs/causalnex) and then choosing the target account the repository will be forked to. + 3. Work in [`contrib`](/causalnex/contrib/) and create a feature branch on your forked repository and push all your local changes to that feature branch. + 4. Before submitting a pull request, please ensure that unit tests and linting are passing for your changes by running `make test` and `make lint` locally, have a look at the section [Running checks locally](/CONTRIBUTING.md#running-checks-locally) below. + 5. Include a `README.md` with instructions on how to use your contribution. + 6. Open a PR against the `quantumblacklabs:develop` branch from your feature branch and reference your issue in the PR description (e.g., `Resolves #<issue-number>`). + 7. Update the PR according to the reviewer's comments. + 8. Your PR will be merged by the CausalNex team once all the comments are addressed. + + > _Note:_ We will work with you to complete your contribution but we reserve the right to takeover abandoned PRs. + +## CI / CD and running checks locally +To run tests you need to install the test requirements. +Also we use [pre-commit](https://pre-commit.com) hooks for the repository to run the checks automatically. +It can all be installed using the following command: + +```bash +make install-test-requirements +make install-pre-commit +``` + +### Running checks locally + +All checks run by our CI / CD servers can be run locally on your computer. + +#### PEP-8 Standards (`pylint` and `flake8`) + +```bash +make lint +``` + +#### Unit tests, 100% coverage (`pytest`, `pytest-cov`) + +```bash +make test +``` + +> Note: We place [conftest.py](https://docs.pytest.org/en/latest/fixture.html#conftest-py-sharing-fixture-functions) files in some test directories to make fixtures reusable by any tests in that directory. If you need to see which test fixtures are available and where they come from, you can issue: + +```bash +pytest --fixtures path/to/the/test/location.py +``` + +#### Others + +Our CI / CD also checks that `causalnex` installs cleanly on a fresh Python virtual environment, a task which depends on successfully building the docs: + +```bash +make build-docs +``` + +This command will only work on Unix-like systems and requires `pandoc` to be installed. + +> ❗ Running `make build-docs` in a Python 3.5 environment may sometimes yield multiple warning messages like the following: `WARNING: toctree contains reference to nonexisting document '04_user_guide/04_user_guide'`. You can simply ignore them or switch to Python 3.6+ when building documentation. + +## Hints on pre-commit usage +The checks will automatically run on all the changed files on each commit. +Even more extensive set of checks (including the heavy set of `pylint` checks) +will run before the push. + +The pre-commit/pre-push checks can be omitted by running with `--no-verify` flag, as per below: + +```bash +git commit --no-verify <...> +git push --no-verify <...> +``` +(`-n` alias works for `git commit`, but not for `git push`) + +All checks will run during CI build, so skipping checks on push will +not allow you to merge your code with failing checks. + +You can uninstall the pre-commit hooks by running: + +```bash +make uninstall-pre-commit +``` +`pre-commit` will still be used by `make lint`, but will not install the git hooks. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..a9ec8fd --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,27 @@ +Copyright 2019-2020 QuantumBlack Visual Analytics Limited + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +(either separately or in combination, "QuantumBlack Trademarks") are +trademarks of QuantumBlack. The License does not grant you any right or +license to the QuantumBlack Trademarks. You may not use the QuantumBlack +Trademarks or any confusingly similar mark as a trademark for your product, +or use the QuantumBlack Trademarks in any other manner that might cause +confusion in the marketplace, including but not limited to in advertising, +on websites, or on software. + +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..33eb411 --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +install: + pip install . -U + +clean: + rm -rf build dist docs/build pip-wheel-metadata .mypy_cache .pytest_cache + find . -regex ".*/__pycache__" -exec rm -rf {} + + find . -regex ".*\.egg-info" -exec rm -rf {} + + pre-commit clean || true + +legal: + python tools/license_and_headers.py + +lint: + pre-commit run -a --hook-stage manual + +test: + pytest tests + +package: clean install + python setup.py sdist bdist_wheel + +SPHINXPROJ = causalnex + +install-doc-requirements: + pip install -r doc_requirements.txt -U + +build-docs: install install-doc-requirements + ./docs/build-docs.sh + +install-test-requirements: + pip install -r test_requirements.txt -U + +install-pre-commit: install-test-requirements + pre-commit install --install-hooks + +uninstall-pre-commit: + pre-commit uninstall + pre-commit uninstall --hook-type pre-push diff --git a/README.md b/README.md index a172bd4..bd91fbb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,80 @@ -# CausalNex +![CausalNex](docs/source/causalnex_banner.png) +----------------- + +| Theme | Status | +|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Latest Release | [![PyPI version](https://badge.fury.io/py/causalnex.svg)](https://pypi.org/project/causalnex/) | +| Python Version | [![Python Version](https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7-blue.svg)](https://pypi.org/project/causalnex/) | +| `master` Branch Build | [![CircleCI](https://circleci.com/gh/quantumblacklabs/causalnex/tree/master.svg?style=shield&circle-token=92ab70f03f3183655473dad16be641959cd31b83)](https://circleci.com/gh/quantumblacklabs/causalnex/tree/master) | +| `develop` Branch Build | [![CircleCI](https://circleci.com/gh/quantumblacklabs/causalnex/tree/develop.svg?style=shield&circle-token=92ab70f03f3183655473dad16be641959cd31b83)](https://circleci.com/gh/quantumblacklabs/causalnex/tree/develop) | +| Documentation Build | [![Documentation](https://readthedocs.org/projects/causalnex/badge/?version=latest)](https://causalnex.readthedocs.io/) | +| License | [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) | +| Code Style | [![Code Style: Black](https://img.shields.io/badge/code%20style-black-black.svg)](https://github.com/ambv/black) | + + +## What is CausalNex? + +> "A toolkit for causal reasoning with Bayesian Networks." + +CausalNex aims to become one of the leading library for causal reasoning and "what-if" analysis using Bayesian Networks. It helps to simplify the steps: + - To learn causal structures, + - To allow domain experts to augment the relationships, + - To estimate the effects of potential interventions using data. + +## Why CausalNex? + +CausalNex is built on our collective experience to leverage Bayesian Networks to identify causal relationships in data so that we can develop the right interventions from analytics. We developed CausalNex because: + +- We believe **leveraging Bayesian Networks** is more intuitive to describe causality compared to traditional machine learning methodology that are built on pattern recognition and correlation analysis. +- Causal relationships are more accurate if we can easily **encode or augment domain expertise** in the graph model +- We can then use the graph model to **assess the impact** from changes to underlying features, i.e. counterfactual analysis, and **identify the right intervention**. + +In our experience, a data scientist generally has to use at least 3-4 different open-source libraries before arriving at the final step of finding the right intervention. CausalNex aims to simplify this end-to-end process for causality and counterfactual analysis. + +## What are the main features of CausalNex? + +The main features of this library are: + +- Use state-of-the-art structure learning methods to understand conditional dependencies between variables +- Allow domain knowledge to augment model relationship +- Build predictive models based on structural relationships +- Fit probability distribution of the Bayesian Networks +- Evaluate model quality with standard statistical checks. +- Visualisation that simplifies how causality is understood in Bayesian Networks +- Analyse the impact of interventions using Do-calculus + +## How do I install CausalNex? + +CausalNex is a Python package. To install it, simply run: + +```bash +pip install causalnex +``` + +See more detailed installation instructions, including how to setup Python virtual environments, in our [installation guide](https://causalnex.readthedocs.io/en/latest/02_getting_started/02_install.html) and get started with our [tutorial](https://causalnex.readthedocs.io/en/latest/03_tutorial/03_tutorial.html). + +## How do I use CausalNex? + +You can find the documentation for the latest stable release [here](https://causalnex.readthedocs.io/en/latest/). It explains: + +- An end-to-end [tutorial on how to use CausalNex](https://causalnex.readthedocs.io/en/latest/03_tutorial/03_tutorial.htm) +- The [main concepts and methods](https://causalnex.readthedocs.io/en/latest/04_user_guide/04_user_guide.htm) in using Bayesian Networks for Causal Inference + +> Note: You can find the notebook and markdown files used to build the docs in [`docs/source`](docs/source). + +## Can I contribute? + +Yes! We'd love you to join us and help us build CausalNex. Check out our [contributing](CONTRIBUTING.md) documentation. + +## How do I upgrade CausalNex? + +We use [SemVer](http://semver.org/) for versioning. The best way to upgrade safely is to check our [release notes](RELEASE.md) for any notable breaking changes. + +## What licence do you use? + +See our [LICENSE](LICENSE.md) for more detail. + +## We're hiring! + +Do you want to be part of the team that builds CausalNex and [other great products](https://quantumblack.com/labs) at QuantumBlack? If so, you're in luck! QuantumBlack is currently hiring Machine Learning Engineers who love using data to drive their decisions. Take a look at [our open positions](https://www.quantumblack.com/careers/current-openings#content) and see if you're a fit. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..3601855 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,9 @@ +# Release 0.4.0: + +The initial release of CausalNex. + +## Thanks for supporting contributions +CausalNex was originally designed by [Paul Beaumont](https://www.linkedin.com/in/pbeaumont/) and [Ben Horsburgh](https://www.linkedin.com/in/benhorsburgh/) to solve challenges they faced in inferencing causality in their project work. This work was later turned into a product thanks to the following contributors: +[Yetunde Dada](https://github.com/yetudada), [Wesley Leong](https://www.linkedin.com/in/wesleyleong/), [Steve Ler](https://www.linkedin.com/in/song-lim-steve-ler-380366106/), [Viktoriia Oliinyk](https://www.linkedin.com/in/victoria-oleynik/), [Roxana Pamfil](https://www.linkedin.com/in/roxana-pamfil-1192053b/), [Nisara Sriwattanaworachai](https://www.linkedin.com/in/nisara-sriwattanaworachai-795b357/) and [Nikolaos Tsaousis](https://www.linkedin.com/in/ntsaousis/). + +CausalNex would also not be possible without the generous sharing from leading researches in the field of causal inference and we are grateful to everyone who advised and supported us, filed issues or helped resolve them, asked and answered questions or simply be part of inspiring discussions. diff --git a/causalnex/__init__.py b/causalnex/__init__.py new file mode 100644 index 0000000..a3b94b8 --- /dev/null +++ b/causalnex/__init__.py @@ -0,0 +1,35 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +causalnex toolkit for causal reasoning (Bayesian Networks / Inference) +""" + +__version__ = "0.4.0" + +__all__ = ["structure", "discretiser", "evaluation", "inference", "network", "plots"] diff --git a/causalnex/contrib/README.md b/causalnex/contrib/README.md new file mode 100644 index 0000000..4c2447f --- /dev/null +++ b/causalnex/contrib/README.md @@ -0,0 +1,39 @@ +# CausalNex contrib + +The contrib directory is meant to contain user contributions, these +contributions might get merged into core CausalNex at some point in the future. + +When create a new module in `contrib`, place it exactly where it would be if it +was merged into core CausalNex. + +For example, functions to plot network diagrams are under the core package `causalnex.plotting`. If you are +contributing a new visualisation or plot you should have the following directory: +`causalnex/contrib/my_project/plotting/` - i.e., the name of your project before the +`causalnex` package path. + +This is how a module would look like under `causalnex/contrib`: +``` +causalnex/contrib/my_project/plotting/ + my_module.py + README.md +``` + +You should put you test files in `tests/contrib/my_project`: +``` +tests/contrib/my_project + test_my_module.py +``` + +## Requirements + +If your project has any requirements that are not in the core `requirements.txt` +file. Please add them in `setup.py` like so: +``` +... +extras_require={ + 'my_project': ['requirement1==1.0.1', 'requirement2==2.0.1'], + }, +``` + +Please notice that a readme with instructions about how to use your module +and 100% test coverage are required to accept a PR. diff --git a/causalnex/contrib/__init__.py b/causalnex/contrib/__init__.py new file mode 100644 index 0000000..5da8261 --- /dev/null +++ b/causalnex/contrib/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/causalnex/discretiser/__init__.py b/causalnex/discretiser/__init__.py new file mode 100644 index 0000000..f8dd21d --- /dev/null +++ b/causalnex/discretiser/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``causalnex.discretiser`` provides functionality to discretise data. +""" + +__version__ = "0.4.0" + +__all__ = ["Discretiser"] + +from .discretiser import Discretiser diff --git a/causalnex/discretiser/discretiser.py b/causalnex/discretiser/discretiser.py new file mode 100644 index 0000000..763a38e --- /dev/null +++ b/causalnex/discretiser/discretiser.py @@ -0,0 +1,214 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tools to help discretise data.""" + +from typing import List + +import numpy as np +from sklearn.base import BaseEstimator, TransformerMixin + + +class Discretiser(BaseEstimator, TransformerMixin): + """Allows the discretisation of numeric data. + + Example: + :: + >>> import causalnex + >>> import pandas as pd + >>> + >>> df = pd.DataFrame({'Age': [12, 13, 18, 19, 22, 60]}) + >>> + >>> from causalnex.discretiser import Discretiser + >>> df["Transformed_Age_1"] = Discretiser(method="fixed", + >>> numeric_split_points=[11,18,50]).transform(df["Age"]) + >>> df.to_dict() + {'Age': {0: 7, 1: 12, 2: 13, 3: 18, 4: 19, 5: 22, 6: 60}, + 'Transformed_Age': {0: 0, 1: 1, 2: 1, 3: 2, 4: 2, 5: 2, 6: 3}} + """ + + def __init__( + self, + method: str = "uniform", + num_buckets: int = None, + outlier_percentile: float = None, + numeric_split_points: List[float] = None, + percentile_split_points: List[float] = None, + ): + """ + Creates a new Discretiser, that provides fit, fit_transform, and transform function to discretise data. + + Args: + method (str): can be one of: + - uniform: discretise data into uniformly spaced buckets. Note, complete uniformity + cannot be guaranteed under all circumstances, for example, if 5 data points are to split + into 2 buckets, then one will contain 2 points, and the other will contain 3. + Provide num_buckets. + - quantile: discretise data according to the distribution of values. For example, providing + num_buckets=4 will discretise data into 4 buckets, [0-25th, 25th-50th, 50th-75th, 75th-100th] + percentiles. Provide num_buckets. + - outlier: discretise data into 3 buckets - [low_outliers, normal, high_outliers] based on + outliers being below outlier_percentile, or above 1-outlier_percentile. Provide outlier_percentile. + - fixed: discretise according to pre-defined split points. Provide numeric_split_points + - percentiles: discretise data according to the distribution of percentiles values. + Provide percentile_split_points. + num_buckets: (int): used by method=uniform and method=quantile. + outlier_percentile: used by method=outlier. + numeric_split_points: used by method=fixed. to split such that values below 10 go into bucket 0, + 10 to 20 go into bucket 1, and above 20 go into bucket 2, provide [10, 21]. Note that split_point + values are non-inclusive. + percentile_split_points: used by method=percentiles. to split such that values below 10th percentiles + go into bucket 0, 10th to below 75th percentiles go into bucket 1, and 75th percentiles and above go into + bucket 2, provide [0.1, 0.75]. + + Raises: + ValueError: If an incorrect argument is passed. + """ + + self.numeric_split_points = [] + + self.method = method + self.num_buckets = num_buckets + self.outlier_percentile = outlier_percentile + self.numeric_split_points = numeric_split_points + self.percentile_split_points = percentile_split_points + + allowed_methods = ["uniform", "quantile", "outlier", "fixed", "percentiles"] + + if self.method not in allowed_methods: + raise ValueError( + "{0} is not a recognised method. Use one of: {1}".format( + self.method, " ".join(allowed_methods) + ) + ) + if self.method in {"uniform", "quantile"} and num_buckets is None: + raise ValueError( + "{0} method expects {1}".format(self.method, "num_buckets") + ) + + if self.method == "outlier" and outlier_percentile is None: + raise ValueError( + "{0} method expects {1}".format(self.method, "outlier_percentile") + ) + + if outlier_percentile is not None and not 0 <= outlier_percentile < 0.5: + raise ValueError( + "{0} must be between 0 and 0.5".format("outlier_percentile") + ) + + if self.method == "fixed" and numeric_split_points is None: + raise ValueError( + "{0} method expects {1}".format(self.method, "numeric_split_points") + ) + + if ( + numeric_split_points is not None + and sorted(numeric_split_points) != numeric_split_points + ): + raise ValueError( + "{0} must be monotonically increasing".format("numeric_split_points") + ) + + if self.method == "percentiles" and percentile_split_points is None: + raise ValueError( + "{0} method expects {1}".format(self.method, "percentile_split_points") + ) + + if percentile_split_points is not None and not all( + 0 <= p <= 1 for p in percentile_split_points + ): + raise ValueError( + "{0} must be between 0 and 1".format("percentile_split_points") + ) + + if ( + percentile_split_points is not None + and sorted(percentile_split_points) != percentile_split_points + ): + raise ValueError( + "{0} must be monotonically increasing".format("percentile_split_points") + ) + + if self.method == "fixed": + self.numeric_split_points = numeric_split_points + + def fit(self, data: np.ndarray) -> "Discretiser": + """ + Fit where split points are based on the input data. + + Args: + data (np.ndarray): values used to learn where split points exist. + + Returns: + self + + Raises: + RuntimeError: If an attempt to fit fixed numeric_split_points is made. + """ + + x = data.flatten() + x.sort() + + if self.method == "uniform": + bucket_width = len(x) / self.num_buckets + self.numeric_split_points = [ + x[int(np.floor((n + 1) * bucket_width))] + for n in range(self.num_buckets - 1) + ] + + elif self.method == "quantile": + bucket_width = 1.0 / self.num_buckets + quantiles = [bucket_width * (n + 1) for n in range(self.num_buckets - 1)] + self.numeric_split_points = np.quantile(x, quantiles) + + elif self.method == "outlier": + self.numeric_split_points = np.quantile( + x, [self.outlier_percentile, 1 - self.outlier_percentile] + ) + + elif self.method == "percentiles": + percentiles = [p * 100 for p in self.percentile_split_points] + self.numeric_split_points = np.percentile(x, percentiles) + + else: + raise RuntimeError("cannot call fit using method=fixed") + + return self + + def transform(self, data: np.ndarray) -> np.ndarray: + """ + Transform the input data into discretised digits, based on the numeric_split_points that were either + learned through using fit(), or from initialisation if method="fixed". + + Args: + data (np.ndarray): values that will be transformed into discretised digits. + + Returns: + input data transformed into discretised digits. + """ + + return np.digitize(data, self.numeric_split_points, right=False) diff --git a/causalnex/ebaybbn/__init__.py b/causalnex/ebaybbn/__init__.py new file mode 100644 index 0000000..8265862 --- /dev/null +++ b/causalnex/ebaybbn/__init__.py @@ -0,0 +1,39 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# The methods found in this file are adapted from a repository under Apache 2.0: +# eBay's Pythonic Bayesian Belief Network Framework. +# @online{ +# author = {Neville Newey,Anzar Afaq}, +# title = {bayesian-belief-networks}, +# organisation = {eBay}, +# codebase = {https://github.com/eBay/bayesian-belief-networks}, +# } +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .bbn import * diff --git a/causalnex/ebaybbn/bbn.py b/causalnex/ebaybbn/bbn.py new file mode 100644 index 0000000..ee53e5c --- /dev/null +++ b/causalnex/ebaybbn/bbn.py @@ -0,0 +1,1009 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# The methods found in this file are adapted from a repository under Apache 2.0: +# eBay's Pythonic Bayesian Belief Network Framework. +# @online{ +# author = {Neville Newey,Anzar Afaq}, +# title = {bayesian-belief-networks}, +# organisation = {eBay}, +# codebase = {https://github.com/eBay/bayesian-belief-networks}, +# } +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Data Structures to represent a BBN as a DAG.""" + +import copy +import heapq +import logging +from collections import defaultdict +from io import StringIO +from itertools import combinations, product +from random import choice, random + +from .exceptions import VariableNotInGraphError, VariableValueNotInDomainError +from .graph import Node, UndirectedGraph, UndirectedNode, connect +from .utils import get_args, get_original_factors + +# from .bayesian import GREEN, NORMAL +GREEN = "\033[92m" +NORMAL = "\033[0m" + + +class BBNNode(Node): + def __init__(self, factor): + super(BBNNode, self).__init__(factor.__name__) + self.func = factor + self.argspec = get_args(factor) + + def __repr__(self): + return "<BBNNode %s (%s)>" % (self.name, self.argspec) + + +class BBN: + """A Directed Acyclic Graph""" + + def __init__(self, nodes_dict, name=None, domains={}): + self.nodes = list(nodes_dict.values()) + self.vars_to_nodes = nodes_dict + self.domains = domains + # For each node we want + # to explicitly record which + # variable it 'introduced'. + # Note that we cannot record + # this duing Node instantiation + # becuase at that point we do + # not yet know *which* of the + # variables in the argument + # list is the one being modeled + # by the function. (Unless there + # is only one argument) + for variable_name, node in list(nodes_dict.items()): + node.variable_name = variable_name + + def get_graphviz_source(self): + fh = StringIO() + fh.write("digraph G {\n") + fh.write(' graph [ dpi = 300 bgcolor="transparent" rankdir="LR"];\n') + edges = set() + for node in sorted(self.nodes, key=lambda x: x.name): + fh.write(' %s [ shape="ellipse" color="blue"];\n' % node.name) + for child in node.children: + edge = (node.name, child.name) + edges.add(edge) + for source, target in sorted(edges, key=lambda x: (x[0], x[1])): + fh.write(" %s -> %s;\n" % (source, target)) + fh.write("}\n") + return fh.getvalue() + + def build_join_tree(self): + jt = build_join_tree(self) + return jt + + def validate_keyvals(self, **kwds): + """ + When evidence in the form of + keyvals are provided to the .query() method + validate that all keys match a variable name + and that all vals are in the domain of + the variable + """ + vars = set([n.variable_name for n in self.nodes]) + for k, v in list(kwds.items()): + if k not in vars: + raise VariableNotInGraphError(k) + domain = self.domains.get(k, (True, False)) + if v not in domain: + s = "{}={}".format(k, v) + raise VariableValueNotInDomainError(s) + return True + + def query(self, **kwds): + # First check that the keyvals + # provided are valid for this graph... + self.validate_keyvals(**kwds) + jt = self.build_join_tree() + assignments = jt.assign_clusters(self) + jt.initialize_potentials(assignments, self, kwds) + + jt.propagate() + marginals = dict() + normalizers = defaultdict(float) + + for node in self.nodes: + for k, v in list(jt.marginal(node).items()): + # For a single node the + # key for the marginal tt always + # has just one argument so we + # will unpack it here + marginals[k[0]] = v + # If we had any evidence then we + # need to normalize all the variables + # not evidenced. + if kwds: + normalizers[k[0][0]] += v + + if kwds: + for k, v in marginals.items(): + if normalizers[k[0]] != 0: + marginals[k] /= normalizers[k[0]] + + return marginals + + def draw_samples(self, query={}, n=1): + """query is a dict of currently evidenced + variables and is none by default.""" + samples = [] + result_cache = dict() + # We need to add evidence variables to the sample... + while len(samples) < n: + sample = dict(query) + while len(sample) < len(self.nodes): + next_node = choice( + [node for node in self.nodes if node.variable_name not in sample] + ) + key = tuple(sorted(sample.items())) + if key not in result_cache: + result_cache[key] = self.query(**sample) + result = result_cache[key] + var_density = [ + r + for r in list(result.items()) + if r[0][0] == next_node.variable_name + ] + cumulative_density = var_density[:1] + for key, mass in var_density[1:]: + cumulative_density.append((key, cumulative_density[-1][1] + mass)) + r = random() + i = 0 + while r > cumulative_density[i][1]: + i += 1 + sample[next_node.variable_name] = cumulative_density[i][0][1] + samples.append(sample) + return samples + + +class JoinTree(UndirectedGraph): + def __init__(self, nodes, name=None): + super(JoinTree, self).__init__(nodes, name) + self._sensitivity_flag = False + + @property + def sepset_nodes(self): + return [n for n in self.nodes if isinstance(n, JoinTreeSepSetNode)] + + @property + def clique_nodes(self): + return [n for n in self.nodes if isinstance(n, JoinTreeCliqueNode)] + + def initialize_potentials(self, assignments, bbn, evidence={}): + # Step 1, assign 1 to each cluster and sepset + for node in self.nodes: + tt = dict() + + vals = [] + variables = node.variable_names + # Lets sort the variables here so that + # the variable names in the keys in + # the tt are always sorted. + variables.sort() + for variable in variables: + domain = bbn.domains.get(variable, [True, False]) + vals.append(list(product([variable], domain))) + permutations = product(*vals) + for permutation in permutations: + tt[permutation] = 1 + node.potential_tt = tt + + # Step 2: Note that in H&D the assignments are + # done as part of step 2 however we have + # seperated the assignment algorithm out and + # done these prior to step 1. + # Now for each assignment we want to + # generate a truth-table from the + # values of the bbn truth-tables that are + # assigned to the clusters... + + for clique, bbn_nodes in assignments.items(): + tt = dict() + vals = [] + variables = list(clique.variable_names) + variables.sort() + for variable in variables: + domain = bbn.domains.get(variable, [True, False]) + vals.append(list(product([variable], domain))) + permutations = product(*vals) + for permutation in permutations: + argvals = dict(permutation) + potential = 1 + for bbn_node in bbn_nodes: + bbn_node.clique = clique + # We could handle evidence here + # by altering the potential_tt. + # This is slightly different to + # the way that H&D do it. + + arg_list = [] + for arg_name in get_args(bbn_node.func): + arg_list.append(argvals[arg_name]) + + potential *= bbn_node.func(*arg_list) + tt[permutation] = potential + clique.potential_tt = tt + + if not evidence: + # We dont need to deal with likelihoods + # if we dont have any evidence. + return + + # Step 2b: Set each liklihood element ^V(v) to 1 + self.initial_likelihoods(assignments, bbn) + for clique, bbn_nodes in assignments.items(): + for node in bbn_nodes: + if node.variable_name in evidence: + for k, v in list(clique.potential_tt.items()): + # Encode the evidence in + # the clique potential... + for variable, value in k: + if variable == node.variable_name: + if value != evidence[variable]: + clique.potential_tt[k] = 0 + + def initial_likelihoods(self, assignments, bbn): + # TODO: Since this is the same every time we should probably + # cache it. + likelihood = defaultdict(dict) + for clique, bbn_nodes in assignments.items(): + for node in bbn_nodes: + for value in bbn.domains.get(node.variable_name, [True, False]): + likelihood[(node.variable_name, value)] = 1 + return likelihood + + def assign_clusters(self, bbn): + assignments_by_family = dict() + assignments_by_clique = defaultdict(list) + assigned = set() + for node in bbn.nodes: + args = get_args(node.func) + if len(args) == 1: + # If the func has only 1 arg + # it means that it does not + # specify a conditional probability + # This is where H&D is a bit vague + # but it seems to imply that we + # do not assign it to any + # clique. + # Revising this for now as I dont + # think its correct, I think + # all CPTs need to be assigned + # once and once only. The example + # in H&D just happens to be a clique + # that f_a could have been assigned + # to but wasnt presumably because + # it got assigned somewhere else. + pass + # continue + # Now we need to find a cluster that + # is a superset of the Family(v) + # Family(v) is defined by D&H to + # be the union of v and parents(v) + family = set(args) + # At this point we need to know which *variable* + # a BBN node represents. Up to now we have + # not *explicitely* specified this, however + # we have been following some conventions + # so we could just use this convention for + # now. Need to come back to this to + # perhaps establish the variable at + # build bbn time... + + containing_cliques = [ + clique_node + for clique_node in self.clique_nodes + if (set(clique_node.variable_names).issuperset(family)) + ] + assert len(containing_cliques) >= 1 + for clique in containing_cliques: + if node in assigned: + # Make sure we assign all original + # PMFs only once each + break + assignments_by_clique[clique].append(node) + assigned.add(node) + assignments_by_family[tuple(family)] = containing_cliques + return assignments_by_clique + + def propagate(self, starting_clique=None): + """Refer to H&D pg. 20""" + + # Step 1 is to choose an arbitrary clique cluster + # as starting cluster + if starting_clique is None: + starting_clique = self.clique_nodes[0] + logging.debug("Starting propagating messages from: %s", starting_clique.name) + # Step 2: Unmark all clusters, call collect_evidence(X) + for node in self.clique_nodes: + node.marked = False + logging.debug("Marking node as not visited Node: %s", node.name) + self.collect_evidence(sender=starting_clique) + + # Step 3: Unmark all clusters, call distribute_evidence(X) + for node in self.clique_nodes: + node.marked = False + + self.distribute_evidence(starting_clique) + + def collect_evidence(self, sender=None, receiver=None): + + logging.debug("Collect evidence from %s", sender.name) + # Step 1, Mark X + sender.marked = True + + # Step 2, call collect_evidence on Xs unmarked + # neighbouring clusters. + for neighbouring_clique in sender.neighbouring_cliques: + if not neighbouring_clique.marked: + logging.debug( + "Collect evidence from %s to %s", + neighbouring_clique.name, + sender.name, + ) + self.collect_evidence(sender=neighbouring_clique, receiver=sender) + # Step 3, pass message from sender to receiver + if receiver is not None: + sender.pass_message(receiver) + + def distribute_evidence(self, sender=None, receiver=None): + logging.debug("Distribute evidence from: %s", sender.name) + # Step 1, Mark X + sender.marked = True + + # Step 2, pass a messagee from X to each of its + # unmarked neighbouring clusters + for neighbouring_clique in sender.neighbouring_cliques: + if not neighbouring_clique.marked: + logging.debug( + "Pass message from: %s to %s", sender.name, neighbouring_clique.name + ) + sender.pass_message(neighbouring_clique) + + # Step 3, call distribute_evidence on Xs unmarked neighbours + for neighbouring_clique in sender.neighbouring_cliques: + if not neighbouring_clique.marked: + logging.debug( + "Distribute evidence from: %s to %s", + neighbouring_clique.name, + sender.name, + ) + self.distribute_evidence(sender=neighbouring_clique, receiver=sender) + + def marginal(self, bbn_node): + """Remember that the original + variables that we are interested in + are actually in the bbn. However + when we constructed the JT we did + it out of the moralized graph. + This means the cliques refer to + the nodes in the moralized graph + and not the nodes in the BBN. + For efficiency we should come back + to this and add some pointers + or an index. + """ + + # First we will find the JT nodes that + # contain the bbn_node ie all the nodes + # that are either cliques or sepsets + # that contain the bbn_node + # Note that for efficiency we + # should probably have an index + # cached in the bbn and/or the jt. + containing_nodes = [] + + for node in self.clique_nodes: + if bbn_node.name in [n.name for n in node.clique.nodes]: + containing_nodes.append(node) + # In theory it doesnt matter which one we + # use so we could bale out after we + # find the first one + # TODO: With some better indexing we could + # avoid searching for this node every time... + clique_node = containing_nodes[0] + tt = defaultdict(float) + for k, v in list(clique_node.potential_tt.items()): + entry = transform(k, clique_node.variable_names, [bbn_node.variable_name]) + tt[entry] += v + + # Now if this node was evidenced we need to normalize + # over the values... + # TODO: It will be safer to copy the defaultdict to a regular dict + return tt + + +class Clique(object): + def __init__(self, cluster): + self.nodes = cluster + + def __repr__(self): + vars = sorted([n.variable_name for n in self.nodes]) + return "Clique_%s" % "".join([v.upper() for v in vars]) + + +def transform(x, X, R): + """Transform a Potential Truth Table + Entry into a different variable space. + For example if we have the + entry [True, True, False] representing + values of variable [A, B, C] in X + and we want to transform into + R which has variables [C, A] we + will return the entry [False, True]. + Here X represents the argument list + for the clique set X and R represents + the argument list for the sepset. + This implies that R is always a subset + of X""" + entry = [] + for r in R: + pos = X.index(r) + entry.append(x[pos]) + return tuple(entry) + + +class JoinTreeCliqueNode(UndirectedNode): + def __init__(self, clique): + super(JoinTreeCliqueNode, self).__init__(clique.__repr__()) + self.clique = clique + self.potential_psi = None + + # Now we create a pointer to + # this clique node as the "parent" clique + # node of each node in the cluster. + # for node in self.clique.nodes: + # node.parent_clique = self + # This is not quite correct, the + # parent cluster as defined by H&D + # is *a* cluster than is a superset + # of Family(v) + + @property + def variable_names(self): + """Return the set of variable names + that this clique represents""" + var_names = [] + for node in self.clique.nodes: + var_names.append(node.variable_name) + return sorted(var_names) + + @property + def neighbouring_cliques(self): + """Return the neighbouring cliques + this is used during the propagation algorithm. + + """ + neighbours = set() + for sepset_node in self.neighbours: + # All *immediate* neighbours will + # be sepset nodes, its the neighbours of + # these sepsets that form the nodes + # clique neighbours (excluding itself) + for clique_node in sepset_node.neighbours: + if clique_node is not self: + neighbours.add(clique_node) + return neighbours + + def pass_message(self, target): + """Pass a message from this node to the + recipient node during propagation. + + NB: It may turnout at this point that + after initializing the potential + Truth table on the JT we could quite + simply construct a factor graph + from the JT and use the factor + graph sum product propagation. + In theory this should be the same + and since the semantics are already + worked out it would be easier.""" + + # Find the sepset node between the + # source and target nodes. + sepset_node = list(set(self.neighbours).intersection(target.neighbours))[0] + + logging.debug("Pass message from: %s to: %s", self.name, target.name) + # Step 1: projection + logging.debug("Project into the Sepset node: %s", str(sepset_node)) + self.project(sepset_node) + + logging.debug(" Send the summed marginals to the target: %s ", str(sepset_node)) + + # Step 2 absorbtion + self.absorb(sepset_node, target) + + def project(self, sepset_node): + """See page 20 of PPTC. + We assign a new potential tt to + the sepset which consists of the + potential of the source node + with all variables not in R marginalized. + """ + assert sepset_node in self.neighbours + # First we make a copy of the + # old potential tt + + # Now we assign a new potential tt + # to the sepset by marginalizing + # out the variables from X that are not + # in the sepset + # ToDO test and check this function + # Todo check on the old sepset potentials and when they will be set + + sepset_node.potential_tt_old = copy.deepcopy(sepset_node.potential_tt) + tt = defaultdict(float) + for k, v in self.potential_tt.items(): + entry = transform(k, self.variable_names, sepset_node.variable_names) + tt[entry] += v + sepset_node.potential_tt = tt + + def absorb(self, sepset, target): + # Assign a new potential tt to + # Y (the target) + logging.debug( + "Absorb potentails from sepset node %s into clique %s", + sepset.name, + target.name, + ) + tt = dict() + + for k, v in list(target.potential_tt.items()): + # For each entry we multiply by + # sepsets new value and divide + # by sepsets old value... + # Note that nowhere in H&D is + # division on potentials defined. + # However in Barber page 12 + # an equation implies that + # the the division is equivalent + # to the original assignment. + # For now we will assume entry-wise + # division which seems logical. + entry = transform(k, target.variable_names, sepset.variable_names) + if target.potential_tt[k] == 0: + tt[k] = 0 + else: + tt[k] = target.potential_tt[k] * ( + sepset.potential_tt[entry] / sepset.potential_tt_old[entry] + ) + # assign the new potentials to the node + target.potential_tt = tt + + def __repr__(self): + return "<JoinTreeCliqueNode: %s>" % self.clique + + +class SepSet(object): + def __init__(self, X, Y): + """X and Y are cliques represented as sets.""" + self.X = X + self.Y = Y + self.label = list(X.nodes.intersection(Y.nodes)) + + @property + def mass(self): + return len(self.label) + + @property + def cost(self): + """Since cost is used as a tie-breaker + and is an optimization for inference time + we will punt on it for now. Instead we + will just use the assumption that all + variables in X and Y are binary and thus + use a weight of 2. + TODO: come back to this and compute + actual weights + """ + return 2 ** len(self.X.nodes) + 2 ** len(self.Y.nodes) + + def insertable(self, forest): + """A sepset can only be inserted + into the JT if the cliques it + separates are NOT already on + the same tree. + NOTE: For efficiency we should + add an index that indexes cliques + into the trees in the forest.""" + X_trees = [t for t in forest if self.X in [n.clique for n in t.clique_nodes]] + Y_trees = [t for t in forest if self.Y in [n.clique for n in t.clique_nodes]] + assert len(X_trees) == 1 + assert len(Y_trees) == 1 + if X_trees[0] is not Y_trees[0]: + return True + return False + + def insert(self, forest): + """Inserting this sepset into + a forest, providing the two + cliques are in different trees, + means that effectively we are + collapsing the two trees into + one. We will explicitely perform + this collapse by adding the + sepset node into the tree + and adding edges between itself + and its clique node neighbours. + Finally we must remove the + second tree from the forest + as it is now joined to the + first. + """ + X_tree = [t for t in forest if self.X in [n.clique for n in t.clique_nodes]][0] + Y_tree = [t for t in forest if self.Y in [n.clique for n in t.clique_nodes]][0] + + # Now create and insert a sepset node into the Xtree + ss_node = JoinTreeSepSetNode(self, self) + X_tree.nodes.append(ss_node) + + # And connect them + self.X.node.neighbours.append(ss_node) + ss_node.neighbours.append(self.X.node) + + # Now lets keep the X_tree and drop the Y_tree + # this means we need to copy all the nodes + # in the Y_tree that are not already in the X_tree + for node in Y_tree.nodes: + if node in X_tree.nodes: + continue + X_tree.nodes.append(node) + + # Now connect the sepset node to the + # Y_node (now residing in the X_tree) + self.Y.node.neighbours.append(ss_node) + ss_node.neighbours.append(self.Y.node) + + # And finally we must remove the Y_tree from + # the forest... + forest.remove(Y_tree) + + def __repr__(self): + return "SepSet_%s" % "".join( + # [x.name[2:].upper() for x in list(self.label)]) + [x.variable_name.upper() for x in list(self.label)] + ) + + +class JoinTreeSepSetNode(UndirectedNode): + def __init__(self, name, sepset): + super(JoinTreeSepSetNode, self).__init__(name) + self.sepset = sepset + self.potential_psi = None + + @property + def variable_names(self): + """Return the set of variable names + that this sepset represents""" + # TODO: we are assuming here + # that X and Y are each separate + # variables from the BBN which means + # we are assuming that the sepsets + # always contain only 2 nodes. + # Need to check whether this is + # the case. + return sorted([x.variable_name for x in self.sepset.label]) + + def __repr__(self): + return "<JoinTreeSepSetNode: %s>" % self.sepset + + +def build_bbn(*args, **kwds): + """Builds a BBN Graph from + a list of functions and domains""" + variables = set() + domains = kwds.get("domains", {}) + name = kwds.get("name") + factor_nodes = dict() + + if isinstance(args[0], list): + # Assume the functions were all + # passed in a list in the first + # argument. This makes it possible + # to build very large graphs with + # more than 255 functions, since + # Python functions are limited to + # 255 arguments. + args = args[0] + + for factor in args: + factor_args = get_args(factor) + variables.update(factor_args) + bbn_node = BBNNode(factor) + factor_nodes[factor.__name__] = bbn_node + + # Now lets create the connections + # To do this we need to find the + # factor node representing the variables + # in a child factors argument and connect + # it to the child node. + + # Note that calling original_factors + # here can break build_bbn if the + # factors do not correctly represent + # a BBN. + original_factors = get_original_factors(list(factor_nodes.values())) + for factor_node in list(factor_nodes.values()): + factor_args = get_args(factor_node) + parents = [ + original_factors[arg] + for arg in factor_args + if original_factors[arg] != factor_node + ] + for parent in parents: + connect(parent, factor_node) + bbn = BBN(original_factors, name=name) + bbn.domains = domains + + return bbn + + +def make_node_func(variable_name, conditions): + # We will enforce the following + # convention. + # The ordering of arguments will + # be firstly the parent variables + # in alphabetical order, followed + # always by the child variable + tt = dict() + domain = set() + for givens, conditionals in conditions: + key = [] + for parent_name, val in sorted(givens): + key.append((parent_name, val)) + # Now we will sort the + # key before we add the child + # node. + # key.sort(key=lambda x: x[0]) + + # Now for each value in + # the conditional probabilities + # we will add a new key + for value, prob in list(conditionals.items()): + key_ = tuple(key + [(variable_name, value)]) + domain.add(value) + tt[key_] = prob + + argspec = [k[0] for k in key_] + + def node_func(*args): + key = [] + for arg, val in zip(argspec, args): + key.append((arg, val)) + return tt[tuple(key)] + + node_func.argspec = argspec + node_func._domain = domain + node_func.__name__ = "f_" + variable_name + return node_func + + +def build_bbn_from_conditionals(conds): + node_funcs = [] + domains = dict() + for variable_name, cond_tt in list(conds.items()): + node_func = make_node_func(variable_name, cond_tt) + node_funcs.append(node_func) + domains[variable_name] = node_func._domain + return build_bbn(*node_funcs, domains=domains) + + +def make_undirected_copy(dag): + """Returns an exact copy of the dag + except that direction of edges are dropped.""" + nodes = dict() + for node in dag.nodes: + undirected_node = UndirectedNode(name=node.name) + undirected_node.func = node.func + undirected_node.argspec = node.argspec + undirected_node.variable_name = node.variable_name + nodes[node.name] = undirected_node + # Now we need to traverse the original + # nodes once more and add any parents + # or children as neighbours. + for node in dag.nodes: + for parent in node.parents: + nodes[node.name].neighbours.append(nodes[parent.name]) + nodes[parent.name].neighbours.append(nodes[node.name]) + + g = UndirectedGraph(list(nodes.values())) + return g + + +def make_moralized_copy(gu, dag): + """gu is an undirected graph being + a copy of dag.""" + gm = copy.deepcopy(gu) + gm_nodes = dict([(node.name, node) for node in gm.nodes]) + for node in dag.nodes: + for parent_1, parent_2 in combinations(node.parents, 2): + if gm_nodes[parent_1.name] not in gm_nodes[parent_2.name].neighbours: + gm_nodes[parent_2.name].neighbours.append(gm_nodes[parent_1.name]) + if gm_nodes[parent_2.name] not in gm_nodes[parent_1.name].neighbours: + gm_nodes[parent_1.name].neighbours.append(gm_nodes[parent_2.name]) + return gm + + +def priority_func(node): + """Specify the rules for computing + priority of a node. See Harwiche and Wang pg 12. + """ + # We need to calculate the number of edges + # that would be added. + # For each node, we need to connect all + # of the nodes in itself and its neighbours + # (the "cluster") which are not already + # connected. This will be the primary + # key value in the heap. + # We need to fix the secondary key, right + # now its just 2 (because mostly the variables + # will be discrete binary) + introduced_arcs = 0 + cluster = [node] + node.neighbours + for node_a, node_b in combinations(cluster, 2): + if node_a not in node_b.neighbours: + assert node_b not in node_a.neighbours + introduced_arcs += 1 + return [introduced_arcs, 2] # TODO: Fix this to look at domains + + +def construct_priority_queue(nodes, priority_func=priority_func): + pq = [] + for node_name, node in nodes.items(): + entry = priority_func(node) + [node.name] + heapq.heappush(pq, entry) + return pq + + +def record_cliques(cliques, cluster): + """We only want to save the cluster + if it is not a subset of any clique + already saved. + Argument cluster must be a set""" + if any([cluster.issubset(c.nodes) for c in cliques]): + return + cliques.append(Clique(cluster)) + + +def triangulate(gm, priority_func=priority_func): + """Triangulate the moralized Graph. (in Place) + and return the cliques of the triangulated + graph as well as the elimination ordering.""" + + # First we will make a copy of gm... + gm_ = copy.deepcopy(gm) + + # Now we will construct a priority q using + # the standard library heapq module. + # See docs for example of priority q tie + # breaking. We will use a 3 element list + # with entries as follows: + # - Number of edges added if V were selected + # - Weight of V (or cluster) + # - Pointer to node in gm_ + # Note that its unclear from Huang and Darwiche + # what is meant by the "number of values of V" + gmnodes = dict([(node.name, node) for node in gm.nodes]) + elimination_ordering = [] + cliques = [] + while True: + gm_nodes = dict([(node.name, node) for node in gm_.nodes]) + if not gm_nodes: + break + pq = construct_priority_queue(gm_nodes, priority_func) + # Now we select the first node in + # the priority q and any arcs that + # should be added in order to fully connect + # the cluster should be added to both + # gm and gm_ + v = gm_nodes[pq[0][2]] + cluster = [v] + v.neighbours + for node_a, node_b in combinations(cluster, 2): + if node_a not in node_b.neighbours: + node_b.neighbours.append(node_a) + node_a.neighbours.append(node_b) + # Now also add this new arc to gm... + gmnodes[node_b.name].neighbours.append(gmnodes[node_a.name]) + gmnodes[node_a.name].neighbours.append(gmnodes[node_b.name]) + gmcluster = set([gmnodes[c.name] for c in cluster]) + record_cliques(cliques, gmcluster) + # Now we need to remove v from gm_... + # This means we also have to remove it from all + # of its neighbours that reference it... + for neighbour in v.neighbours: + neighbour.neighbours.remove(v) + gm_.nodes.remove(v) + elimination_ordering.append(v.name) + return cliques, elimination_ordering + + +def build_join_tree(dag, clique_priority_func=priority_func): + # First we will create an undirected copy + # of the dag + gu = make_undirected_copy(dag) + + # Now we create a copy of the undirected graph + # and connect all pairs of parents that are + # not already parents called the 'moralized' graph. + gm = make_moralized_copy(gu, dag) + + # Now we triangulate the moralized graph... + cliques, elimination_ordering = triangulate(gm, clique_priority_func) + + # Now we initialize the forest and sepsets + # Its unclear from Darwiche Huang whether we + # track a sepset for each tree or whether its + # a global list???? + # We will implement the Join Tree as an undirected + # graph for now... + + # First initialize a set of graphs where + # each graph initially consists of just one + # node for the clique. As these graphs get + # populated with sepsets connecting them + # they should collapse into a single tree. + forest = set() + for clique in cliques: + jt_node = JoinTreeCliqueNode(clique) + # Track a reference from the clique + # itself to the node, this will be + # handy later... (alternately we + # could just collapse clique and clique + # node into one class... + clique.node = jt_node + tree = JoinTree([jt_node]) + forest.add(tree) + + # Initialize the SepSets + S = set() # track the sepsets + for X, Y in combinations(cliques, 2): + if X.nodes.intersection(Y.nodes): + S.add(SepSet(X, Y)) + sepsets_inserted = 0 + while sepsets_inserted < (len(cliques) - 1): + # Adding in name to make this sort deterministic + deco = [(s, -1 * s.mass, s.cost, s.__repr__()) for s in S] + deco.sort(key=lambda x: x[1:]) + candidate_sepset = deco[0][0] + for candidate_sepset, _, _, _ in deco: + if candidate_sepset.insertable(forest): + # Insert into forest and remove the sepset + candidate_sepset.insert(forest) + S.remove(candidate_sepset) + sepsets_inserted += 1 + break + + assert len(forest) == 1 + jt = list(forest)[0] + return jt diff --git a/causalnex/ebaybbn/exceptions.py b/causalnex/ebaybbn/exceptions.py new file mode 100644 index 0000000..b515d3a --- /dev/null +++ b/causalnex/ebaybbn/exceptions.py @@ -0,0 +1,100 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# The methods found in this file are adapted from a repository under Apache 2.0: +# eBay's Pythonic Bayesian Belief Network Framework. +# @online{ +# author = {Neville Newey,Anzar Afaq}, +# title = {bayesian-belief-networks}, +# organisation = {eBay}, +# codebase = {https://github.com/eBay/bayesian-belief-networks}, +# } +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class InvalidGraphException(Exception): + """ + Raised if the graph verification + method fails. + """ + + pass + + +class InvalidSampleException(Exception): + """Should be raised if a + sample is invalid.""" + + pass + + +class InvalidInferenceMethod(Exception): + """Raise if the user tries to set + the inference method to an unknown string.""" + + pass + + +class InsufficientSamplesException(Exception): + """Raised when the inference method + is 'sample_db' and there are less + pre-generated samples than the + graphs n_samples attribute.""" + + pass + + +class NoSamplesInDB(Warning): + pass + + +class VariableNotInGraphError(Exception): + """Exception raised when + a graph is queried with + a variable that is not part of + the graph. + """ + + pass + + +class VariableValueNotInDomainError(Exception): + """Raised when a BBN is queried with + a value for a variable that is not within + that variables domain.""" + + pass + + +class IncorrectInferenceMethodError(Exception): + """Raise when attempt is made to + generate samples when the inference + method is not 'sample_db' + """ + + pass diff --git a/causalnex/ebaybbn/graph.py b/causalnex/ebaybbn/graph.py new file mode 100644 index 0000000..df7ac01 --- /dev/null +++ b/causalnex/ebaybbn/graph.py @@ -0,0 +1,74 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# The methods found in this file are adapted from a repository under Apache 2.0: +# eBay's Pythonic Bayesian Belief Network Framework. +# @online{ +# author = {Neville Newey,Anzar Afaq}, +# title = {bayesian-belief-networks}, +# organisation = {eBay}, +# codebase = {https://github.com/eBay/bayesian-belief-networks}, +# } +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Generic Graph Classes""" + + +class Node(object): + def __init__(self, name, parents=[], children=[]): + self.name = name + self.parents = parents[:] + self.children = children[:] + + def __repr__(self): + return "<Node %s>" % self.name + + +class UndirectedNode(object): + def __init__(self, name, neighbours=[]): + self.name = name + self.neighbours = neighbours[:] + + def __repr__(self): + return "<UndirectedNode %s>" % self.name + + +class UndirectedGraph(object): + def __init__(self, nodes, name=None): + self.nodes = nodes + self.name = name + + +def connect(parent, child): + """ + Make an edge between a parent + node and a child node. + a - parent + b - child + """ + parent.children.append(child) + child.parents.append(parent) diff --git a/causalnex/ebaybbn/utils.py b/causalnex/ebaybbn/utils.py new file mode 100644 index 0000000..16265f0 --- /dev/null +++ b/causalnex/ebaybbn/utils.py @@ -0,0 +1,94 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# The methods found in this file are adapted from a repository under Apache 2.0: +# eBay's Pythonic Bayesian Belief Network Framework. +# @online{ +# author = {Neville Newey,Anzar Afaq}, +# title = {bayesian-belief-networks}, +# organisation = {eBay}, +# codebase = {https://github.com/eBay/bayesian-belief-networks}, +# } +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Some Useful Helper Functions""" +import inspect + +# TODO: Find a better location for get_args + + +def get_args(func): + """ + Return the names of the arguments + of a function as a list of strings. + This is so that we can omit certain + variables when we marginalize. + Note that functions created by + make_product_func do not return + an argspec, so we add a argspec + attribute at creation time. + """ + + if hasattr(func, "argspec"): + return func.argspec + # return inspect.getargspec(func).args + return [p for p in inspect.signature(func).parameters] + + +def make_key(*args): + """Handy for short truth table keys""" + key = "" + for a in args: + if hasattr(a, "value"): + raise ValueError("Unexpected type") + else: + key += str(a).lower()[0] + return key + + +def get_original_factors(factors): + """ + For a set of factors, we want to + get a mapping of the variables to + the factor which first introduces the + variable to the set. + To do this without enforcing a special + naming convention such as 'f_' for factors, + or a special ordering, such as the last + argument is always the new variable, + we will have to discover the 'original' + factor that introduces the variable + iteratively. + """ + original_factors = dict() + while len(original_factors) < len(factors): + for factor in factors: + args = get_args(factor) + unaccounted_args = [a for a in args if a not in original_factors] + if len(unaccounted_args) == 1: + original_factors[unaccounted_args[0]] = factor + return original_factors diff --git a/causalnex/evaluation/__init__.py b/causalnex/evaluation/__init__.py new file mode 100644 index 0000000..e450aea --- /dev/null +++ b/causalnex/evaluation/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``causalnex.evaluation`` provides functionality to evaluate causal models using standard metrics. +""" + +__version__ = "0.4.0" + +__all__ = ["roc_auc", "classification_report"] + +from .evaluation import classification_report, roc_auc diff --git a/causalnex/evaluation/evaluation.py b/causalnex/evaluation/evaluation.py new file mode 100644 index 0000000..2636c0b --- /dev/null +++ b/causalnex/evaluation/evaluation.py @@ -0,0 +1,207 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +"""Evaluation metrics for causal models.""" + +from typing import List, Tuple + +import pandas as pd +from sklearn import metrics + +from causalnex.network import BayesianNetwork + + +def _build_ground_truth( + bn: BayesianNetwork, data: pd.DataFrame, node: str +) -> pd.DataFrame: + + ground_truth = pd.get_dummies(data[node]) + + # it's possible that not all states are present in the test set, so we need to add them to ground truth + for dummy in bn.node_states[node]: + if dummy not in ground_truth.columns: + ground_truth[dummy] = [0 for _ in range(len(ground_truth))] + + # update ground truth column names to be correct, since we may have added missing columns + return ground_truth[sorted(ground_truth.columns)] + + +def roc_auc( + bn: BayesianNetwork, data: pd.DataFrame, node: str +) -> Tuple[List[Tuple[float, float]], float]: + """ + Build a report of the micro-average Receiver-Operating Characteristics (ROC), and the Area Under the ROC curve + Micro-average computes roc_auc over all predictions for all states of node. + + Args: + bn (BayesianNetwork): model to compute roc_auc. + data (pd.DataFrame): test data that will be used to calculate ROC. + node (str): name of the variable to generate the report for. + + Returns: + roc - auc tuple + - roc (List[Tuple[float, float]]): list of [(fpr, tpr)] observations. + - auc float: auc for the node predictions. + + Example: + :: + >>> from causalnex.structure import StructureModel + >>> from causalnex.network import BayesianNetwork + >>> + >>> sm = StructureModel() + >>> sm.add_edges_from([ + >>> ('rush_hour', 'traffic'), + >>> ('weather', 'traffic') + >>> ]) + >>> bn = BayesianNetwork(sm) + >>> import pandas as pd + >>> data = pd.DataFrame({ + >>> 'rush_hour': [True, False, False, False, True, False, True], + >>> 'weather': ['Terrible', 'Good', 'Bad', 'Good', 'Bad', 'Bad', 'Good'], + >>> 'traffic': ['heavy', 'light', 'heavy', 'light', 'heavy', 'heavy', 'heavy'] + >>> } + >>> bn = bn.fit_node_states_and_cpds(data) + >>> test_data = pd.DataFrame({ + >>> 'rush_hour': [False, False, True, True], + >>> 'weather': ['Good', 'Bad', 'Good', 'Bad'], + >>> 'traffic': ['light', 'heavy', 'heavy', 'light'] + >>> }) + >>> from causalnex.evaluation import roc_auc + >>> roc, auc = roc_auc(bn, test_data, "traffic") + >>> print(auc) + 0.75 + """ + + ground_truth = _build_ground_truth(bn, data, node) + predictions = bn.predict_probability(data, node) + + # update column names to match those of ground_truth + predictions.rename(columns=lambda x: x.lstrip(node + "_"), inplace=True) + predictions = predictions[sorted(predictions.columns)] + + fpr, tpr, _ = metrics.roc_curve( + ground_truth.values.ravel(), predictions.values.ravel() + ) + roc = list(zip(fpr, tpr)) + auc = metrics.auc(fpr, tpr) + + return roc, auc + + +def classification_report( + bn: BayesianNetwork, data: pd.DataFrame, node: str +) -> pd.DataFrame: + """ + Build a report showing the main classification metrics. + + Args: + bn (BayesianNetwork): model to compute classification report using. + data (pd.DataFrame): test data that will be used for predictions. + node (str): name of the variable to generate report for. + + Returns: + Text summary of the precision, recall, F1 score for each class. + + The reported averages include micro average (averaging the + total true positives, false negatives and false positives), macro + average (averaging the unweighted mean per label), weighted average + (averaging the support-weighted mean per label) and sample average + (only for multilabel classification). + + Note that in binary classification, recall of the positive class + is also known as "sensitivity"; recall of the negative class is + "specificity". + + Example: + :: + >>> from causalnex.structure import StructureModel + >>> from causalnex.network import BayesianNetwork + >>> + >>> sm = StructureModel() + >>> sm.add_edges_from([ + >>> ('rush_hour', 'traffic'), + >>> ('weather', 'traffic') + >>> ]) + >>> bn = BayesianNetwork(sm) + >>> import pandas as pd + >>> data = pd.DataFrame({ + >>> 'rush_hour': [True, False, False, False, True, False, True], + >>> 'weather': ['Terrible', 'Good', 'Bad', 'Good', 'Bad', 'Bad', 'Good'], + >>> 'traffic': ['heavy', 'light', 'heavy', 'light', 'heavy', 'heavy', 'heavy'] + >>> } + >>> bn = bn.fit_node_states_and_cpds(data) + >>> test_data = pd.DataFrame({ + >>> 'rush_hour': [False, False, True, True], + >>> 'weather': ['Good', 'Bad', 'Good', 'Bad'], + >>> 'traffic': ['light', 'heavy', 'heavy', 'light'] + >>> }) + >>> from causalnex.evaluation import classification_report + >>> classification_report(bn, test_data, "traffic").to_dict() + {'precision': { + 'macro avg': 0.8333333333333333, 'micro avg': 0.75, + 'traffic_heavy': 0.6666666666666666, + 'traffic_light': 1.0, + 'weighted avg': 0.8333333333333333 + }, + 'recall': { + 'macro avg': 0.75, + 'micro avg': 0.75, + 'traffic_heavy': 1.0, + 'traffic_light': 0.5, + 'weighted avg': 0.75 + }, + 'f1-score': { + 'macro avg': 0.7333333333333334, + 'micro avg': 0.75, + 'traffic_heavy': 0.8, + 'traffic_light': 0.6666666666666666, + 'weighted avg': 0.7333333333333334 + }, + 'support': { + 'macro avg': 4, + 'micro avg': 4, + 'traffic_heavy': 2, + 'traffic_light': 2, + 'weighted avg': 4 + }} + """ + + predictions = bn.predict(data, node) + + labels = sorted(list(bn.node_states[node])) + target_names = [ + "{0}_{1}".format(node, str(v)) for v in sorted(bn.node_states[node]) + ] + report = metrics.classification_report( + y_true=data[node], + y_pred=predictions, + labels=labels, + target_names=target_names, + output_dict=True, + ) + + return pd.DataFrame.from_dict(report, orient="index") diff --git a/causalnex/inference/__init__.py b/causalnex/inference/__init__.py new file mode 100644 index 0000000..4628b6e --- /dev/null +++ b/causalnex/inference/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``causalnex.inference`` provides functionality to make inferences based on interventions and observations. +""" + +__version__ = "0.4.0" + +__all__ = ["InferenceEngine"] + +from .inference import InferenceEngine diff --git a/causalnex/inference/inference.py b/causalnex/inference/inference.py new file mode 100644 index 0000000..352dda4 --- /dev/null +++ b/causalnex/inference/inference.py @@ -0,0 +1,333 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module contains the implementation of ``InferenceEngine``. + +``InferenceEngine`` provides tools to make inferences based on interventions and observations. +""" + +import copy +import inspect +import re +import types +from typing import Callable, Dict, Hashable, Tuple, Union + +import pandas as pd + +from causalnex.ebaybbn import build_bbn +from causalnex.network import BayesianNetwork + + +class InferenceEngine: + """ + An ``InferenceEngine`` provides methods to query marginals based on observations and + make interventions (Do-Calculus) on a ``BayesianNetwork``. + + Example: + :: + >>> # Create a Bayesian Network with a manually defined DAG + >>> from causalnex.structure.structuremodel import StructureModel + >>> from causalnex.network import BayesianNetwork + >>> from causalnex.inference import InferenceEngine + >>> + >>> sm = StructureModel() + >>> sm.add_edges_from([ + >>> ('rush_hour', 'traffic'), + >>> ('weather', 'traffic') + >>> ]) + >>> data = pd.DataFrame({ + >>> 'rush_hour': [True, False, False, False, True, False, True], + >>> 'weather': ['Terrible', 'Good', 'Bad', 'Good', 'Bad', 'Bad', 'Good'], + >>> 'traffic': ['heavy', 'light', 'heavy', 'light', 'heavy', 'heavy', 'heavy'] + >>> }) + >>> bn = BayesianNetwork(sm) + >>> # Inference can only be performed on the `BayesianNetwork` with learned nodes states and CPDs + >>> bn = bn.fit_node_states_and_cpds(data) + >>> + >>> # Create an `InferenceEngine` to query marginals and make interventions + >>> ie = InferenceEngine(bn) + >>> # Query the marginals as learned from data + >>> ie.query()['traffic'] + {'heavy': 0.7142857142857142, 'light': 0.2857142857142857} + >>> # Query the marginals given observations + >>> ie.query({'rush_hour': True, 'weather': 'Terrible'})['traffic'] + {'heavy': 1.0, 'light': 0.0} + >>> # Make an intervention on the `BayesianNetwork` + >>> ie.do_intervention('rush_hour', False) + >>> # Query marginals on the intervened `BayesianNetwork` + >>> ie.query()['traffic'] + {'heavy': 0.5, 'light': 0.5} + >>> # Reset interventions + >>> ie.reset_do('rush_hour') + >>> ie.query()['traffic'] + {'heavy': 0.7142857142857142, 'light': 0.2857142857142857} + """ + + def __init__(self, bn: BayesianNetwork): + """ + Create a new ``InferenceEngine`` from an existing ``BayesianNetwork``. + + It is expected that structure and probability distribution has already been learned + for the ``BayesianNetwork`` that is to be used for inference. + This Bayesian Network cannot contain any isolated nodes. + + Args: + bn: Bayesian Network that inference will act on. + + Raises: + ValueError: if the Bayesian Network contains isolates, or if a variable name is invalid, + or if the CPDs have not been learned yet. + """ + + bad_nodes = [node for node in bn.nodes if not re.match("^[0-9a-zA-Z_]+$", node)] + if bad_nodes: + raise ValueError( + "Variable names must match ^[0-9a-zA-Z_]+$ - please fix the " + "following nodes: {0}".format(bad_nodes) + ) + + if not bn.cpds: + raise ValueError( + "Bayesian Network does not contain any CPDs. You should fit CPDs " + "before doing inference (see `BayesianNetwork.fit_cpds`)." + ) + + self._cpds = None + + self._create_cpds_dict_bn(bn) + self._generate_domains_bn(bn) + self._generate_bbn() + + def query( + self, observations: Dict[str, Hashable] = None + ) -> Dict[str, Dict[Hashable, float]]: + """ + Query the ``BayesianNetwork`` for marginals given some observations. + + Args: + observations: observed states of nodes in the Bayesian Network. + For instance, query({"node_a": 1, "node_b": 3}) + If None or {}, the marginals for all nodes in the ``BayesianNetwork`` are returned. + + Returns: + A dictionary of marginal probabilities of the network. + For instance, :math:`P(a=1) = 0.3, P(a=2) = 0.7` -> {a: {1: 0.3, 2: 0.7}} + """ + bbn_results = ( + self._bbn.query(**observations) if observations else self._bbn.query() + ) + + results = {node: dict() for node in self._cpds} + for (node, state), prob in bbn_results.items(): + results[node][state] = prob + + return results + + def _do(self, observation: str, state: Dict[Hashable, float]) -> None: + """ + Makes an intervention on the Bayesian Network. + + Args: + observation: observation that the intervention is on. + state: mapping of state -> probability. + + Raises: + ValueError: if states do not match original states of the node, or probabilities do not sum to 1. + """ + + if sum(state.values()) != 1.0: + raise ValueError("The cpd for the provided observation must sum to 1") + + if not set(state.keys()) == set(self._cpds_original[observation]): + raise ValueError( + "The cpd states do not match expected states: expected {expected}, found {found}".format( + expected=set(self._cpds_original[observation]), + found=set(state.keys()), + ) + ) + + self._cpds[observation] = {s: {(): p} for s, p in state.items()} + + def do_intervention( + self, node: str, state: Union[Hashable, Dict[Hashable, float]] = None + ) -> None: + """ + Make an intervention on the Bayesian Network. + + For instance, + `do_intervention('X', 'x')` will set :math:`P(X=x)` to 1, and :math:`P(X=y)` to 0 + `do_intervention('X', {'x': 0.2, 'y': 0.8})` will set :math:`P(X=x)` to 0.2, and :math:`P(X=y)` to 0.8 + + Args: + node: the node that the intervention acts upon. + state: state to update node it. + - if Hashable: the intervention updates the state to 1, and all other states to 0; + - if Dict[Hashable, float]: update states to all state -> probabilitiy in the dict. + + Raises: + ValueError: if performing intervention would create an isolated node. + """ + if not any( + [ + node in inspect.getargs(f.__code__)[0][1:] + for _, f in self._node_functions.items() + ] + ): + raise ValueError( + "Do calculus cannot be applied because it would result in an isolate" + ) + + if isinstance(state, int): + state = {s: float(s == state) for s in self._cpds[node]} + + self._do(node, state) + self._generate_bbn() + + def reset_do(self, observation: str) -> None: + """ + Resets any do_interventions that have been applied to the observation. + + Args: + observation: observation that will be reset. + """ + + self._cpds[observation] = self._cpds_original[observation] + self._generate_bbn() + + def _generate_bbn(self): + """Re-create the _bbn.""" + self._node_functions = self._create_node_functions() + + self._bbn = build_bbn( + list(self._node_functions.values()), domains=self._domains + ) + + def _generate_domains_bn(self, bn): + + self._domains = { + variable: list(cpd.index.values) for variable, cpd in bn.cpds.items() + } + + def _create_cpds_dict_bn(self, bn: BayesianNetwork) -> None: + """ + Map CPDs in the ``BayesianNetwork`` to required format: + + >>> {"observation": + >>> {"state": + >>> {(("condition1_observation", "condition1_state"), ("conditionN_observation", "conditionN_state")): + >>> "probability" + >>> } + >>> } + + For example, :math:`P( Colour=red | Make=fender, Model=stratocaster) = 0.4`: + >>> {"colour": + >>> {"red": + >>> {(("make", "fender"), ("model", "stratocaster")): + >>> 0.4 + >>> } + >>> } + >>> } + """ + + lookup = { + variable: { + state: { + tuple(zip(cpd.columns.names, parent_value)): cpd.loc[state][ + parent_value + ] + for parent_value in pd.MultiIndex.from_frame(cpd).names + } + for state in cpd.index.values + } + for variable, cpd in bn.cpds.items() + } + + self._cpds = lookup + self._cpds_original = copy.deepcopy(self._cpds) + + def _create_node_function(self, name: str, args: Tuple[str]): + """Creates a new function that describes a node in the ``BayesianNetwork``.""" + + def template() -> float: + """Template node function.""" + # use inspection to determine arguments to the function + # initially there are none present, but caller will add appropriate arguments to the function + # getargvalues was "inadvertently marked as deprecated in Python 3.5" + # https://docs.python.org/3/library/inspect.html#inspect.getfullargspec + arg_spec = inspect.getargvalues( # pylint: disable=deprecated-method + inspect.currentframe() + ) + + return self._cpds[arg_spec.args[0]][ # target name + arg_spec.locals[arg_spec.args[0]] + ][ # target state + tuple([(arg, arg_spec.locals[arg]) for arg in arg_spec.args[1:]]) + ] # conditions + + code = template.__code__ + template.__code__ = types.CodeType( + len(args), + code.co_kwonlyargcount, + len(args), + code.co_stacksize, + code.co_flags, + code.co_code, + code.co_consts, + code.co_names, + args, + code.co_filename, + name, + code.co_firstlineno, + code.co_lnotab, + code.co_freevars, + code.co_cellvars, + ) + template.__name__ = name + + return template + + def _create_node_functions(self) -> Dict[str, Callable]: + """Creates all functions required to create a ``BayesianNetwork``.""" + + node_functions = dict() + + for node, states in self._cpds.items(): + # since we only need condition names, which are consistent across all states, + # then we can inspect the 0th element + states_conditions = list(states.values())[0] + + # take any state, and get its conditions + state_conditions = list(states_conditions.items())[0] + condition_nodes = [n for n, v in state_conditions[0]] + + node_args = tuple([node] + condition_nodes) # type: Tuple[str] + function_name = "f_{node}".format(node=node) + node_function = self._create_node_function(function_name, node_args) + node_functions[node] = node_function + + return node_functions diff --git a/causalnex/network/__init__.py b/causalnex/network/__init__.py new file mode 100644 index 0000000..bcd9ff2 --- /dev/null +++ b/causalnex/network/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``causalnex.network`` provides functionality to learn joint probability distribution of networks. +""" + +__version__ = "0.4.0" + +__all__ = ["BayesianNetwork"] + +from .network import BayesianNetwork diff --git a/causalnex/network/network.py b/causalnex/network/network.py new file mode 100644 index 0000000..84fe5fa --- /dev/null +++ b/causalnex/network/network.py @@ -0,0 +1,572 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module contains the implementation of ``BayesianNetwork``. + +``BayesianNetwork`` is a class that represents a probabilistic, weighted, directed acyclic graph (DAG) +describing causal relationships between variables and their distribution in a factorised way. +""" + +import re +from typing import Dict, Hashable, List, Set, Tuple + +import networkx as nx +import pandas as pd +from pgmpy.estimators import BayesianEstimator, MaximumLikelihoodEstimator +from pgmpy.models import BayesianModel + +from causalnex.structure import StructureModel + + +class BayesianNetwork: + """ + Base class for Bayesian Network (BN), a probabilistic weighted DAG where nodes represent variables, + edges represent the causal relationships between variables. + + ``BayesianNetwork`` stores nodes with their possible states, edges and + conditional probability distributions (CPDs) of each node. + + ``BayesianNetwork`` is built on top of the ``StructureModel``, which is an extension of ``networkx.DiGraph`` + (see :func:`causalnex.structure.structuremodel.StructureModel`). + + In order to define the ``BayesianNetwork``, users should provide a relevant ``StructureModel``. + Once ``BayesianNetwork`` is initialised, no changes to the ``StructureModel`` can be made + and CPDs can be learned from the data. + + The learned CPDs can be then used for likelihood estimation and predictions. + + Example: + :: + >>> # Create a Bayesian Network with a manually defined DAG. + >>> from causalnex.structure import StructureModel + >>> from causalnex.network import BayesianNetwork + >>> + >>> sm = StructureModel() + >>> sm.add_edges_from([ + >>> ('rush_hour', 'traffic'), + >>> ('weather', 'traffic') + >>> ]) + >>> bn = BayesianNetwork(sm) + >>> # A created ``BayesianNetwork`` stores nodes and edges defined by the ``StructureModel`` + >>> bn.nodes + ['rush_hour', 'traffic', 'weather'] + >>> + >>> bn.edges + [('rush_hour', 'traffic'), ('weather', 'traffic')] + >>> # A ``BayesianNetwork`` doesn't store any CPDs yet + >>> bn.cpds + >>> {} + >>> + >>> # Learn the nodes' states from the data + >>> import pandas as pd + >>> data = pd.DataFrame({ + >>> 'rush_hour': [True, False, False, False, True, False, True], + >>> 'weather': ['Terrible', 'Good', 'Bad', 'Good', 'Bad', 'Bad', 'Good'], + >>> 'traffic': ['heavy', 'light', 'heavy', 'light', 'heavy', 'heavy', 'heavy'] + >>> }) + >>> bn = bn.fit_node_states(data) + >>> bn.node_states + {'rush_hour': {False, True}, 'weather': {'Bad', 'Good', 'Terrible'}, 'traffic': {'heavy', 'light'}} + >>> # Learn the CPDs from the data + >>> bn = bn.fit_cpds(data) + >>> # Use the learned CPDs to make predictions on the unseen data + >>> test_data = pd.DataFrame({ + >>> 'rush_hour': [False, False, True, True], + >>> 'weather': ['Good', 'Bad', 'Good', 'Bad'] + >>> }) + >>> bn.predict(test_data, "traffic").to_dict() + >>> {'traffic_prediction': {0: 'light', 1: 'heavy', 2: 'heavy', 3: 'heavy'}} + >>> bn.predict_probability(test_data, "traffic").to_dict() + {'traffic_prediction': {0: 'light', 1: 'heavy', 2: 'heavy', 3: 'heavy'}} + {'traffic_light': {0: 0.75, 1: 0.25, 2: 0.3333333333333333, 3: 0.3333333333333333}, + 'traffic_heavy': {0: 0.25, 1: 0.75, 2: 0.6666666666666666, 3: 0.6666666666666666}} + """ + + def __init__(self, structure: StructureModel): + """ + Create a ``BayesianNetwork`` with a DAG defined by ``StructureModel``. + + Args: + structure: a graph representing a causal relationship between variables. + In the structure + - cycles are not allowed; + - multiple (parallel) edges are not allowed; + - isolated nodes and multiple components are not allowed. + + Raises: + ValueError: If the structure is not a connected DAG. + """ + n_components = nx.number_weakly_connected_components(structure) + + if n_components > 1: + raise ValueError( + "The given structure has {n_components} separated graph components. " + "Please make sure it has only one.".format(n_components=n_components) + ) + + if not nx.is_directed_acyclic_graph(structure): + cycle = nx.find_cycle(structure) + raise ValueError( + "The given structure is not acyclic. Please review the following cycle: {cycle}".format( + cycle=cycle + ) + ) + + # _node_states is a Dict in the form `dict: {node: dict: {state: index}}`. + # Underlying libraries expect all states to be integers from zero, and + # thus this dict is used to convert from state -> idx, and then back from idx -> state as required + self._node_states = None # type: Dict[str: Dict[Hashable, int]] + self._structure = structure + + # _model is a pgmpy Bayesian Model. + # It is used for: + # - probability fitting + # - predictions + self._model = BayesianModel() + self._model.add_edges_from(structure.edges) + + @property + def structure(self) -> StructureModel: + """ + ``StructureModel`` defining the DAG of the Bayesian Network. + + Returns: + A ``StructureModel`` of the Bayesian Network. + """ + return self._structure + + @property + def nodes(self) -> List[str]: + """ + List of all nodes contained within the Bayesian Network. + + Returns: + A list of node names. + """ + return list(self._model.nodes) + + @property + def node_states(self) -> Dict[str, Set[Hashable]]: + """ + Dictionary of all states that each node can take. + + Returns: + A dictionary of node and its possible states, in format of `dict: {node: state}`. + """ + return {node: set(states.keys()) for node, states in self._node_states.items()} + + @node_states.setter + def node_states(self, nodes: Dict[str, Set[Hashable]]): + """ + Set the list of nodes that are contained within the Bayesian Network. + The states of all nodes must be provided. + + Args: + nodes: A dictionary of node and its possible states, in format of `dict: {node: state}`. + + Raises: + ValueError: if a node contains a None state. + KeyError: if a node is missing. + """ + missing_feature = set(self.nodes).difference(set(nodes.keys())) + if missing_feature: + raise KeyError( + "The data does not cover all the features found in the Bayesian Network. " + "Please check the following features: {nodes}".format( + nodes=missing_feature + ) + ) + + for node, states in nodes.items(): + if any(pd.isnull(list(states))): + raise ValueError("node '{node}' contains None state".format(node=node)) + self._node_states = { + n: {v: k for k, v in enumerate(sorted(nodes[n]))} for n in nodes + } + + @property + def edges(self) -> List[Tuple[str, str]]: + """ + List of all edges contained within the Bayesian Network, as a Tuple(from_node, to_node). + + Returns: + A list of all edges. + """ + return list(self._model.edges) + + @property + def cpds(self) -> Dict[str, pd.DataFrame]: + """ + Conditional Probability Distributions of each node within the Bayesian Network. + + The row-index of each dataframe is all possible states for the node. + The col-index of each dataframe is a MultiIndex that describes all possible permutations of parent states. + + For example, for a node :math:`P(A | B, D)`, where + .. math:: + - A \\in \\text{{"a", "b", "c", "d"}} + - B \\in \\text{{"x", "y", "z"}} + - C \\in \\text{{False, True}} + + >>> b x y z + >>> d False True False True False True + >>> a + >>> a 0.265306 0.214286 0.066667 0.25 0.444444 0.000000 + >>> b 0.183673 0.214286 0.200000 0.25 0.222222 0.666667 + >>> c 0.285714 0.285714 0.400000 0.25 0.333333 0.333333 + >>> d 0.265306 0.285714 0.333333 0.25 0.000000 0.000000 + + Returns: + Conditional Probability Distributions of each node within the Bayesian Network. + """ + cpds = dict() + for cpd in self._model.cpds: + + iterables = [ + sorted(self._node_states[var].keys()) for var in cpd.variables[1:] + ] + cols = [""] + if iterables: + cols = pd.MultiIndex.from_product(iterables, names=cpd.variables[1:]) + + cpds[cpd.variable] = pd.DataFrame( + cpd.values.reshape( + len(self._node_states[cpd.variable]), max(1, len(cols)) + ) + ) + cpds[cpd.variable][cpd.variable] = sorted( + self._node_states[cpd.variable].keys() + ) + cpds[cpd.variable].set_index([cpd.variable], inplace=True) + cpds[cpd.variable].columns = cols + + return cpds + + def fit_node_states(self, df: pd.DataFrame) -> "BayesianNetwork": + """ + Fit all states of nodes that can appear in the data. + The dataframe provided should contain every possible state (values that can be taken) for every column. + + Args: + df: data to fit node states from. Each column indicates a node and each row + an observed combination of states. + + Returns: + self + + Raises: + ValueError: if dataframe contains any missing data. + """ + self.node_states = {c: set(df[c].unique()) for c in df.columns} + + return self + + def _state_to_index( + self, df: pd.DataFrame, nodes: List[str] = None + ) -> pd.DataFrame: + """ + Transforms all values in df to an integer, as defined by the mapping from fit_node_states. + + Args: + df: data to transform + nodes: list of nodes to map to index. None means all. + + Returns: + The transformed dataframe. + + Raises: + ValueError: if nodes have not been fit, or if column names do not match node names. + """ + + df.is_copy = False + cols = nodes if nodes else df.columns + for col in cols: + df[col] = df[col].map(self._node_states[col]) + df.is_copy = True + return df + + def fit_cpds( + self, + data: pd.DataFrame, + method: str = "MaximumLikelihoodEstimator", + bayes_prior: str = None, + equivalent_sample_size: int = None, + ) -> "BayesianNetwork": + """ + Learn conditional probability distributions for all nodes in the Bayesian Network, conditioned on + their incoming edges (parents). + + Args: + data: dataframe containing one column per node in the Bayesian Network. + method: how to fit probabilities. One of: + - "MaximumLikelihoodEstimator": fit probabilities using Maximum Likelihood Estimation; + - "BayesianEstimator": fit probabilities using Bayesian Parameter Estimation. Use bayes_prior. + bayes_prior: how to construct the Bayesian prior used by method="BayesianEstimator". One of: + - "K2": shorthand for dirichlet where all pseudo_counts are 1 + regardless of variable cardinality; + - "BDeu": equivalent of using Dirichlet and using uniform 'pseudo_counts' of + `equivalent_sample_size / (node_cardinality * np.prod(parents_cardinalities))` + for each node. Use equivelant_sample_size. + equivalent_sample_size: used by BDeu bayes_prior to compute pseudo_counts. + + Returns: + self + + Raises: + ValueError: if an invalid method or bayes_prior is specified. + + """ + + transformed_data = data.copy(deep=True) # type: pd.DataFrame + transformed_data = self._state_to_index(transformed_data[self.nodes]) + + if method == "MaximumLikelihoodEstimator": + self._model.fit(data=transformed_data, estimator=MaximumLikelihoodEstimator) + + elif method == "BayesianEstimator": + valid_bayes_priors = ["BDeu", "K2"] + if bayes_prior not in valid_bayes_priors: + raise ValueError( + "unrecognised bayes_prior, please use on of %s" + % " ".join(valid_bayes_priors) + ) + + self._model.fit( + data=transformed_data, + estimator=BayesianEstimator, + prior_type=bayes_prior, + equivalent_sample_size=equivalent_sample_size, + ) + else: + valid_methods = ["MaximumLikelihoodEstimator", "BayesianEstimator"] + raise ValueError( + "unrecognised method, please use on of %s" % " ".join(valid_methods) + ) + + return self + + def fit_node_states_and_cpds( + self, + data: pd.DataFrame, + method: str = "MaximumLikelihoodEstimator", + bayes_prior: str = None, + equivalent_sample_size: int = None, + ) -> "BayesianNetwork": + """ + Call `fit_node_states` and then `fit_cpds`. + + Args: + data: dataframe containing one column per node in the Bayesian Network. + method: how to fit probabilities. One of: + - "MaximumLikelihoodEstimator": fit probabilities using Maximum Likelihood Estimation; + - "BayesianEstimator": fit probabilities using Bayesian Parameter Estimation. Use bayes_prior. + bayes_prior: how to construct the Bayesian prior used by method="BayesianEstimator". One of: + - "K2": shorthand for dirichlet where all pseudo_counts are 1 + regardless of variable cardinality; + - "BDeu": equivalent of using dirichlet and using uniform 'pseudo_counts' of + `equivalent_sample_size / (node_cardinality * np.prod(parents_cardinalities))` + for each node. Use equivelant_sample_size. + equivalent_sample_size: used by BDeu bayes_prior to compute pseudo_counts. + + Returns: + self + """ + + return self.fit_node_states(data).fit_cpds( + data, method, bayes_prior, equivalent_sample_size + ) + + def predict(self, data: pd.DataFrame, node: str) -> pd.DataFrame: + """ + Predict the state of a node based on some input data, using the Bayesian Network. + + Args: + data: data to make prediction. + node: the node to predict. + + Returns: + A dataframe of predictions, containing a single column name {node}_prediction. + """ + + if all(parent in data.columns for parent in self._model.get_parents(node)): + return self._predict_from_complete_data(data, node) + + return self._predict_from_incomplete_data(data, node) + + def _predict_from_complete_data( + self, data: pd.DataFrame, node: str + ) -> pd.DataFrame: + """ + Predicts state of node given all parents of node exist within data. + This method inspects the CPD of node directly, since all parent states are known. + This avoids traversing the full network to compute marginals. + This method is fast. + + Args: + data: data to make prediction. + node: the node to predict. + + Returns: + A dataframe of predictions, containing a single column named {node}_prediction. + """ + transformed_data = data.copy(deep=True) # type: pd.DataFrame + + parents = sorted(self._model.get_parents(node)) + cpd = self.cpds[node] + + transformed_data[ + "{node}_prediction".format(node=node) + ] = transformed_data.apply( + lambda row: cpd[tuple([row[parent] for parent in parents])].idxmax() + if parents + else cpd[""].idxmax(), + axis=1, + ) + return transformed_data[[node + "_prediction"]] + + def _predict_from_incomplete_data( + self, data: pd.DataFrame, node: str + ) -> pd.DataFrame: + """ + Predicts state of node when some parents of node do not exist within data. + This method uses the pgmpy predict function, which predicts the most likely state for every node + that is not contained within data. + With incomplete data, pgmpy goes beyond parents in the network to determine the most likely predictions. + This method is slow. + + Args: + data: data to make prediction. + node: the node to predict. + + Returns: + A dataframe of predictions, containing a single column name {node}_prediction. + """ + + transformed_data = data.copy(deep=True) # type: pd.DataFrame + self._state_to_index(transformed_data) + + # pgmpy will predict all missing data, so drop column we want to predict + transformed_data.drop(node, axis=1, inplace=True) + + predictions = self._model.predict(transformed_data)[[node]] + + return predictions.rename(columns={node: node + "_prediction"}) + + def predict_probability(self, data: pd.DataFrame, node: str) -> pd.DataFrame: + """ + Predict the probability of each possible state of a node, based on some input data. + + Args: + data: data to make prediction. + node: the node to predict probabilities. + + Returns: + A dataframe of predicted probabilities, contained one column per possible state, named {node}_{state}. + """ + + if all(parent in data.columns for parent in self._model.get_parents(node)): + return self._predict_probability_from_complete_data(data, node) + + return self._predict_probability_from_incomplete_data(data, node) + + def _predict_probability_from_complete_data( + self, data: pd.DataFrame, node: str + ) -> pd.DataFrame: + """ + Predict the probability of each possible state of a node, based on some input data. + This method inspects the CPD of node directly, since all parent states are known. + This avoids traversing the full network to compute marginals. + This method is fast. + + Args: + data: data to make prediction. + node: the node to predict probabilities. + + Returns: + A dataframe of predicted probabilities, contained one column per possible state, named {node}_{state}. + """ + transformed_data = data.copy(deep=True) # type: pd.DataFrame + + parents = sorted(self._model.get_parents(node)) + cpd = self.cpds[node] + + def lookup_probability(row, s): + """Retrieve probability from CPD""" + if parents: + return cpd[tuple([row[parent] for parent in parents])].loc[s] + return cpd.at[s, ""] + + for state in self.node_states[node]: + transformed_data[ + "{n}_{s}".format(n=node, s=state) + ] = transformed_data.apply( + lambda row, st=state: lookup_probability(row, st), axis=1 + ) + + return transformed_data[ + ["{n}_{s}".format(n=node, s=state) for state in self.node_states[node]] + ] + + def _predict_probability_from_incomplete_data( + self, data: pd.DataFrame, node: str + ) -> pd.DataFrame: + """ + Predict the probability of each possible state of a node, based on some input data. + This method uses the pgmpy predict_probability function, which predicts the probability + of every state for every node that is not contained within data. + With incomplete data, pgmpy goes beyond parents in the network to determine the most likely predictions. + This method is slow. + + Args: + data: data to make prediction. + node: the node to predict probabilities. + + Returns: + A dataframe of predicted probabilities, contained one column per possible state, named {node}_{state}. + """ + transformed_data = data.copy(deep=True) # type: pd.DataFrame + self._state_to_index(transformed_data) + + # pgmpy will predict all missing data, so drop column we want to predict + transformed_data.drop(node, axis=1, inplace=True) + + probability = self._model.predict_probability( + transformed_data + ) # type: pd.DataFrame + + # keep only probabilities for the node we are interested in + cols = [] + pattern = re.compile("^{node}_[0-9]+$".format(node=node)) + # disabled open pylint issue (https://github.com/PyCQA/pylint/issues/2962) + for col in probability.columns: # pylint: disable=E1133 + if pattern.match(col): + cols.append(col) + probability = probability[cols] + probability.columns = cols + + return probability diff --git a/causalnex/plots/__init__.py b/causalnex/plots/__init__.py new file mode 100644 index 0000000..0657d97 --- /dev/null +++ b/causalnex/plots/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``causalnex.plots`` provides functionality to visualise structure models. +""" + +__version__ = "0.4.0" + +__all__ = ["plot_structure"] + +from .plots import plot_structure diff --git a/causalnex/plots/plots.py b/causalnex/plots/plots.py new file mode 100644 index 0000000..29bc478 --- /dev/null +++ b/causalnex/plots/plots.py @@ -0,0 +1,116 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +"""Plot Methods.""" +from typing import Dict, List, Tuple + +import matplotlib.pyplot as plt +import networkx as nx +from matplotlib.axes import Axes +from matplotlib.figure import Figure + +from causalnex.structure.structuremodel import StructureModel + + +def _setup_plot(ax: plt.Axes = None, title: str = None) -> (plt.Figure, plt.Axes): + """Initial setup of fig and ax to plot to.""" + + if not ax: + fig = plt.figure() # type: plt.Figure + ax = fig.add_subplot(1, 1, 1) # type: plt.Axes + + if title: + ax.set_title(title) + + return ax.get_figure(), ax + + +def plot_structure( + g: StructureModel, + ax: plt.Axes = None, + title: str = None, + show_labels: bool = True, + node_color: str = "r", + edge_color: str = "k", + label_color: str = "k", + node_positions: Dict[str, List[float]] = None, +) -> Tuple[Figure, Axes, Dict[str, List[float]]]: + """Plot the structure model to visualise the relationships between nodes. + + Args: + g: the structure model to plot. + ax: if provided then figure will be drawn to this Axes, otherwise a new Axes will be created. + title: if provided then the title will be drawn on the plot. + show_labels: if True then node labels will be drawn. + node_color: a single color format string, for example 'r' or '#ff0000'. default "r". + edge_color: a single color format string, for example 'r' or '#ff0000'. default "k". + label_color: a single color format string, for example 'r' or '#ff0000'. default "k". + node_positions: coordinates for node positions, ie {"node_a": [0, 0]}. + + Returns: + fig, ax, node_positions. + + Example: + :: + >>> # Create a Bayesian Network with a manually defined DAG. + >>> from causalnex.structure import StructureModel + >>> from causalnex.network import BayesianNetwork + >>> + >>> sm = StructureModel() + >>> sm.add_edges_from([ + >>> ('rush_hour', 'traffic'), + >>> ('weather', 'traffic') + >>> ]) + >>> from causalnex.plots import plot_structure + >>> plot_structure(sm) + """ + + fig, ax = _setup_plot(ax, title) + + if not node_positions: + node_positions = nx.circular_layout(g) + + node_color = node_color if node_color else "r" + edge_color = edge_color if edge_color else "k" + label_color = label_color if label_color else "k" + + nx.draw_networkx_nodes( + g, node_positions, ax=ax, nodelist=g.nodes, node_color=node_color + ) + + for u, v in g.edges: + nx.draw_networkx_edges( + g, node_positions, ax=ax, edgelist=[(u, v)], edge_color=edge_color + ) + + if show_labels: + nx.draw_networkx_labels(g, node_positions, ax=ax, font_color=label_color) + + ax.set_axis_off() + plt.tight_layout() + + return fig, ax, node_positions diff --git a/causalnex/structure/__init__.py b/causalnex/structure/__init__.py new file mode 100644 index 0000000..b30fc19 --- /dev/null +++ b/causalnex/structure/__init__.py @@ -0,0 +1,37 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +``causalnex.structure`` provides functionality to define or learn structure. +""" + +__version__ = "0.4.0" + +__all__ = ["StructureModel", "notears"] + +from .structuremodel import StructureModel diff --git a/causalnex/structure/notears.py b/causalnex/structure/notears.py new file mode 100644 index 0000000..febc99f --- /dev/null +++ b/causalnex/structure/notears.py @@ -0,0 +1,553 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# The methods found in this file are derived from a repository under Apache 2.0: +# DAGs with NO TEARS. +# @inproceedings{zheng2018dags, +# author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P.}, +# booktitle = {Advances in Neural Information Processing Systems}, +# title = {{DAGs with NO TEARS: Continuous Optimization for Structure Learning}}, +# year = {2018}, +# codebase = {https://github.com/xunzheng/notears} +# } +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Tools to learn a ``StructureModel`` which describes the conditional dependencies between variables in a dataset. +""" + +import logging +import warnings +from copy import deepcopy +from typing import List, Tuple + +import numpy as np +import pandas as pd +import scipy.linalg as slin +import scipy.optimize as sopt + +from causalnex.structure.structuremodel import StructureModel + +__all__ = ["from_numpy", "from_pandas", "from_numpy_lasso", "from_pandas_lasso"] + + +def from_numpy( + X: np.ndarray, + max_iter: int = 100, + h_tol: float = 1e-8, + w_threshold: float = 0.0, + tabu_edges: List[Tuple[int, int]] = None, + tabu_parent_nodes: List[int] = None, + tabu_child_nodes: List[int] = None, +) -> StructureModel: + """ + Learn the `StructureModel`, the graph structure describing conditional dependencies between variables + in data presented as a numpy array. + + The optimisation is to minimise a score function :math:`F(W)` over the graph's + weighted adjacency matrix, :math:`W`, subject to the a constraint function :math:`h(W)`, + where :math:`h(W) == 0` characterises an acyclic graph. + :math:`h(W) > 0` is a continuous, differentiable function that encapsulated how acyclic the graph is + (less == more acyclic). + Full details of this approach to structure learning are provided in the publication: + + Based on DAGs with NO TEARS. + @inproceedings{zheng2018dags, + author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P.}, + booktitle = {Advances in Neural Information Processing Systems}, + title = {{DAGs with NO TEARS: Continuous Optimization for Structure Learning}}, + year = {2018}, + codebase = {https://github.com/xunzheng/notears} + } + + Args: + X: 2d input data, axis=0 is data rows, axis=1 is data columns. Data must be row oriented. + max_iter: max number of dual ascent steps during optimisation. + h_tol: exit if h(W) < h_tol (as opposed to strict definition of 0). + w_threshold: fixed threshold for absolute edge weights. + tabu_edges: list of edges(from, to) not to be included in the graph. + tabu_parent_nodes: list of nodes banned from being a parent of any other nodes. + tabu_child_nodes: list of nodes banned from being a child of any other nodes. + + Returns: + StructureModel: a graph of conditional dependencies between data variables. + + Raises: + ValueError: If X does not contain data. + """ + + # n examples, d properties + _, d = X.shape + + bnds = [ + (0, 0) + if i == j + else (0, 0) + if tabu_edges is not None and (i, j) in tabu_edges + else (0, 0) + if tabu_parent_nodes is not None and i in tabu_parent_nodes + else (0, 0) + if tabu_child_nodes is not None and j in tabu_child_nodes + else (None, None) + for i in range(d) + for j in range(d) + ] + + return _learn_structure(X, bnds, max_iter, h_tol, w_threshold) + + +def from_numpy_lasso( + X: np.ndarray, + beta: float, + max_iter: int = 100, + h_tol: float = 1e-8, + w_threshold: float = 0.0, + tabu_edges: List[Tuple[int, int]] = None, + tabu_parent_nodes: List[int] = None, + tabu_child_nodes: List[int] = None, +) -> StructureModel: + """ + Learn the `StructureModel`, the graph structure with lasso regularisation + describing conditional dependencies between variables in data presented as a numpy array. + + Based on DAGs with NO TEARS. + @inproceedings{zheng2018dags, + author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P.}, + booktitle = {Advances in Neural Information Processing Systems}, + title = {{DAGs with NO TEARS: Continuous Optimization for Structure Learning}}, + year = {2018}, + codebase = {https://github.com/xunzheng/notears} + } + + Args: + X: 2d input data, axis=0 is data rows, axis=1 is data columns. Data must be row oriented. + beta: Constant that multiplies the lasso term. + max_iter: max number of dual ascent steps during optimisation. + h_tol: exit if h(W) < h_tol (as opposed to strict definition of 0). + w_threshold: fixed threshold for absolute edge weights. + tabu_edges: list of edges(from, to) not to be included in the graph. + tabu_parent_nodes: list of nodes banned from being a parent of any other nodes. + tabu_child_nodes: list of nodes banned from being a child of any other nodes. + + Returns: + StructureModel: a graph of conditional dependencies between data variables. + + Raises: + ValueError: If X does not contain data. + """ + + # n examples, d properties + _, d = X.shape + + bnds = [ + (0, 0) + if i == j + else (0, 0) + if tabu_edges is not None and (i, j) in tabu_edges + else (0, 0) + if tabu_parent_nodes is not None and i in tabu_parent_nodes + else (0, 0) + if tabu_child_nodes is not None and j in tabu_child_nodes + else (None, None) + for i in range(d) + for j in range(d) + ] * 2 + + return _learn_structure_lasso(X, beta, bnds, max_iter, h_tol, w_threshold) + + +def from_pandas( + X: pd.DataFrame, + max_iter: int = 100, + h_tol: float = 1e-8, + w_threshold: float = 0.0, + tabu_edges: List[Tuple[str, str]] = None, + tabu_parent_nodes: List[str] = None, + tabu_child_nodes: List[str] = None, +) -> StructureModel: + """ + Learn the `StructureModel`, the graph structure describing conditional dependencies between variables + in data presented as a pandas dataframe. + + The optimisation is to minimise a score function :math:`F(W)` over the graph's + weighted adjacency matrix, :math:`W`, subject to the a constraint function :math:`h(W)`, + where :math:`h(W) == 0` characterises an acyclic graph. + :math:`h(W) > 0` is a continuous, differentiable function that encapsulated how acyclic the graph is + (less == more acyclic). + Full details of this approach to structure learning are provided in the publication: + + Based on DAGs with NO TEARS. + @inproceedings{zheng2018dags, + author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P.}, + booktitle = {Advances in Neural Information Processing Systems}, + title = {{DAGs with NO TEARS: Continuous Optimization for Structure Learning}}, + year = {2018}, + codebase = {https://github.com/xunzheng/notears} + } + + Args: + X: input data. + max_iter: max number of dual ascent steps during optimisation. + h_tol: exit if h(W) < h_tol (as opposed to strict definition of 0). + w_threshold: fixed threshold for absolute edge weights. + tabu_edges: list of edges(from, to) not to be included in the graph. + tabu_parent_nodes: list of nodes banned from being a parent of any other nodes. + tabu_child_nodes: list of nodes banned from being a child of any other nodes. + + Returns: + StructureModel: graph of conditional dependencies between data variables. + + Raises: + ValueError: If X does not contain data. + """ + + data = deepcopy(X) + + non_numeric_cols = data.select_dtypes(exclude="number").columns + + if len(non_numeric_cols) > 0: + raise ValueError( + "All columns must have numeric data. " + "Consider mapping the following columns to int {non_numeric_cols}".format( + non_numeric_cols=non_numeric_cols + ) + ) + + col_idx = {c: i for i, c in enumerate(data.columns)} + idx_col = {i: c for c, i in col_idx.items()} + + if tabu_edges: + tabu_edges = [(col_idx[u], col_idx[v]) for u, v in tabu_edges] + if tabu_parent_nodes: + tabu_parent_nodes = [col_idx[n] for n in tabu_parent_nodes] + if tabu_child_nodes: + tabu_child_nodes = [col_idx[n] for n in tabu_child_nodes] + + g = from_numpy( + data.values, + max_iter, + h_tol, + w_threshold, + tabu_edges, + tabu_parent_nodes, + tabu_child_nodes, + ) + + sm = StructureModel() + sm.add_nodes_from(data.columns) + sm.add_weighted_edges_from( + [(idx_col[u], idx_col[v], w) for u, v, w in g.edges.data("weight")], + origin="learned", + ) + + return sm + + +def from_pandas_lasso( + X: pd.DataFrame, + beta: float, + max_iter: int = 100, + h_tol: float = 1e-8, + w_threshold: float = 0.0, + tabu_edges: List[Tuple[str, str]] = None, + tabu_parent_nodes: List[str] = None, + tabu_child_nodes: List[str] = None, +) -> StructureModel: + """ + Learn the `StructureModel`, the graph structure with lasso regularisation + describing conditional dependencies between variables in data presented as a pandas dataframe. + + Based on DAGs with NO TEARS. + @inproceedings{zheng2018dags, + author = {Zheng, Xun and Aragam, Bryon and Ravikumar, Pradeep and Xing, Eric P.}, + booktitle = {Advances in Neural Information Processing Systems}, + title = {{DAGs with NO TEARS: Continuous Optimization for Structure Learning}}, + year = {2018}, + codebase = {https://github.com/xunzheng/notears} + } + + Args: + X: input data. + beta: Constant that multiplies the lasso term. + max_iter: max number of dual ascent steps during optimisation. + h_tol: exit if h(W) < h_tol (as opposed to strict definition of 0). + w_threshold: fixed threshold for absolute edge weights. + tabu_edges: list of edges(from, to) not to be included in the graph. + tabu_parent_nodes: list of nodes banned from being a parent of any other nodes. + tabu_child_nodes: list of nodes banned from being a child of any other nodes. + + Returns: + StructureModel: graph of conditional dependencies between data variables. + + Raises: + ValueError: If X does not contain data. + """ + + data = deepcopy(X) + + non_numeric_cols = data.select_dtypes(exclude="number").columns + + if not non_numeric_cols.empty: + raise ValueError( + "All columns must have numeric data. " + "Consider mapping the following columns to int {non_numeric_cols}".format( + non_numeric_cols=non_numeric_cols + ) + ) + + col_idx = {c: i for i, c in enumerate(data.columns)} + idx_col = {i: c for c, i in col_idx.items()} + + if tabu_edges: + tabu_edges = [(col_idx[u], col_idx[v]) for u, v in tabu_edges] + if tabu_parent_nodes: + tabu_parent_nodes = [col_idx[n] for n in tabu_parent_nodes] + if tabu_child_nodes: + tabu_child_nodes = [col_idx[n] for n in tabu_child_nodes] + + g = from_numpy_lasso( + data.values, + beta, + max_iter, + h_tol, + w_threshold, + tabu_edges, + tabu_parent_nodes, + tabu_child_nodes, + ) + + sm = StructureModel() + sm.add_nodes_from(data.columns) + sm.add_weighted_edges_from( + [(idx_col[u], idx_col[v], w) for u, v, w in g.edges.data("weight")], + origin="learned", + ) + + return sm + + +def _learn_structure( + X: np.ndarray, + bnds, + max_iter: int = 100, + h_tol: float = 1e-8, + w_threshold: float = 0.0, +) -> StructureModel: + """ + Based on initial implementation at https://github.com/xunzheng/notears + """ + + def _h(w: np.ndarray) -> float: + """ + Constraint function of the NOTEARS algorithm. + + Args: + w: current adjacency matrix. + + Returns: + float: DAGness of the adjacency matrix (0 == DAG, >0 == cyclic). + """ + + W = w.reshape([d, d]) + return np.trace(slin.expm(W * W)) - d + + def _func(w: np.ndarray) -> float: + """ + Objective function that the NOTEARS algorithm tries to minimise. + + Args: + w: current adjacency matrix. + + Returns: + float: objective. + """ + + W = w.reshape([d, d]) + loss = 0.5 / n * np.square(np.linalg.norm(X.dot(np.eye(d, d) - W), "fro")) + h = _h(W) + return loss + 0.5 * rho * h * h + alpha * h + + def _grad(w: np.ndarray) -> np.ndarray: + """ + Gradient function used to compute next step in NOTEARS algorithm. + + Args: + w: the current adjacency matrix. + + Returns: + np.ndarray: gradient vector. + """ + + W = w.reshape([d, d]) + loss_grad = -1.0 / n * X.T.dot(X).dot(np.eye(d, d) - W) + E = slin.expm(W * W) + obj_grad = loss_grad + (rho * (np.trace(E) - d) + alpha) * E.T * W * 2 + return obj_grad.flatten() + + if X.size == 0: + raise ValueError("Input data X is empty, cannot learn any structure") + logging.info("Learning structure using 'NOTEARS' optimisation.") + + # n examples, d properties + n, d = X.shape + # initialise matrix to zeros + w_est, w_new = np.zeros(d * d), np.zeros(d * d) + + # initialise weights and constraints + rho, alpha, h, h_new = 1.0, 0.0, np.inf, np.inf + + # start optimisation + for n_iter in range(max_iter): + while rho < 1e20: + sol = sopt.minimize(_func, w_est, method="L-BFGS-B", jac=_grad, bounds=bnds) + w_new = sol.x + h_new = _h(w_new) + if h_new > 0.25 * h: + rho *= 10 + else: + break + w_est, h = w_new, h_new + alpha += rho * h + if h <= h_tol: + break + if h > h_tol and n_iter == max_iter - 1: + warnings.warn("Failed to converge. Consider increasing max_iter.") + + w_est[np.abs(w_est) <= w_threshold] = 0 + return StructureModel(w_est.reshape([d, d])) + + +def _learn_structure_lasso( + X: np.ndarray, + beta: float, + bnds, + max_iter: int = 100, + h_tol: float = 1e-8, + w_threshold: float = 0.0, +) -> StructureModel: + """ + Based on initial implementation at https://github.com/xunzheng/notears + """ + + def _h(w_vec: np.ndarray) -> float: + """ + Constraint function of the NOTEARS algorithm with lasso regularisation. + + Args: + w_vec: weight vector (wpos and wneg). + + Returns: + float: DAGness of the adjacency matrix (0 == DAG, >0 == cyclic). + """ + + W = w_vec.reshape([d, d]) + return np.trace(slin.expm(W * W)) - d + + def _func(w_vec: np.ndarray) -> float: + """ + Objective function that the NOTEARS algorithm with lasso regularisation tries to minimise. + + Args: + w_vec: weight vector (wpos and wneg). + + Returns: + float: objective. + """ + + w_pos = w_vec[: d ** 2] + w_neg = w_vec[d ** 2 :] + + wmat_pos = w_pos.reshape([d, d]) + wmat_neg = w_neg.reshape([d, d]) + + wmat = wmat_pos - wmat_neg + loss = 0.5 / n * np.square(np.linalg.norm(X.dot(np.eye(d, d) - wmat), "fro")) + h_val = _h(wmat) + return loss + 0.5 * rho * h_val * h_val + alpha * h_val + beta * w_vec.sum() + + def _grad(w_vec: np.ndarray) -> np.ndarray: + """ + Gradient function used to compute next step in NOTEARS algorithm with lasso regularisation. + + Args: + w_vec: weight vector (wpos and wneg). + + Returns: + np.ndarray: gradient vector. + """ + + w_pos = w_vec[: d ** 2] + w_neg = w_vec[d ** 2 :] + + grad_vec = np.zeros(2 * d ** 2) + wmat_pos = w_pos.reshape([d, d]) + wmat_neg = w_neg.reshape([d, d]) + + wmat = wmat_pos - wmat_neg + + loss_grad = -1.0 / n * X.T.dot(X).dot(np.eye(d, d) - wmat) + exp_hdmrd = slin.expm(wmat * wmat) + obj_grad = ( + loss_grad + + (rho * (np.trace(exp_hdmrd) - d) + alpha) * exp_hdmrd.T * wmat * 2 + ) + lbd_grad = beta * np.ones(d * d) + grad_vec[: d ** 2] = obj_grad.flatten() + lbd_grad + grad_vec[d ** 2 :] = -obj_grad.flatten() + lbd_grad + + return grad_vec + + if X.size == 0: + raise ValueError("Input data X is empty, cannot learn any structure") + logging.info( + "Learning structure using 'NOTEARS' optimisation with lasso regularisation." + ) + + n, d = X.shape + w_est, w_new = np.zeros(2 * d * d), np.zeros(2 * d * d) + rho, alpha, h_val, h_new = 1.0, 0.0, np.inf, np.inf + for n_iter in range(max_iter): + while rho < 1e20: + sol = sopt.minimize(_func, w_est, method="L-BFGS-B", jac=_grad, bounds=bnds) + w_new = sol.x + + h_new = _h( + w_new[: d ** 2].reshape([d, d]) - w_new[d ** 2 :].reshape([d, d]) + ) + if h_new > 0.25 * h_val: + rho *= 10 + else: + break + w_est, h_val = w_new, h_new + alpha += rho * h_val + if h_val <= h_tol: + break + if h_val > h_tol and n_iter == max_iter - 1: + warnings.warn("Failed to converge. Consider increasing max_iter.") + + w_new = w_est[: d ** 2].reshape([d, d]) - w_est[d ** 2 :].reshape([d, d]) + w_new[np.abs(w_new) < w_threshold] = 0 + return StructureModel(w_new.reshape([d, d])) diff --git a/causalnex/structure/structuremodel.py b/causalnex/structure/structuremodel.py new file mode 100644 index 0000000..4cf0d23 --- /dev/null +++ b/causalnex/structure/structuremodel.py @@ -0,0 +1,269 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module contains the implementation of ``StructureModel``. + +``StructureModel`` is a class that describes relationships between variables as a graph. +""" + +from typing import List, Set, Union + +import networkx as nx +import numpy as np + + +def _validate_origin(origin: str) -> None: + """ + Checks that origin has a valid value. One of: + - unknown: edge exists for an unknown reason; + - learned: edge was created as the output of a machine-learning process; + - expert: edge was created by a domain expert. + + Args: + origin: the value to validate. + + Raises: + ValueError: if origin is not valid. + """ + + allowed = {"unknown", "learned", "expert"} + + if origin not in allowed: + raise ValueError( + "Unknown origin: must be one of {allowed} - got `{origin}`.".format( + allowed=allowed, origin=origin + ) + ) + + +class StructureModel(nx.DiGraph): + """ + Base class for structure models, which are an extension of ``networkx.DiGraph``. + + A ``StructureModel`` stores nodes and edges with optional data, or attributes. + + Edges have one required attribute, "origin", which describes how the edge was created. + Origin can be one of either unknown, learned, or expert. + + StructureModel hold directed edges, describing a cause -> effect relationship. + Cycles are permitted within a ``StructureModel``. + + Nodes can be arbitrary (hashable) Python objects with optional key/value attributes. + By convention None is not used as a node. + + Edges are represented as links between nodes with optional key/value attributes. + """ + + def __init__(self, incoming_graph_data=None, origin="unknown", **attr): + """ + Create a ``StructureModel`` with incoming_graph_data, which has come from some origin. + + Args: + incoming_graph_data (Optional): input graph (optional, default: None) + Data to initialize graph. If None (default) an empty graph is created. + The data can be any format that is supported by the to_networkx_graph() + function, currently including edge list, dict of dicts, dict of lists, + NetworkX graph, NumPy matrix or 2d ndarray, SciPy sparse matrix, or PyGraphviz graph. + + origin (str): label for how the edges were created. Can be one of: + - unknown: edges exist for an unknown reason; + - learned: edges were created as the output of a machine-learning process; + - expert: edges were created by a domain expert. + + attr : Attributes to add to graph as key/value pairs (no attributes by default). + """ + + _validate_origin(origin) + super().__init__(incoming_graph_data, **attr) + for u_of_edge, v_of_edge in self.edges: + self[u_of_edge][v_of_edge]["origin"] = origin + + def to_directed_class(self): + """ + Returns the class to use for directed copies. + See :func:`networkx.DiGraph.to_directed()`. + """ + return StructureModel + + def to_undirected_class(self): + """ + Returns the class to use for undirected copies. + See :func:`networkx.DiGraph.to_undirected()`. + """ + return nx.Graph + + # disabled: W0221: Parameters differ from overridden 'add_edge' method (arguments-differ) + # this has been disabled because origin tracking is required for CausalGraphs + # implementing it in this way allows all 3rd party libraries and applications to + # integrate seamlessly, where edges will be given origin="unknown" where not provided + def add_edge( + self, u_of_edge: str, v_of_edge: str, origin: str = "unknown", **attr + ): # pylint: disable=W0221 + """ + Adds a causal relationship from u to v. + + If u or v do not currently exists in the ``StructureModel`` then they will be created. + + By default a relationship will be given origin="unknown", but + may also be given "learned" or "expert" origin. + + Adding an edge that already exists will replace the existing edge. + See :func:`networkx.DiGraph.add_edge`. + + Args: + u_of_edge: causal node. + v_of_edge: effect node. + origin: label for how the edge was created. Can be one of: + - unknown: edge exists for an unknown reason; + - learned: edge was created as the output of a machine-learning process; + - expert: edge was created by a domain expert. + **attr: Attributes to add to edge as key/value pairs (no attributes by default). + """ + _validate_origin(origin) + + attr.update({"origin": origin}) + super().add_edge(u_of_edge, v_of_edge, **attr) + + # disabled: W0221: Parameters differ from overridden 'add_edge' method (arguments-differ) + # this has been disabled because origin tracking is required for CausalGraphs + # implementing it in this way allows all 3rd party libraries and applications to + # integrate seamlessly, where edges will be given origin="unknown" where not provided + def add_edges_from( + self, + ebunch_to_add: Union[Set[tuple], List[tuple]], + origin: str = "unknown", + **attr + ): # pylint: disable=W0221 + """ + Adds a bunch of causal relationships, u -> v. + + If u or v do not currently exists in the ``StructureModel`` then they will be created. + + By default relationships will be given origin="unknown", + but may also be given "learned" or "expert" origin. + + Notes: + Adding an edge that already exists will replace the existing edge. + See :func:`networkx.DiGraph.add_edges_from`. + + Args: + ebunch_to_add: container of edges. + Each edge given in the container will be added to the graph. + The edges must be given as 2-tuples (u, v) or + 3-tuples (u, v, d) where d is a dictionary containing edge data. + origin: label for how the edges were created. One of: + - unknown: edges exist for an unknown reason. + - learned: edges were created as the output of a machine-learning process. + - expert: edges were created by a domain expert. + **attr: Attributes to add to edge as key/value pairs (no attributes by default). + """ + + _validate_origin(origin) + + attr.update({"origin": origin}) + super().add_edges_from(ebunch_to_add, **attr) + + # disabled: W0221: Parameters differ from overridden 'add_edge' method (arguments-differ) + # this has been disabled because origin tracking is required for CausalGraphs + # implementing it in this way allows all 3rd party libraries and applications to + # integrate seamlessly, where edges will be given origin="unknown" where not provided + def add_weighted_edges_from( + self, + ebunch_to_add: Union[Set[tuple], List[tuple]], + weight: str = "weight", + origin: str = "unknown", + **attr + ): # pylint: disable=W0221 + """ + Adds a bunch of weighted causal relationships, u -> v. + + If u or v do not currently exists in the ``StructureModel`` then they will be created. + + By default relationships will be given origin="unknown", + but may also be given "learned" or "expert" origin. + + Notes: + Adding an edge that already exists will replace the existing edge. + See :func:`networkx.DiGraph.add_edges_from`. + + Args: + ebunch_to_add: container of edges. + Each edge given in the container will be added to the graph. + The edges must be given as 2-tuples (u, v) or + 3-tuples (u, v, d) where d is a dictionary containing edge data. + weight : string, optional (default='weight'). + The attribute name for the edge weights to be added. + origin: label for how the edges were created. One of: + - unknown: edges exist for an unknown reason; + - learned: edges were created as the output of a machine-learning process; + - expert: edges were created by a domain expert. + **attr: Attributes to add to edge as key/value pairs (no attributes by default). + """ + _validate_origin(origin) + + attr.update({"origin": origin}) + super().add_weighted_edges_from(ebunch_to_add, weight=weight, **attr) + + def edges_with_origin(self, origin) -> list: + """ + List of edges created with given origin attribute. + + Returns: + A list of edges with the given origin. + """ + + return [(u, v) for u, v in self.edges if self[u][v]["origin"] == origin] + + def remove_edges_below_threshold(self, threshold: float): + """ + Remove edges whose absolute weights are less than a defined threshold. + + Args: + threshold: edges whose absolute weight is less than this value are removed. + """ + + self.remove_edges_from( + [(u, v) for u, v, w in self.edges(data="weight") if np.abs(w) < threshold] + ) + + def get_largest_subgraph(self) -> "StructureModel": + """ + Get the largest subgraph of the Structure Model. + + Returns: + The largest subgraph of the Structure Model. If no subgraph exists, None is returned. + """ + largest_n_edges = 0 + largest_subgraph = None + + for subgraph in nx.weakly_connected_component_subgraphs(self): + if len(subgraph.edges) > largest_n_edges: + largest_n_edges = len(subgraph.edges) + largest_subgraph = subgraph + + return largest_subgraph diff --git a/doc_requirements.txt b/doc_requirements.txt new file mode 100644 index 0000000..f60e9a6 --- /dev/null +++ b/doc_requirements.txt @@ -0,0 +1,12 @@ +click>=7.0, <8.0 +ipykernel>=4.8.1, <5.0 +jupyter_client>=5.1.0, <6.0 +nbsphinx==0.4.2 +nbstripout==0.3.3 +patchy>=1.5, <2.0 +recommonmark==0.5.0 +sphinx-autodoc-typehints>=1.6.0, < 2.0 +sphinx-markdown-tables==0.0.9 +sphinx>=1.8.4, <2.0 +sphinx_copybutton==0.2.5 +sphinx_rtd_theme==0.4.3 diff --git a/docs/_templates/autosummary/base.rst b/docs/_templates/autosummary/base.rst new file mode 100644 index 0000000..b7556eb --- /dev/null +++ b/docs/_templates/autosummary/base.rst @@ -0,0 +1,5 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. auto{{ objtype }}:: {{ objname }} diff --git a/docs/_templates/autosummary/class.rst b/docs/_templates/autosummary/class.rst new file mode 100644 index 0000000..aa25df8 --- /dev/null +++ b/docs/_templates/autosummary/class.rst @@ -0,0 +1,33 @@ +{{ fullname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :undoc-members: + :show-inheritance: + :inherited-members: + + {% block attributes %} + {% if attributes %} + .. rubric:: Attributes + + .. autosummary:: + {% for item in all_attributes %} + {%- if not item.startswith('_') %} + {{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block methods %} + {% if methods %} + .. rubric:: Methods + + .. autosummary:: + {% for item in methods %} + {{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/_templates/autosummary/module.rst b/docs/_templates/autosummary/module.rst new file mode 100644 index 0000000..68f7527 --- /dev/null +++ b/docs/_templates/autosummary/module.rst @@ -0,0 +1,58 @@ +{{ fullname | escape | underline }} + +.. rubric:: Description + +.. automodule:: {{ fullname }} + + {% block public_modules %} + {% if public_modules %} + .. rubric:: Modules + + .. autosummary:: + :toctree: + :template: autosummary/module.rst + {% for item in public_modules %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: Functions + + .. autosummary:: + :toctree: + {% for item in functions %} + {%- if not item.startswith('_') %} + {{ item }} + {% endif %} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: Classes + + .. autosummary:: + :toctree: + :template: autosummary/class.rst + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: Exceptions + + .. autosummary:: + :toctree: + :template: autosummary/class.rst + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/build-docs.sh b/docs/build-docs.sh new file mode 100755 index 0000000..e96218c --- /dev/null +++ b/docs/build-docs.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +python -m ipykernel install --user --name=causalnex --display-name=causalnex + +# Move some files around. We need a separate build directory, which would +# have all the files, build scripts would shuffle the files, +# we don't want that happening on the actual code locally. +# When running on ReadTheDocs, sphinx-build would run directly on the original files, +# but we don't care about the code state there. +rm -rf docs/build +mkdir docs/build/ +cp -r docs/_templates docs/conf.py docs/build/ + +sphinx-build -c docs/ -Ea -j auto -D language=en docs/build/ docs/build/html diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..506e091 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,503 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# causalnex documentation build configuration file, +# created by, sphinx-quickstart on Mon Dec 18 11:31:24 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import importlib +import re +import shutil +import sys +from distutils.dir_util import copy_tree +from inspect import getmembers, isclass, isfunction +from pathlib import Path +from typing import List + +import patchy +from click import secho, style +from sphinx.ext.autosummary.generate import generate_autosummary_docs + +from causalnex import __version__ as release + +# -- Project information ----------------------------------------------------- + +project = "causalnex" +copyright = "2020, QuantumBlack" +author = "QuantumBlack" + +# The short X.Y version. +version = re.match(r"^([0-9]+\.[0-9]+).*", release).group(1) + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx_autodoc_typehints", + "sphinx.ext.doctest", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "nbsphinx", + "recommonmark", + "sphinx_markdown_tables", + "sphinx_copybutton", +] + +# enable autosummary plugin (table of contents for modules/classes/class +# methods) +autosummary_generate = True +autosummary_imported_members = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = {".rst": "restructuredtext", ".md": "markdown"} + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ["**cli*", "_build", "**.ipynb_checkpoints", "_templates"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" +here = Path(__file__).parent.absolute() +# html_logo = str(here / "causalnex_logo.svg") + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + "collapse_navigation": False, + "style_external_links": True, + # "logo_only": True + # "github_url": "https://github.com/quantumblacklabs/causalnex" +} + +html_context = { + "display_github": True, + "github_url": "https://github.com/quantumblacklabs/causalnex/tree/develop/docs/source", +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. + +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + +html_show_sourcelink = False + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "causalnexdoc" + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, "causalnex.tex", "causalnex Documentation", "QuantumBlack", "manual") +] + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "causalnex", "causalnex Documentation", [author], 1)] + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "causalnex", + "causalnex Documentation", + author, + "causalnex", + "Toolkit for causal reasoning (Bayesian Networks / Inference)", + "Data-Science", + ) +] + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Extension configuration ------------------------------------------------- + +# nbsphinx_prolog = """ +# see here for prolog/epilog details: +# https://nbsphinx.readthedocs.io/en/0.4.0/prolog-and-epilog.html +# """ + +nbsphinx_epilog = """ +.. note:: + + Found a bug, or didn't find what you were looking for? `🙏Please file a + ticket <https://github.com/quantumblacklabs/causalnex/issues/new/choose>`_ +""" + +# -- NBconvert kernel config ------------------------------------------------- +nbsphinx_kernel_name = "causalnex" + + +# -- causalnex specific configuration ------------------ +MODULES = [] + + +def get_classes(module): + importlib.import_module(module) + return [obj[0] for obj in getmembers(sys.modules[module], lambda obj: isclass(obj))] + + +def get_functions(module): + importlib.import_module(module) + return [ + obj[0] for obj in getmembers(sys.modules[module], lambda obj: isfunction(obj)) + ] + + +def remove_arrows_in_examples(lines): + for i, line in enumerate(lines): + lines[i] = line.replace(">>>", "") + + +def autolink_replacements(what): + """ + Create a list containing replacement tuples of the form: + (``regex``, ``replacement``, ``obj``) for all classes and methods which are + imported in ``MODULES`` ``__init__.py`` files. The ``replacement`` + is a reStructuredText link to their documentation. + For example, if the docstring reads: + This DataSet loads and saves ... + Then the word ``DataSet``, will be replaced by + :class:`~causalnex.io.DataSet` + Works for plural as well, e.g: + These ``DataSet``s load and save + Will convert to: + These :class:`causalnex.io.DataSet` s load and + save + Args: + what (str) : The objects to create replacement tuples for. Possible + values ["class", "func"] + Returns: + List[Tuple[regex, str, str]]: A list of tuples: (regex, replacement, + obj), for all "what" objects imported in __init__.py files of + ``MODULES`` + """ + replacements = [] + suggestions = [] + for module in MODULES: + if what == "class": + objects = get_classes(module) + elif what == "func": + objects = get_functions(module) + + # Look for recognised class names/function names which are + # surrounded by double back-ticks + if what == "class": + # first do plural only for classes + replacements += [ + ( + r"``{}``s".format(obj), + ":{}:`~{}.{}`\\\\s".format(what, module, obj), + obj, + ) + for obj in objects + ] + + # singular + replacements += [ + (r"``{}``".format(obj), ":{}:`~{}.{}`".format(what, module, obj), obj) + for obj in objects + ] + + # Look for recognised class names/function names which are NOT + # surrounded by double back-ticks, so that we can log these in the + # terminal + if what == "class": + # first do plural only for classes + suggestions += [ + (r"(?<!\w|`){}s(?!\w|`{{2}})".format(obj), "``{}``s".format(obj), obj) + for obj in objects + ] + + # then singular + suggestions += [ + (r"(?<!\w|`){}(?!\w|`{{2}})".format(obj), "``{}``".format(obj), obj) + for obj in objects + ] + + return replacements, suggestions + + +def log_suggestions(lines: List[str], name: str): + """Use the ``suggestions`` list to log in the terminal places where the + developer has forgotten to surround with double back-ticks class + name/function name references. + + Args: + lines: The docstring lines. + name: The name of the object whose docstring is contained in lines. + """ + title_printed = False + + for i in range(len(lines)): + if ">>>" in lines[i]: + continue + + for existing, replacement, obj in suggestions: + new = re.sub(existing, r"{}".format(replacement), lines[i]) + if new == lines[i]: + continue + if ":rtype:" in lines[i] or ":type " in lines[i]: + continue + + if not title_printed: + secho("-" * 50 + "\n" + name + ":\n" + "-" * 50, fg="blue") + title_printed = True + + print( + "[" + + str(i) + + "] " + + re.sub(existing, r"{}".format(style(obj, fg="magenta")), lines[i]) + ) + print( + "[" + + str(i) + + "] " + + re.sub(existing, r"``{}``".format(style(obj, fg="green")), lines[i]) + ) + + if title_printed: + print("\n") + + +def autolink_classes_and_methods(lines): + for i in range(len(lines)): + if ">>>" in lines[i]: + continue + + for existing, replacement, obj in replacements: + lines[i] = re.sub(existing, r"{}".format(replacement), lines[i]) + + +# Sphinx build passes six arguments +def autodoc_process_docstring(app, what, name, obj, options, lines): + try: + # guarded method to make sure build never fails + log_suggestions(lines, name) + autolink_classes_and_methods(lines) + except Exception as e: + print( + style( + "Failed to check for class name mentions that can be " + "converted to reStructuredText links in docstring of {}. " + "Error is: \n{}".format(name, str(e)), + fg="red", + ) + ) + + remove_arrows_in_examples(lines) + + +# Sphinx build method passes six arguments +def skip(app, what, name, obj, skip, options): + if name == "__init__": + return False + return skip + + +def _prepare_build_dir(app, config): + """Get current working directory to the state expected + by the ReadTheDocs builder. Shortly, it does the same as + ./build-docs.sh script except not running `sphinx-build` step.""" + build_root = Path(app.srcdir) + build_out = Path(app.outdir) + copy_tree(str(here / "source"), str(build_root)) + copy_tree(str(build_root / "api_docs"), str(build_root)) + shutil.rmtree(str(build_root / "api_docs")) + shutil.rmtree(str(build_out), ignore_errors=True) + copy_tree(str(build_root / "css"), str(build_out / "_static" / "css")) + copy_tree(str(build_root / "04_user_guide/images"), str(build_out / "04_user_guide")) + shutil.rmtree(str(build_root / "css")) + + +def setup(app): + app.connect("config-inited", _prepare_build_dir) + app.connect("autodoc-process-docstring", autodoc_process_docstring) + app.connect("autodoc-skip-member", skip) + app.add_stylesheet("css/qb1-sphinx-rtd.css") + # fix a bug with table wraps in Read the Docs Sphinx theme: + # https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html + app.add_stylesheet("css/theme-overrides.css") + # add "Copy" button to code snippets + app.add_stylesheet("css/copybutton.css") + app.add_stylesheet("css/causalnex.css") + + # when using nbsphinx, to allow mathjax render properly + app.config._raw_config.pop('mathjax_config') + + +def fix_module_paths(): + """ + This method fixes the module paths of all class/functions we import in the + __init__.py file of the various causalnex submodules. + """ + for module in MODULES: + mod = importlib.import_module(module) + if not hasattr(mod, "__all__"): + mod.__all__ = get_classes(module) + get_functions(module) + + +# (regex, restructuredText link replacement, object) list +replacements = [] + +# (regex, class/function name surrounded with back-ticks, object) list +suggestions = [] + +try: + # guarded code to make sure build never fails + replacements_f, suggestions_f = autolink_replacements("func") + replacements_c, suggestions_c = autolink_replacements("class") + replacements = replacements_f + replacements_c + suggestions = suggestions_f + suggestions_c +except Exception as e: + print( + style( + "Failed to create list of (regex, reStructuredText link " + "replacement) for class names and method names in docstrings. " + "Error is: \n{}".format(str(e)), + fg="red", + ) + ) + +fix_module_paths() + +patchy.patch( + generate_autosummary_docs, + """\ +@@ -3,7 +3,7 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', + base_path=None, builder=None, template_dir=None, + imported_members=False, app=None): + # type: (List[unicode], unicode, unicode, Callable, Callable, unicode, Builder, unicode, bool, Any) -> None # NOQA +- ++ imported_members = True + showed_sources = list(sorted(sources)) + if len(showed_sources) > 20: + showed_sources = showed_sources[:10] + ['...'] + showed_sources[-10:] +""", +) + +patchy.patch( + generate_autosummary_docs, + """\ +@@ -96,6 +96,21 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', + if x in include_public or not x.startswith('_')] + return public, items + ++ import importlib ++ def get_public_modules(obj, typ): ++ # type: (Any, str) -> List[str] ++ items = [] # type: List[str] ++ for item in getattr(obj, '__all__', []): ++ try: ++ importlib.import_module(name + '.' + item) ++ except ImportError: ++ continue ++ finally: ++ if item in sys.modules: ++ sys.modules.pop(name + '.' + item) ++ items.append(name + '.' + item) ++ return items ++ + ns = {} # type: Dict[unicode, Any] +""", +) + +patchy.patch( + generate_autosummary_docs, + """\ +@@ -106,6 +106,9 @@ def generate_autosummary_docs(sources, output_dir=None, suffix='.rst', + get_members(obj, 'class', imported=imported_members) + ns['exceptions'], ns['all_exceptions'] = \\ + get_members(obj, 'exception', imported=imported_members) ++ ns['public_modules'] = get_public_modules(obj, 'module') ++ ns['functions'] = [m for m in ns['functions'] if not hasattr(obj, '__all__') or m in obj.__all__] ++ ns['classes'] = [m for m in ns['classes'] if not hasattr(obj, '__all__') or m in obj.__all__] + elif doc.objtype == 'class': + ns['members'] = dir(obj) + ns['inherited_members'] = \\ +""", +) diff --git a/docs/source/01_introduction/01_introduction.md b/docs/source/01_introduction/01_introduction.md new file mode 100644 index 0000000..e7e85e1 --- /dev/null +++ b/docs/source/01_introduction/01_introduction.md @@ -0,0 +1,42 @@ +# Introduction + + +CausalNex is a Python library that uses Bayesian Networks to combine machine learning and domain expertise for causal reasoning. +You can use CausalNex to uncover structural relationships in your data, learn complex distributions, +and observe the effect of potential interventions. + +## Main features of CausalNex + +The CausalNex library has the following features: + +- Deploys state-of-the-art structure learning method, [DAG with NO TEARS](https://papers.nips.cc/paper/8157-dags-with-no-tears-continuous-optimization-for-structure-learning.pdf), to understand conditional dependencies between variables +- Allows domain knowledge to augment model relationships +- Builds predictive models based on structural relationships +- Understands model probability +- Evaluates model quality with standard statistical checks +- Visualisation which simplifies how causality is understood +- Analyses the impact of interventions using Do-calculus + +## Learning About CausalNex + +In the next few chapters, you will learn how to install and set up CausalNex, and how to use it on your own projects. +Once you are set up, to get a feel for CausalNex, we suggest working through our example tutorial project. +Advanced users looking for in-depth information should consult the User Guide. +You can also check out the resources section for answers to frequently asked questions and the API reference documentation for further, detailed information. + +## Assumptions + +We have designed the documentation in general, and the tutorial in particular, for beginners to get started using Bayesian Networks on their projects. If you an have elementary knowledge of Python and Bayesian Networks then you may find the CausalNex learning curve more challenging. However, we have simplified the tutorial by providing all the Python functions necessary to create your first CausalNex project. + +Note: There are a number of excellent online resources for learning Python, but be aware that +you should choose those that reference Python 3, as CausalNex is built for Python 3.5+. +There are many curated lists of online resources, such as: + +- [Official Python programming language website](https://www.python.org/) +- [List of free programming books and tutorials](https://github.com/EbookFoundation/free-programming-books/blob/master/free-programming-books.md#python) + +There are also several excellent online resources for learning about Bayesian Networks, such as: + +- [Lecture notes](https://ermongroup.github.io/cs228-notes/) on Probabilistic graphical models based on Stanford CS228; +- [An Introduction to Bayesian Network Theory and Usage](http://infoscience.epfl.ch/record/82584) by T. Stephenson; +- [PGMPY tutorial](https://github.com/pgmpy/pgmpy_notebook/blob/master/notebooks/2.%20Bayesian%20Networks.ipynb). diff --git a/docs/source/02_getting_started/01_prerequisites.md b/docs/source/02_getting_started/01_prerequisites.md new file mode 100644 index 0000000..a133417 --- /dev/null +++ b/docs/source/02_getting_started/01_prerequisites.md @@ -0,0 +1,71 @@ +# Installation prerequisites + +CausalNex supports macOS, Linux and Windows (7 / 8 / 10 and Windows Server 2016+). If you encounter any problems on +these platforms, please check the FAQ, and / or the Alchemy community support on Slack. + +## macOS / Linux + +In order to work effectively with CausalNex projects, we highly recommend you download and install +[Anaconda](https://www.anaconda.com/download/#macos) (Python 3.x version). + +## Windows + +You will require admin rights to complete the installation of the following tools on your machine: + +* [Anaconda](https://www.anaconda.com/download/#windows) (Python 3.x version) + +## Python virtual environments + +Python's virtual environments can be used to isolate the dependencies of different individual projects, +avoiding Python version conflicts. They also prevent permission issues for non-administrator users. +For more information, please refer to this +[guide](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html). + +### Using `conda` + +We recommend creating your virtual environment using [`conda`](https://conda.io/docs/), a package and environment +manager program bundled with Anaconda. + +#### Create an environment with `conda` + +Use [`conda create`](https://conda.io/docs/user-guide/tasks/manage-environments.html#id1) to create a python 3.6 +environment called `environment_name` by running: + +```bash +conda create --name environment_name python=3.6 +``` + +#### Activate an environment with `conda` + +Use [`conda activate`](https://conda.io/docs/user-guide/tasks/manage-environments.html#activating-an-environment) +to activate an environment called `environment_name` by running: + +```bash +conda activate environment_name +``` + +When you want to deactivate the environment you are using with CausalNex, you can use +[`conda deactivate`](https://conda.io/docs/user-guide/tasks/manage-environments.html#id6): + +```bash +conda deactivate +``` + +#### Other `conda` commands + +To list all existing `conda` environments: + +```bash +conda env list +``` + +To delete an environment: + +```bash +conda remove --name environment_name --all +``` + +### Alternatives to `conda` + +If you prefer an alternative environment manager such as [`venv`](https://docs.python.org/3/library/venv.html), +[`pyenv`](https://github.com/pyenv/pyenv), etc, please read their respective documentation. diff --git a/docs/source/02_getting_started/02_install.md b/docs/source/02_getting_started/02_install.md new file mode 100644 index 0000000..de3cddc --- /dev/null +++ b/docs/source/02_getting_started/02_install.md @@ -0,0 +1,21 @@ +## Installation guide + +We recommend installing CausalNex in a new virtual environment for *each* of your projects. To install CausalNex: + +```bash +pip install causalnex +``` + +To check your installation: + +```bash +python -c "import causalnex" +``` + +If CausalNex is not installed correctly you will see an error message similar to the following: + +```bash +ModuleNotFoundError: No module named 'causalnex' +``` + +You should not see any output if CausalNex is correctly installed. diff --git a/docs/source/03_tutorial/03_tutorial.ipynb b/docs/source/03_tutorial/03_tutorial.ipynb new file mode 100644 index 0000000..2a2e5af --- /dev/null +++ b/docs/source/03_tutorial/03_tutorial.ipynb @@ -0,0 +1,2112 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# A first CausalNex tutorial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This tutorial will walk you through an example workflow using CausalNex to estimate whether a student will pass or fail an exam, by looking at various influences like school support, relationship between family members, and others. We will use the [Student Performance Data Set](https://archive.ics.uci.edu/ml/datasets/Student+Performance) published in the [UCI Machine Learning Repository](http://archive.ics.uci.edu/ml).\n", + "\n", + "\n", + "To work through this tutorial, you first need to create a new Python 3 notebook and download the [student.zip](https://archive.ics.uci.edu/ml/machine-learning-databases/00320/student.zip) file and extract `student-por.csv` from the zip file into the same directory, then copy and paste the code cells from this tutorial into your notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Structure Learning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Defining the structure of a Bayesian Network (BN) model can be done based on machine learning, domain knowledge, or a combination of both, where experts and algorithms contribute as equal partners.\n", + "\n", + "Regardless of the approach, it is important to validate the structure by evaluating the BN - this will be covered later in the tutorial. In this section, we will focus on how to define a structure." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structure from Domain Knowledge" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can manually define a structure model by specifying the relationships between different features.\n", + "\n", + "First, we must create an empty structure model." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from causalnex.structure import StructureModel\n", + "sm = StructureModel()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we can specify the relationships between features. For example, let's assume that experts tell us the following causal relationships are known (G1 is grade in semester 1):\n", + "* `health` -> `absences`\n", + "* `health` -> `G1`\n", + "\n", + "We can add these relationships into our structure model:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "sm.add_edges_from([\n", + " ('health', 'absences'),\n", + " ('health', 'G1')\n", + "])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualising the Structure" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can examine a StructureModel by looking at the output of `sm.edges`" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "OutEdgeView([('health', 'absences'), ('health', 'G1')])" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sm.edges" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "but it can often be more intuitive to visualise it. CausalNex provides a plotting module that allows us to do this." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/ben_horsburgh/opt/anaconda3/envs/causal-test/lib/python3.7/site-packages/networkx/drawing/nx_pylab.py:563: MatplotlibDeprecationWarning: \n", + "The iterable function was deprecated in Matplotlib 3.1 and will be removed in 3.3. Use np.iterable instead.\n", + " if not cb.iterable(width):\n", + "/Users/ben_horsburgh/opt/anaconda3/envs/causal-test/lib/python3.7/site-packages/networkx/drawing/nx_pylab.py:660: MatplotlibDeprecationWarning: \n", + "The iterable function was deprecated in Matplotlib 3.1 and will be removed in 3.3. Use np.iterable instead.\n", + " if cb.iterable(node_size): # many node sizes\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWgElEQVR4nO3de5ScdX3H8fc3QJINigEBEfDCzQpCihBEFAgK5S6IQDZYjyBajb1oj9Sj2NODpbX0VKFqtaXWKnraHpZLALkErFig5SaIGi5aASHK/WYQSCCR/fWP37Od2WV2s7szs8/zzLxf53CS7O7MPskfvM8z+8zziZQSkiRVzayyD0CSpFYMlCSpkgyUJKmSDJQkqZIMlCSpkgyUJKmSDJQkqZIMlCSpkgyUJKmSDJQkqZIMlCSpkgyUJKmSDJQkqZIMlCSpkjYs+wDWK2JL4ERgATAfWAWsAM4hpcfLPDRJUvdEZfegIvYCTgUOAxIw0PTZNUAAy4EzSOmWmT9ASVI3VTNQEUuBM4G5TPwy5DDwPHAKKZ09E4cmSZoZ1QtUI07zpvCo1RgpSeopXQtURJwEfCiltO8UHrQXcA1Ti9OI1cAiUrp1Go+VJFVM1a7iO5X8st50zC0eL0nqAdUJVL5a7zCmf0yzgMOJ2KJzByVJKkvbgYqIT0fEvRHxTETcFRHHjP50fCUino6In0XEgU2fOCkiflE87r4/gy+Sr9bjG8DOwKbAIcDK5icEzgZ2Il9z/kcjD8rSR+EfI+KnTcezR/H9to6ICyPi8Yi4LyI+1nQsb4mIWyPiNxHxaESc1e6/iySpPZ04g7oX2A94BfCXwL9FxKuLz+1dfH5z4DRgWURsFhEbA18GDkspvRx421HwMmDgEuBvgGXA48UTnzDmG14G3EJ+M9R5wFXFx8+HgSE4Ang/sAlwFPBkRMwCLgV+AmwDHAj8aUQcUjz0S8CXUkqbADsUTytJKlHbgUopnZ9SeiilNJxSGgLuBt5SfPox4IsppXXF5/6XHBDIl4jvGhEDKaWH988nR5xN/kHSzuR3EX8G+DGjz6I+TT57ei3wjuLzAF8HlsK9KaVbUnZPSmklsBewRUrp9JTS2pTSL4B/AZYUD10H7BgRm6eUnk0p3dTuv4skqT2deInv/RHx44hYFRGrgF3JZ0wAD6bRlwmuBLZOKT0HDAJLgYcj4vIbcrBYCXycHKD5wGbkl/AebHqSrZp+Pw94tvj9r4DX5SiO9Tpg65FjLI7zM8Cris9/EHgD8LOIuCUijpz6v4QkqZPautVRRLyOfCZyIHBjSunFiPgxxdkQsE1ERFOkXgt8ByCldBVwVUQMAH/9XjjmfljzGhj4c+D3p3E828DwNZCWRhxKfqnujcAc4JvAfSmlnVo9LqV0N3BC8VLge4ALIuKVRUglSSVo9wxqY/IJzuMAEfEB8hnUiC2Bj0XERhFxPPmVuysi4lURcXTxs6gXgGcfhkeAWAqcAdxZPMHTwPmTPJiTIZ2XY3kpcBbwx8BBwA+AZyLiUxExEBEbRMSukd93RUS8LyK2SCkNk+/1B8UZnSSpHG0FKqV0F/muDzcCjwK7Adc3fcnN5AvungA+BxyXUnqy+L6fAB4CngIWrYUPAcuPgeFPkX84tAm5dssndzjDS+CS4fw9NwBmFx9/DjgAOBrYHbivOJ6vky/sADgUuDMiniVfMLEkpbRmCv8UkqQOq9atjjpwJ4mAHwJfA95LDtVZ5KvVtwUuAIaA/ynOliRJFVWtQEFH7sUXEQH8K7AgpbSw+NiO5AszBoFXkl85HAJuSpX7R5AkVS9Q0LG7mUfE7JTS2hYf35lGrOaR3/c0BPzQWElSNVQzUAARC8lviTqc8fegriDvQU3rBrHFmdZuNGIFjVitMFaSVJ7qBmpEvrdeq0Xdb3VyUbeI1R7kUC0mn5kNAUPFxSCSpBlU/UCVoIjV3jRi9RSNWN1d5rFJUr8wUOtRvHn37eRYHUe+NH4kVveXeGiS1NMM1BRExAbAInKsjgXuIcfq/JTSA2UemyT1GgM1TRGxEfmuFYPkNwHfSY7VBSmlR8o8NknqBQaqAyJiDnAwOVZHAreRY3VhSumJMo9NkurKQHVYcfPbw8ixOhS4iRyri1JKvy7z2CSpTgxUFxU3wz2SHKuDgOvIsbokpfSbMo9NkqrOQM2QiBhZ+B0kX2hxNTlWlzrrIUkvZaBKEBGbAu8mx2of8mr9EHCFd1GXpMxAlSwiNiePJA4CewKXA+cC300pvVDmsUlSmQxUhUTEVuT3Vw2Sp7AuIZ9ZXZ1SWlfmsUnSTDNQFRUR2wLHk2O1A3AROVbXpJReLPPYJGkmGKgaiIjXk+8JOAhsA1yIw4uSepyBqhmHFyX1CwNVYw4vSuplBqoHOLwoqRcZqB7TYnhxDY15kJ+WeWySNBUGqoe1GF58kkas7inz2CRpfQxUn2gxvPggOVbnObwoqYoMVB8aM7z4HuBeHF6UVDEGqs8Vw4vvBJbg8KKkCjFQ+n8OL0qqEgOllhxelFQ2A6X1GjO8eCCN4cXvOLwoqVsMlKZkzPDi/jSGFy9zeFFSJxkoTVuL4cUrybFa7vCipHYZKHXEmOHFPcjDi0M4vChpmgyUOs7hRUmdYKDUVQ4vSpouA6UZM87w4rnA9Q4vShrLQKkUY4YXN6MxvHiz8yCSwECpAsYMLw7Q2LK6zVhJ/ctAqTLGGV4cKv673VhJ/cVAqZIcXpRkoFR5Di9K/clAqVYcXpT6h4FSbY0zvHgueXjxwTKPTVL7DJR6QtPw4iAvHV58tMxjkzQ9Bko9pxhe/D3ySvARNIYXlzm8KNWHgVJPazG8eCM5Vhc7vChVm4FS33B4UaoXA6W+5PCiVH0GSn3P4UWpmgyU1KTF8OJlNIYX15Z5bFK/MVDSOMYML76JxvDi9x1elLrPQEmT0GJ4cRk5Vtc6vCh1h4GSpqjF8OIF5Fg5vCh1kIGS2uDwotQ9BkrqEIcXpc4yUFKHtRheTDRi5fCiNEkGSuoihxel6TNQ0gxxeFGaGgMllaDF8OIDNIYXV5Z5bFJVGCipZC2GF+8hx8rhRfU1AyVVSDG8eCCN4cU7cHhRfcpASRVVDC8eTI6Vw4vqOwZKqoEJhhcvSimtKvPYpG4xUFLNOLyofmGgpBpzeFG9zEBJPWLM8OJbgatweFE1ZqCkHtQ0vLgEeDMOL6qGDJTU44rhxePIZ1a74PCiasJASX3E4UXViYGS+pTDi6o6AyWJiNiJRqwcXlQlGChJozi8qKowUJJacnhRZTNQktZrzPDiILAahxfVZQZK0pQ0DS8uIV8R6PCiusJASZq2YnhxXxrDi7/C4UV1iIGS1BERsSGjhxfvxuFFtcFASeo4hxfVCQZKUleNGV48EvghDi9qEgyUpBnj8KKmwkBJKoXDi1ofAyWpdA4vqhUDJalSiuHFY2gML16Jw4t9yUBJqqyI2IJ8yfogDi/2HQMlqRYcXuw/BkpS7Ti82B8MlKRac3ixdxkoST3D4cXeYqAk9aSI2IUcqyXAXBxerB0DJamnFfMgC2hsWQ3j8GItGChJfaOI1Z7kUC3G4cVKM1CS+lKxZbU3OVYOL1aQgZLU9xxerCYDJUlNHF6sDgMlSeNweLFcBkqSJqFpeHEJcAQOL3adgZKkKSqGFw8nn1kdgsOLXWGgJKkNEfEyRg8vXovDix1hoCSpQ4rhxaPJsdoPhxfbYqAkqQscXmyfgZKkLhszvLgHcCkOL66XgZKkGeTw4uQZKEkqSUS8hsbw4vY4vDiKgZKkCoiI7WhsWW1Np4cXI7YETiTf2X0+sApYAZxDSo+3/fxdYKAkqWKK4cWReZBNaWd4MWIv4FTgMCABA02fXQMEsBw4g5RuafvgO8hASVKFFcOLI7Ga2vBixFLgzOJxsyb4ymHgeeAUUjq7A4fdERMdsCSpZCmlu1JKpwE7AxsB25Ij9fOI+FxELCh2rkZrxGkeE/y//hxg3/z5ecCZxeOaniYOiIgHOvTXmRIDJUk1UJwtrQO+AewInADMJl+yfldEfDYidgZGXtYbidNUzAv4p7Mi3t2xA2+DgZKkmknZrSmlTwLbASeTL3y4OiJW/ATOSfllvWk5AJau94tmgIGSpHrZPSJWRMTTETEEzE4p3Qh8D3gsYPuPwC63N/3//W+BHYCXk994ddE4T7x/8et+cMisiGcjYnDkcxFxSkQ8FhEPR8QHuvI3G8NASVK9LAYOJZ85LQBOiog3k1/6+8gaOP3DsPYo4IXiATsA/w08DZwGvA94uMUTX1f8+iN4fhg+m1IaKj60FfAKYBvgg8BXi1s5dZWBkqR6+XJK6aGU0lPknz/tDnwY+OeU0s1zYLeTYfYc4KbiAceT31g1i3wp4E7ADyb4BrPyy4MLmj60Djg9pbQupXQF8CzwOx39W7WwYbe/gSSpox5p+v1qcns2A06MiD/ZGDbeEFgLPFR80beBs4D7iz8/C0xiYXF+0++fTCn9dsz3fdk0jn1KPIOSpPr7FfC5lNL8x2DZKnJBTgBWAn8AfAV4knz7iF3J79hdj9KHFw2UJNXffwCfiIjrT4ejn4IXLweeAZ4j3ypii+ILvwncMcETvQq4J//4akU3D3gyDJQk1VCx5LsrsIj8s6ifA1v9Hax9E2xwTvF1uwCnAPuQ43M78PYJnvezwAdgziz4i4hY3KXDnxRvdSRJNRER84DDydc6HAzcQL7t0cUppVVNX7iMvOw7nZOQYeBiUjq27QNuk4GSpAqLiDnky8oHyXG6lRylZSmlJ8d50F7ANUz9ThKQf3y1iJRunc7xdpKBkqSKiYjZwEHkKB1F/nnQEHBhSunRST5J8734Jms1FbphrIGSpAqIiA2Bd5Cj9G7yz5SGgAtSSg9O80lrfTdzAyVJJYmIDYD9yFE6lnxV+BBwfkppZYe+yULyHtThjL8HdQV5D6r0l/WaGShJmkERMQt4KzlKxwOPkaN0Xkrp3i5+4y1ovaj7LRd1JalPFXtNC8lRWky+mcO55Cj9rMxjqzJvdSRJXVBE6XdpRGmYfKZ0BHDHlKfb+5CBkqQOiog30Zhon01evz0e+JFRmhpf4pOkNkXEG2hEaT45SkPAD4zS9BkoSZqGiNiORpS2Ai4gR+mGlNJwmcfWKwyUJE1SRLyG/POkQfJg4IXkKF2XUnqxzGPrRQZKkiYQEa8GjiNHaWfgYnKUvj9mI0kdZqAkaYzI7xk6lhyl3cl3Cx8C/jOltLbMY+snBkqSgIjYDDiGHKW9yXdXGAKuTCk9X+ax9SsDJalvRcQryLMUg8C+wPfIUbo8pfRcmccmAyWpzxRDf+8iR+md5FmKIeA7KaVnSjw0jWGgJPW8iBgg38Fh4qE/VYqBktSTiqG/Q4AlTHboT5VioCT1jIjYiNFDf7cz1aE/VYaBklRrxdDfAeQoHUMnhv5UCd4sVlLtFEN/+9IY+vslOUp7dmzoT6UzUJJqYYKhv7d1dehPpTFQkiprgqG/dzr01/sMlKRKcehPIwyUpEqIiF1ozFfMwaG/vudVfJJKExE70YjSpjj0pyYGStKMKob+RjaVXo1DfxqHgZLUdcXQ3/E0hv6W4dCf1sNASeoKh/7ULgMlqWMc+lMnGShJbXHoT91ioCRNmUN/mgkGStKkFEN/R9IY+rsWh/7URQZK0riKob/DyVE6BIf+NIMMlKRRmob+Bsm3F3LoT6UwUJIc+lMlGSipTzn0p6rzZrFSH3HoT3VioKQe59Cf6spAST2o2FTak8adwh36U+0YKKlHFFFaQCNKDv2p1gyUVHMO/alXeRWfVEMO/akfGCipJhz6U78xUFKFRcS2NKLk0J/6ioGSKiYitqKxPuvQn/qWgZIqICI2J79xdgkO/UmAgZJKExGbMnrobzkO/Un/z0BJMygiNqEx9LcfDv1J4zJQUpdFxMbAu3DoT5oSAyV1gUN/UvsMlNQhDv1JnWWgpDY49Cd1j4GSpsihP2lmeLNYaRKKTaX9cOhPmjEGShpHMV/RPPT3OA79STPGQElNxgz9LQaeIw/9HejQnzSzDJT63pihv8VAIp8pHYlDf1JpDJT61pihv7nkKC3GoT+pEryKT33FoT+pPgyUel5EvJ5GlBz6k2rCQKknjRn62x64EIf+pFoxUOoZxdDfceQo7UJj6O+/Ukrryjw2SVNnoFRrTUN/g8CbcehP6hkGSrXTYujvSnKUljv0J/UOA6VacOhP6j8GSpXl0J/U3wyUKsWhP0kjDJRKVwz9HQwswaE/SQUDpVIUQ38Hks+UjsahP0ljGCjNmGLobxGNob97yHcKd+hP0kt4s1h1VTH0ty+Nob8HyGdKCx36kzQRA6WOazH09wQ5SvumlO4p89gk1YeBUkeMM/Q3hEN/kqbJQGnaHPqT1E0GSlMWETvTmK8YwKE/SV3gVXyalIjYkUaUNsOhP0ldZqA0rmLob2RTaRvy0N+5OPQnaQYYKI1SDP0dT47SDsAy8pnStQ79SZpJBkqthv4uIUfp+w79SSqLgepTLYb+LiNH6bsO/UmqAgPVRxz6k1QnBqrHOfQnqa4MVA8qhv6OJM9XOPQnqZYMVI8ohv4OI58pHQrcSGPo79dlHpskTYeBqrGmob9B8tDfbTSG/p4o89gkqV0GqmZaDP3dQY7SBQ79SeolBqoGImID4ABGD/0NAec79CepV3mz2Ipy6E9SvzNQFeLQnyQ1GKiStRj6W02+IetBKaWflnlsklQmA1WCFkN/kM+U3gXc7nyFJBmoGdVi6O+84ve3GSVJGs2r+LqsxdDf+eSzpZuNkiSNz0B1wThDf0PA9Q79SdLkGKgOcehPkjrLQLXBoT9J6h4DNUUO/UnSzDBQkzBm6O+tjB76W1PmsUlSrzJQ4yiG/o4iR2l/4GpylC5z6E+Sus9ANWka+hsk3zH8OhpDf78p89gkqd9UP1ARWwInku+8MB9YBawAziGlx9t/eof+JKmKqhuoiL2AU8nxSOQ7L4xYAwSwHDiDlG6Z2lM79CdJVVfNQEUsBc4E5gKzJvjKYeB54BRSOnv0U8RSYJ+U0onFn8cb+rswpfRIx/8OkqS2VC9QjTjNm8KjVtMUqSJOZ5HjdgL5LMyhP0mqkWoFKr+sd825MO/vyac4GwPbkX8I9VHgGuB08mtymwL3Nx69GlgU+b1J/wDMIb80+Evgq8B5Dv1JUn1M9PJZGU79Agx8HPgk8AjwKHA2cD2wlhysk4HPv/Sxc++Cr5H/m1N8LIBHU0qfN06SVC/VOYOK2HIVrNwG5n6bfKuGiXwP+BCjzqBI8MJu8Fd35j/uCrwReDGltLDzByxJ6qYq7UGdeAPEC+QrGKYjYPgOeIGUvtDJA5MkzbwqvcS34CmYszmjq/k28pufBsjvml2PAfL7pSRJNVelM6j5rwSeAH5L48BuKH7dlnxN+WSep9MHJkmaeVU6g1q1D/nqhkvafJ5OHIwkqVxVOoNaMR/WnAYDf0i+PvwQ8lV7K4CRu7MOk6/mW1d8zfPkys7On15TfLkkqeYqdRUfsBKY++/Al2i8D2p74IPASeSX/N4x5qGLyO+PIvfqtZ24R58kqVzVCRRAxDLyRXzTeelxGLiYlNZ3hbokqQaq9DMogDPIZ0HT8XzxeElSD6hWoPJdyU8h37ZoKkbuxXdr5w9KklSGKl0kkaV0NhHQ5t3MJUn1Vq2fQTWLWEjegzqc8fegriDvQXnmJEk9prqBGhGxBa0Xdb/l1XqS1LuqHyhJUl+q1kUSkiQVDJQkqZIMlCSpkgyUJKmSDJQkqZIMlCSpkgyUJKmSDJQkqZIMlCSpkgyUJKmSDJQkqZIMlCSpkgyUJKmSDJQkqZL+D8kWvkPMq7vfAAAAAElFTkSuQmCC\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "from causalnex.plots import plot_structure\n", + "\n", + "_, _, _ = plot_structure(sm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Learning the Structure" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As the number of variables grows, or when domain knowledge does not exist, it can be tedious to define a structure manually. We can use CausalNex to learn the structure model from data. The structure learning algorithm we are going to use here is the [NOTEARS](https://arxiv.org/abs/1803.01422) algorithm.\n", + "\n", + "When learning structure, we can use the entire dataset. Since structure should be considered as a joint effort between machine learning and domain experts, it is not always necessary to use a train / test split.\n", + "\n", + "But before we begin, we have to pre-process the data so that the NOTEARS algorithm can be used." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Preparing the Data for Structure Learning" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>school</th>\n", + " <th>sex</th>\n", + " <th>age</th>\n", + " <th>address</th>\n", + " <th>famsize</th>\n", + " <th>Pstatus</th>\n", + " <th>Medu</th>\n", + " <th>Fedu</th>\n", + " <th>Mjob</th>\n", + " <th>Fjob</th>\n", + " <th>...</th>\n", + " <th>famrel</th>\n", + " <th>freetime</th>\n", + " <th>goout</th>\n", + " <th>Dalc</th>\n", + " <th>Walc</th>\n", + " <th>health</th>\n", + " <th>absences</th>\n", + " <th>G1</th>\n", + " <th>G2</th>\n", + " <th>G3</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>GP</td>\n", + " <td>F</td>\n", + " <td>18</td>\n", + " <td>U</td>\n", + " <td>GT3</td>\n", + " <td>A</td>\n", + " <td>4</td>\n", + " <td>4</td>\n", + " <td>at_home</td>\n", + " <td>teacher</td>\n", + " <td>...</td>\n", + " <td>4</td>\n", + " <td>3</td>\n", + " <td>4</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>3</td>\n", + " <td>4</td>\n", + " <td>0</td>\n", + " <td>11</td>\n", + " <td>11</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>GP</td>\n", + " <td>F</td>\n", + " <td>17</td>\n", + " <td>U</td>\n", + " <td>GT3</td>\n", + " <td>T</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>at_home</td>\n", + " <td>other</td>\n", + " <td>...</td>\n", + " <td>5</td>\n", + " <td>3</td>\n", + " <td>3</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>9</td>\n", + " <td>11</td>\n", + " <td>11</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>GP</td>\n", + " <td>F</td>\n", + " <td>15</td>\n", + " <td>U</td>\n", + " <td>LE3</td>\n", + " <td>T</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>at_home</td>\n", + " <td>other</td>\n", + " <td>...</td>\n", + " <td>4</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>2</td>\n", + " <td>3</td>\n", + " <td>3</td>\n", + " <td>6</td>\n", + " <td>12</td>\n", + " <td>13</td>\n", + " <td>12</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>GP</td>\n", + " <td>F</td>\n", + " <td>15</td>\n", + " <td>U</td>\n", + " <td>GT3</td>\n", + " <td>T</td>\n", + " <td>4</td>\n", + " <td>2</td>\n", + " <td>health</td>\n", + " <td>services</td>\n", + " <td>...</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>2</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>14</td>\n", + " <td>14</td>\n", + " <td>14</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>GP</td>\n", + " <td>F</td>\n", + " <td>16</td>\n", + " <td>U</td>\n", + " <td>GT3</td>\n", + " <td>T</td>\n", + " <td>3</td>\n", + " <td>3</td>\n", + " <td>other</td>\n", + " <td>other</td>\n", + " <td>...</td>\n", + " <td>4</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>1</td>\n", + " <td>2</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>11</td>\n", + " <td>13</td>\n", + " <td>13</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>5 rows × 33 columns</p>\n", + "</div>" + ], + "text/plain": [ + " school sex age address famsize Pstatus Medu Fedu Mjob Fjob ... \\\n", + "0 GP F 18 U GT3 A 4 4 at_home teacher ... \n", + "1 GP F 17 U GT3 T 1 1 at_home other ... \n", + "2 GP F 15 U LE3 T 1 1 at_home other ... \n", + "3 GP F 15 U GT3 T 4 2 health services ... \n", + "4 GP F 16 U GT3 T 3 3 other other ... \n", + "\n", + " famrel freetime goout Dalc Walc health absences G1 G2 G3 \n", + "0 4 3 4 1 1 3 4 0 11 11 \n", + "1 5 3 3 1 1 3 2 9 11 11 \n", + "2 4 3 2 2 3 3 6 12 13 12 \n", + "3 3 2 2 1 1 5 0 14 14 14 \n", + "4 4 3 2 1 2 5 0 11 13 13 \n", + "\n", + "[5 rows x 33 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "data = pd.read_csv('student-por.csv', delimiter=';')\n", + "data.head(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Looking at the data, we can see that features consist of numeric and non-numeric columns. We can drop sensitive features such as sex that we do not want to include in our model." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>address</th>\n", + " <th>famsize</th>\n", + " <th>Pstatus</th>\n", + " <th>Medu</th>\n", + " <th>Fedu</th>\n", + " <th>traveltime</th>\n", + " <th>studytime</th>\n", + " <th>failures</th>\n", + " <th>schoolsup</th>\n", + " <th>famsup</th>\n", + " <th>...</th>\n", + " <th>famrel</th>\n", + " <th>freetime</th>\n", + " <th>goout</th>\n", + " <th>Dalc</th>\n", + " <th>Walc</th>\n", + " <th>health</th>\n", + " <th>absences</th>\n", + " <th>G1</th>\n", + " <th>G2</th>\n", + " <th>G3</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>U</td>\n", + " <td>GT3</td>\n", + " <td>A</td>\n", + " <td>4</td>\n", + " <td>4</td>\n", + " <td>2</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>yes</td>\n", + " <td>no</td>\n", + " <td>...</td>\n", + " <td>4</td>\n", + " <td>3</td>\n", + " <td>4</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>3</td>\n", + " <td>4</td>\n", + " <td>0</td>\n", + " <td>11</td>\n", + " <td>11</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>U</td>\n", + " <td>GT3</td>\n", + " <td>T</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>no</td>\n", + " <td>yes</td>\n", + " <td>...</td>\n", + " <td>5</td>\n", + " <td>3</td>\n", + " <td>3</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>9</td>\n", + " <td>11</td>\n", + " <td>11</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>U</td>\n", + " <td>LE3</td>\n", + " <td>T</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>yes</td>\n", + " <td>no</td>\n", + " <td>...</td>\n", + " <td>4</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>2</td>\n", + " <td>3</td>\n", + " <td>3</td>\n", + " <td>6</td>\n", + " <td>12</td>\n", + " <td>13</td>\n", + " <td>12</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>U</td>\n", + " <td>GT3</td>\n", + " <td>T</td>\n", + " <td>4</td>\n", + " <td>2</td>\n", + " <td>1</td>\n", + " <td>3</td>\n", + " <td>0</td>\n", + " <td>no</td>\n", + " <td>yes</td>\n", + " <td>...</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>2</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>14</td>\n", + " <td>14</td>\n", + " <td>14</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>U</td>\n", + " <td>GT3</td>\n", + " <td>T</td>\n", + " <td>3</td>\n", + " <td>3</td>\n", + " <td>1</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>no</td>\n", + " <td>yes</td>\n", + " <td>...</td>\n", + " <td>4</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>1</td>\n", + " <td>2</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>11</td>\n", + " <td>13</td>\n", + " <td>13</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>5 rows × 26 columns</p>\n", + "</div>" + ], + "text/plain": [ + " address famsize Pstatus Medu Fedu traveltime studytime failures \\\n", + "0 U GT3 A 4 4 2 2 0 \n", + "1 U GT3 T 1 1 1 2 0 \n", + "2 U LE3 T 1 1 1 2 0 \n", + "3 U GT3 T 4 2 1 3 0 \n", + "4 U GT3 T 3 3 1 2 0 \n", + "\n", + " schoolsup famsup ... famrel freetime goout Dalc Walc health absences G1 \\\n", + "0 yes no ... 4 3 4 1 1 3 4 0 \n", + "1 no yes ... 5 3 3 1 1 3 2 9 \n", + "2 yes no ... 4 3 2 2 3 3 6 12 \n", + "3 no yes ... 3 2 2 1 1 5 0 14 \n", + "4 no yes ... 4 3 2 1 2 5 0 11 \n", + "\n", + " G2 G3 \n", + "0 11 11 \n", + "1 11 11 \n", + "2 13 12 \n", + "3 14 14 \n", + "4 13 13 \n", + "\n", + "[5 rows x 26 columns]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "drop_col = ['school','sex','age','Mjob', 'Fjob','reason','guardian']\n", + "data = data.drop(columns=drop_col)\n", + "data.head(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we want to make our data numeric, since this is what the NOTEARS expects. We can do this by label encoding non-numeric variables." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['address', 'famsize', 'Pstatus', 'schoolsup', 'famsup', 'paid', 'activities', 'nursery', 'higher', 'internet', 'romantic']\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "struct_data = data.copy()\n", + "\n", + "non_numeric_columns = list(struct_data.select_dtypes(exclude=[np.number]).columns)\n", + "print(non_numeric_columns)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>address</th>\n", + " <th>famsize</th>\n", + " <th>Pstatus</th>\n", + " <th>Medu</th>\n", + " <th>Fedu</th>\n", + " <th>traveltime</th>\n", + " <th>studytime</th>\n", + " <th>failures</th>\n", + " <th>schoolsup</th>\n", + " <th>famsup</th>\n", + " <th>...</th>\n", + " <th>famrel</th>\n", + " <th>freetime</th>\n", + " <th>goout</th>\n", + " <th>Dalc</th>\n", + " <th>Walc</th>\n", + " <th>health</th>\n", + " <th>absences</th>\n", + " <th>G1</th>\n", + " <th>G2</th>\n", + " <th>G3</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>4</td>\n", + " <td>4</td>\n", + " <td>2</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " <td>...</td>\n", + " <td>4</td>\n", + " <td>3</td>\n", + " <td>4</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>3</td>\n", + " <td>4</td>\n", + " <td>0</td>\n", + " <td>11</td>\n", + " <td>11</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " <td>...</td>\n", + " <td>5</td>\n", + " <td>3</td>\n", + " <td>3</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>9</td>\n", + " <td>11</td>\n", + " <td>11</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " <td>...</td>\n", + " <td>4</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>2</td>\n", + " <td>3</td>\n", + " <td>3</td>\n", + " <td>6</td>\n", + " <td>12</td>\n", + " <td>13</td>\n", + " <td>12</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " <td>4</td>\n", + " <td>2</td>\n", + " <td>1</td>\n", + " <td>3</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " <td>...</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>2</td>\n", + " <td>1</td>\n", + " <td>1</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>14</td>\n", + " <td>14</td>\n", + " <td>14</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>1</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " <td>3</td>\n", + " <td>3</td>\n", + " <td>1</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>1</td>\n", + " <td>...</td>\n", + " <td>4</td>\n", + " <td>3</td>\n", + " <td>2</td>\n", + " <td>1</td>\n", + " <td>2</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>11</td>\n", + " <td>13</td>\n", + " <td>13</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>5 rows × 26 columns</p>\n", + "</div>" + ], + "text/plain": [ + " address famsize Pstatus Medu Fedu traveltime studytime failures \\\n", + "0 1 0 0 4 4 2 2 0 \n", + "1 1 0 1 1 1 1 2 0 \n", + "2 1 1 1 1 1 1 2 0 \n", + "3 1 0 1 4 2 1 3 0 \n", + "4 1 0 1 3 3 1 2 0 \n", + "\n", + " schoolsup famsup ... famrel freetime goout Dalc Walc health \\\n", + "0 1 0 ... 4 3 4 1 1 3 \n", + "1 0 1 ... 5 3 3 1 1 3 \n", + "2 1 0 ... 4 3 2 2 3 3 \n", + "3 0 1 ... 3 2 2 1 1 5 \n", + "4 0 1 ... 4 3 2 1 2 5 \n", + "\n", + " absences G1 G2 G3 \n", + "0 4 0 11 11 \n", + "1 2 9 11 11 \n", + "2 6 12 13 12 \n", + "3 0 14 14 14 \n", + "4 0 11 13 13 \n", + "\n", + "[5 rows x 26 columns]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from sklearn.preprocessing import LabelEncoder\n", + "\n", + "le = LabelEncoder()\n", + "for col in non_numeric_columns:\n", + " struct_data[col] = le.fit_transform(struct_data[col])\n", + " \n", + "struct_data.head(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now apply the NOTEARS algorithm to learn the structure." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from causalnex.structure.notears import from_pandas\n", + "sm = from_pandas(struct_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and visualise the learned StructureModel using the plot function." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/ben_horsburgh/opt/anaconda3/envs/causal-test/lib/python3.7/site-packages/networkx/drawing/nx_pylab.py:563: MatplotlibDeprecationWarning: \n", + "The iterable function was deprecated in Matplotlib 3.1 and will be removed in 3.3. Use np.iterable instead.\n", + " if not cb.iterable(width):\n", + "/Users/ben_horsburgh/opt/anaconda3/envs/causal-test/lib/python3.7/site-packages/networkx/drawing/nx_pylab.py:660: MatplotlibDeprecationWarning: \n", + "The iterable function was deprecated in Matplotlib 3.1 and will be removed in 3.3. Use np.iterable instead.\n", + " if cb.iterable(node_size): # many node sizes\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOydd7xdRbXHvzshQAwQIIQQWkJRsGCjqAiCWJGHYkVBEH0WQFR8iIAIguUhKgoWFHxIFUVBegfpCKGTQHpuSO/t9nvKvD9+azFz9t3npgjcRPZ8PvM5Z7eZNW2tWXWyEAJlKlOZylSmMq1taUB/A1CmMpWpTGUqU1EqCVSZylSmMpVprUwlgSpTmcpUpjKtlakkUGUqU5nKVKa1MpUEqkxlKlOZyrRWppJAlalMZSpTmdbKVBKoMpWpTGUq01qZSgJVpjKVqUxlWitTSaDKVKYylalMa2UqCVSZylSmMpVprUwlgSpTmcpUpjKtlakkUGUqU5nKVKa1MpUEqkxlKlOZyrRWppJAlalMZSpTmdbKVBKoMpWpTGUq01qZSgJVpjKVqUxlWitTSaDKVKYylalMa2UqCVSZylSmMpVprUwlgSpTmcpUpjKtlakkUGUqU5nKVKa1Mq3X3wCUqUyveMqyLYEvAG8GNgWWAc8ClxDCwv4E7cW0LsBYpjK9zCkLIfQ3DGUq0yuTsmxP4BTgQCAAg5OnnUAG3AqcRQiPvfIAsm7AWKYyvUKpJFBlenWkLDsaOAfYkL5F23WgCziBEP7wSoD2YloXYCxTmV7BVOqgyrTOpCzLdsmy7Oksy1qzLPtmH+9tn2VZW5ZlAwFGZdmkC+A84DWsfM4PsPfOMYLxiqRNsmzxHfAr1gDGLMv+kGXZaS87kGUq0yucSh1Umdal9F3gnhDCW/t6KYQwA9gIgCzbcxTsNHD1N2NOAB4nhMf7ejHLsjOAnUMIn1/NOryAPTeDzQZIfNdnugT4P+DBBMYA+60MxjKVaV1MJQdVpnUpjQKeW81vThmwhvO8DhvWpA96udMp2SoQpyZpQ14ZGMtUplc8lQSqTC99yrItybITybLLybIb7fdEsmz4mheZ/RN4L/BbE999K8uyp7IsW5Fl2UzjYvzd0VmWhQlZNhIZG7yYzgBSNmc6ogxVu94fOBV4NzAEBkyDg87Lsh2zLLsoy7K5WZZ1ZllWz7KsPcuyiVmWfRk4HTg8y7KQZdkyg2F6lmXvT2C6Osuy5VmWfTLLsgFZlh2RZdkLA7JsyY/gYH9vHmKLFicwPgkMB8YCRwP/Quzhpno84Ej42LAs+6XVs3+WZbOyLPtulmULDOZDsiz7SJZlk7IsW5Jl2fcSuAZkWXZylmVTsyxbnGXZ37Is23wVBuQlH+MylalXCiGUucwvTYY9A/wjQGeAjgAhyR12/x8B9lyT8oF7gS/b//2B3dAm683AfOAQezYaCB1wUoCO/SD80eD4AYTDE7haZCkXKna9H4TtIIyze93Q8UYYBzwELMfeB/5s9dwI3A1cgbiZfQyG6cD7E9jH2XcdwExkkbffMjj5W1AZCOFOg+FACOcnMB4P4Tj7fzGEdzf2azgCqv8FdyX9UkVEcxDwFWAhcCWwMfBGq3sHe/9bwCPAtsAGwAXAX/prjMtc5jT3OwBl/g/JcHSA9gC1HNLK55q9d/Tq1pESqNz99wOTgZsMCY8GQhdc4URndQjUacnzeRDWi0Tppc71y6HeBmFQQqD+CmFv+1+FMALCo30QqC8oP2t98XGgB7gZ2MH6IwDvSPrriYSYjwfelzwbCVSA9fpjjMtc5jSXIr4y9ZlMbLXzSl5y8+hVtkAbBeeflWXn/htwvSPLsnuyLFuIuJidgQ8j6dhDAI/DfmtS9nbJ/xeAWvNXK8AiRBAC0A3MBSYiLmaB5dYm33cPh+4hwLDk5seA54EW4E5gKLDXSmDuhtdmWTYX+BvinA5EUsBOe2V+8nonbkQivd61WZYtM/HkeGvyiIYK+hjj6TSKSSmwhDSx7I4raUaZytSQSgJVpn8vybHUEdeqfwbZ7vA1smyPxuKy12dZdkWWZRs0rzIbClyP5u94IhIeiJDz1gAteSQLDEEyNk/zimF7MW1Hn6aug4At7Bd7dSSwk/0fDmyJuJiitP4i2KCDRp3ThsBnkMzwcuCIJrClqUOfbZWAmwH/QFwlwD5N+nQmImZvB65F1ogbhhBmx0pXb4z3R5aGRCK1RwhhoxDCtFX5vkxl8lQSqDL9u+kUhBxXO2Wwvn1PpnQ0Ej99DvhglmVbZ1n2xSzL/phl2SPA3sAfUNifEUj3NAw5rfZKYyF0irN5Mb0VuB+YgRRKZ60ExqHA9sWPFloRzj1lxPXkRCIAsxAjFvIFAAOeRB1Qzz04EpmU30AjgRphBfYk96o0skdJ6k7AvxjoyrKsC9gTODUTB/sw8FMD40hgUoFP1RqPMaWVYZn+ndTfMsYyv7IZOAmYjcROE4H3Ic7je8BUu/8EsJ29H5Dx2GREGH6HRSCZDyPOhMr2EIZLWR+WJbqI6yG8AcJQ0+08nzwbFXUunbvCVxEX1KCfSX5bgHsQw9MNTMu9X899G4CwBYSOnA4qQDjWYNoJwoUFOqj03Q4Im6+eXqkbSb2atachZ1D/AdST/ngx7wzhPbl73RA+AmEzCMPs3uchDOxDz2W/FeAxRJtrlruJBDb/ze3AEevBcwE6axDOgrCj9cenISy2+luSPvwehAEQNoAwBMLXbYyt3J1tTl0CnI9CNrUhkexWwLnAUmAC8LZkzm4NXIM2BS3AN/t7HZX5FcJX/Q1AmV/BwYZdkEhna7sejcRRJyIr5l0QJ/AWYJi9E5CYaFO0G18IfDiEwHHwtx2hPhVCK4SPG7IMECZCeA2EOyD0QDjbCEJ3jkC1QxjZHLHmkbznGiKkXbl7vd69A5ZUc0h+VXMVwtV9I/0FwNXI8rs993wpUiHdbEi4KVG7FqpFML6XRmK5mjBWk99lSLJZ2Ed9wFffD7o7oXYuhHdAmAmhC8JXIXy2gEAFehP5oLrzBGoRsDvisP5phOdItFn6MXLIBnGlTyCrxPWBHdEG5UP9vZ7K/ArgrP4GoMyv4GDLkGABsnoblNyfCHysyTcBM522678BJ4cQeDPM/V2CiCYgi7cKhB+iXbY/q0HYGsI9OQIVIOzShLtIcjdS07QYknXk6//bm327J1Tb1pBAtUHYvTlMVRrhrhscc3KEM+RgLoKxlodxDOLyVqw5jA7bioQwdSEudDoioEXcU0MZv4auAGFXCHcl9c5JxnoVCJSXlxKoPyZz6hvA+OR6N2CZ/X8HMCM3J08BLu7v9VTmlz+XOqhXUQohTAGOR/6qC7Is+2uWZVsjW4CpfXya2hJ0YBZgy2CjUcmDUUR9yBy79jTAKplN7/TtJuAibq1qv69BHN8ARGQfRWKrgfZbCP9jMPC7UGlv3rbC1A6cgLbuTWDLLK9QNbQZjCPRTn8h8rHtIuqmQhMYB5wMbQ7jF9AO4lyaW1esBMY60nuBxmox4vIWI8ONUYgjdl1Z0+M7dlBbeAHZr29q+fWo45vovlYl5a0K+7Iy3NqtDM3S8HsUGMCU6T8vlQTqVZZCCFeGEPZBCz8AZyOx306rW9am0PZCcj0DYbwRSGmQPgtWyTYF5Qxubpy2uRW5DXIi9TQS7awHIQI2xOAvNOc+HwadDB0dQL0JkfBUAzognIA8VgvStSqGGci8fUNkdDAEuA/4L2ASIgRvR8TzKkTAupHYr1f6LWx0OsztRL5Oy4Gj+oDRiVMTGDsRMQeJ94YB70LDMoDI0Xi/D21SFUvtne2QwmhZkrsoHs81jdnUJM0EWkIImyZ54xDCR17aasq0NqaSQL2KkkUDP8DMjbsQIqsjq+AfZVn2WrOme3OWZcNy3z6TZdkS4KPAJ7Msu30HqP8Syd3a0Lb2UERRPoOUL3cjDH0OojB752DqhHBVMdFIreJ6iGI8ECGaZ++sZ7lOHwzHb+E1+0H9Wsi6aDQ1dzg6geuA90DWBPEvRWGJFiMCcDDSpXzAmvsepK/bEfgN8EWD8VBgE2AJYo4WFRX+Sxi5L2TNYOwQnFyHHLyawAgilm4YuJn9n020dnQdnr8ziOLUMQE6OpGVzKnETcdCZOdflEYgJZGnruhKdkGWZeOBwxEhX5U0BmjNsuykLMsGZ1k2MMuyN2UyfS/Tf3rqbxljmV+CDFsGODHA5QFutN8TAwxP30Nm2WMQclqCkOnWSFrzfURrWpHIalv7JiDd1YPkdBTDof00CNsii7nDISxJ9A7/gPB6CJsga7RxyTPXQXUii6982UTdzcr0Uz2IM3F9y9/7+KZmcFdOgHApsjS8FMJ3oL5F3/XkdTvjgJ8gYuP35iGFfxtR7zMV4faFue/7zFuIiwuXQrjBfk+CLoOx0gdsS3P91lQ/V9CmXve3hPZOqNcgnAPhdRA2QtZ8p9hY5nVQD0N4LYRNIXwDWUIWlP1gMi+/DNybXO8MVJPrrYG/WP8uRaGZ3r8ma6DM61budwDK/G/klz/2XYZUIo9TbAFWu66JBdqq5D4s0NJcJVqBVQyOHuCHSMRWtTyXRuOJZhZrecRcS3IzQ4aaIcaApFsrVoXIFOQlRjDq9n9lBCNtz8rK7km+c52Oc04fRsYtdSvT29u5KmX/w8bqJR7jbuAO4IC1eQ2UuX9zvwNQ5jXML0FcNCQK+kLuXgZ8EnFMFUN8DyMley9EswcyFV8T5NWm75vt3pcjY4iAuJTlhmCno510SsDON2RXRU6+TxIJ2poQkpArPyVUKQGpW34Yifj83Xakd5rQhMDUm5S/ptnb+Wdkjl0HLkSRJFIi9wKyX6kjjs7FfLOJm4DCMX4ZLCHvQq4NrlK7EXhnbi4ORqq47OVaA2Veu3O/A1DmNchxYYbVyA0LFFmcPWLI4r3AB5HKqNuQ2qPAD5A6IY9U09xzLHR3rBxJNOROqB7dvMwxBsNyYhQGN4mehYhUDZ0NNZtiQtROJGR9+iE1yU5cvgb8jOiXVbH/V1k//in55iZkIj026bOjkQh1MtE0fYX9fg7p/2rWptWBr24ExvukqA86ULSk++3954mcm3Ojk2zM25vAUAOqP4F/tUN9dYnT0c05NCfuY5Df0wS7Xo64vd2QOXlAxDZ7qddAmdf+3O8AvJozcox92hDNqnnHS6SxugszJAt0D2R59mCC9F3U9Sza+Y8jIhZHnq1NkGAAwtcMIa1MFFSF0A09X2tO9NxwY1byvIZEeC1NvvHs3EL6To/1cQ1Z4D2ZIMeQe38G4izq9OZsvI+OQAr+CjGW7AyDe7Y99/B6buzm/9+KiF2KoL1816O1IHFi/rmPx3XAU8RIIEX90RdXVkXEYHGuj5yzKhrjClA/B+7sXAWRbtXmwteK63f92EIb0xSGBxDXNy3Xri6SCCarswZ+QGP0el8D/b32y7yKOLK/AXg1Z+Ai4Fer9Z3k6avFrSS5thxuIYrO0uxnHTlimAX8lah7WZkoqnt3pG/ohNCR2213QbUDwo3Q0Yfza1FeTuQUUoQ1legQWzdk3Uo8duKnhvSuIXINKdKfjAjvzfb+CuCbSKzk3Ia/Ox4R9LxzbkDE6ELEaTnSfxY4zvqviIPoQCIuN7CoWnuWGCw3AE/Z/5mIm3VuMK2/xRB6HTjAYF9uYzcneW8ZIqYp8XGDiqaivaK8D0y+Dnq6oNYNlXSM2zXu4WoIe/TtBOzEPlj78mMUKNbR1YDXrc4aKCBQtQDX9PfaL/Oq5cwWQZn6IWVZdhfw1xDC/63iB1siRLOmgTvpAraHsLC5u0oNiaQCeqcbIdPdkNvLz5GzbxEMAciGQ+1IGPhm5NS5HHgWwiWQFdpXxzSf6IA53epLTaAfQtHDX4eQesX+b4MI7zCE4CYAuwLnoYP4PolM1mchs+v1rNwBCEH/BYk5FyL/pR2RWPAjyET8i8hM/L1I14XVV0Om5oHeLhu+sKrAZcCbkO9WN/GcpgnAIQbTn6yMMQbzeERYDrH7NTQei+13OOLa/oTG5u0IqXtw2GfQyR07Id+rCnK6fRT5Pb0+gbUDEeit7HqR9XOzFLaA7CgIu0G2qQE6duVjPMng+hTiArusT9L5lp9/aapuCT0zYcP1V9FF5gxgCooMnyQtgxCaOiiXaS1J/U0hX60ZmSPX0GJpQyebPoUQxUzgjOTd0UD4Bly1DdQ3hfB7FA5nNxQS5+vJLnEyMuveBAUU/Yzdb7Gd6AnFu/oeJHLpplG01l3w7sqyfzMrKSe/g+7re4/B58FrHaaHEPGYZ/DW7P9diBC8AZ3onoYacljebX05HwUxzxDXNQvhV3+/3eqpISK2AULgrbnyvF1tyf8fImJ2RfJOG71FeXUrc5LBsZc9e4hGzsOD+v7FYH8AM8dGxDdveTjfyjrC6rjO6ve+egCJeKcl385CRC8f+7CvMfLxdO7rsaStqztX2pLvemzMpzSpv+NUaL0Hwrts3r+ZGD4rQJhmc38jCO+3deEc1D0QttH/jgDfsX6cTt5kvcxrTe53AF7NmdU8wvxgmNQJ4XbkO/QxCPMhzELRxO+1hfhZCD9G8e86ITyQI1B/ao50HPG0Ik7BCcwiGkUwd9h1M3PrVfL1yeUFyBqujriCu+1+BTgWid/mEwnYpxF34wYJVRoJ070oSsai5LmL7b6DuKplRLP1cfbODJqLp2YjbuRBK+cJ+73L+mw+cIzB0YKMOGrIGKWOLNXu66PfOhFyDla2j8nzwEFIh9aOuGjv4+eAM4nGLF6OP78JOVf/T+6bw4FfJPd+SyTMd9A8UG9fuWjcK4goj0fck9+fi5hrN8i5lyhmbrYpqv0WujeHcLPN7ztQdPUFNsffCeHbKKDtfUaoCghUCHCZra3plARqrc39DsCrOdPkCHN7di6mn8II1CS403eKm6Ojwf36ExB+Zf+PgPAVFHk6JNkJ1LUrRzB9cU3dREKWcltrmt2PKSDCkHIElaT8IgOIvnx4upN2VHNlrQqiXRUOwmFO6/Lv/V4Kd1/wrqzOfP09yTfNjs1o1k+pEUiV3u2pJP//3fF1WL2uZnCuSjvCF4gR8z1/EMIlEF5Ax46kJvGfoymBusHW1nRKArXW5jLU0VqS0iPMsyxbjsyTG/QA2ycBNQfTGC1zMJKTQDQT2wt4I2JH0rQshrjpBYb9rm+/bURfmVYkGrvdnv0XMgwIKHzPErsf+myokB+I67hJ4DAI7dpr9nyJ1fmHEMIAa9JAJFI6mxhxZ4C995EQQoYiEgTE6QxKYHrevvX2ObL0/4+jc7F6kHHD8wm8r7Wyd7N3x9Ko/xiAlPzfN9i7rE3fRdzSQHvP656AEPH6Vu53kz67A5mFtwBfQYYceXi9rOVEEeFypFdzLmoXK/v7CYwO+1n2fwCK/zoIiegqiGtabs++j2Ie1hGHNt3KSs9KLErelieA/zb43oTUQQOQQcmFxCj0U63MDM27ZmGXANpmQf3vxKC1myJ2di6yCtkMOfd5GtW8rGUraUeZ1obU3xTy1ZxpFPFNRYG9N7Trc4Er7P9oIHTAScG85behUfZ+OIQf5XaWAYn3NkB6qQW2C/3GynfFi5B12A123Y2Q4apa9AVEHOYi5HM+0XR6KkJMZyC62o10NxfYdysQsjoEEaxbiIjfy74N6euWItFcQLoo34WnjrTp7wpgYtL/jyFx13M09skEZMgwE7jc3h1ItKQ722D8iPVFKg7LcyPLkMHGQCQeDMDtCQwTrI37E60A3dz7IWS4cW7yfidRZ1bP1encqBPJOppjVeRb5OLQYOUvQ8S4HYUXGkk0Az/Gvn3B6r0HESmvezzRB2waK7cGTDnIeUS/MHdqvmkVyqi+BypHFczzAGF6AQd1WMJBjUEHPdoa+o6NSTslB7XW5n4H4NWcaSRQC7CoDoj5WUCOQI2HkXWFbumTQP2NKN4bB2FDCFPtemt00mnB4k/jty2hkSAsQvoD95V6jEaE7kruW5HxR93eX1xQjyMr94VZmtx7yJDWE+ho9wXJswcNIS6iN9zpmUd3Am8jcgoeBsn1OU/Ai9arYwwxnkwUx7lYa6m1oR0RmAVEX7AfIg73dmKsO3d6fYpGfdB8FB/wE0ivtsTqeYoYUf5hRHjcjL1mfXcI0omda/AOse/deMWJchVxswcgww8njs79pmM1GXFpqT5oPCK6C60vxyXfzE3GsZmxy3TiRuGvRKOJlOD0WH8tsraOo9E4pc1gm0kjMWvIw6BjBITbkL9Vp60Dn+/vQDEMu9HmbOOEQC2DMBjCDdD9hIjxD6zvSgK1luZ+B+DVnGkkUJ9CoqtWQ5q/RaFrhgH72gL94S3QWl0JgToxIUQ7QrjA7lfRkdxFC5/Gnbkjh0qCcIq4rm5DNIsMkb4dmaHPRafyOoG50JBtDzJl/qTdv5lIGJfSGE+vGWFLfXmWI6LlkR4eTGB3fdYG1r9DiASoC0VwmGT/e4AfAacZMh4K/C8xikVAiPuhBOl2GGKuIyOIGnawIzqhuIaIRQ2J7BzuhfS21hsLfB3t6OtInHiV/e9B3M+tRKvPRUQiu0duzLqsvA4UoeFqGnVJKwv/1GPtdk60E83THrRxerO9NxyJaYO1Z7jBsC8yx3eDjykUE5seRLDPQkQ+jYc4kz7mwVkQ9kXc0BYQPoL0TwFtxPaxuZ+34gsQLtJ3Lkb+sbX1A/2NC8rcBEf2NwBlbjIw8O4E4by4E90HXsg7wa5q7iP2nYupHAE2Q2QVhNgfQHqeEw0B+g44H6uu3RDdexGBCMCWyM/H61mWqyuFYRpwGAoJ1G111pBY7AGi4UNfRG2pISEXb11NJCiOKPe0Pn+LlTcUiVuXJuV4m1qtrzw0z+n27T9snA6yso+3+zcZnAsNKXbnygtI7HkYOq6jlsyBLybvdSMOxAP3jkOIPO/EnN9A+Njcigibm8wvRLrDrxE54Hzk886k/BlI5XOc3dsI+Vn5e0XzqYJ8wL5psJxDdK/IbzxcPOkc1DKK21QpOoF4VXM71HeP86ydaOSyaX+v+TIX4MH+BqDMTQZGCuOiRbriGKis7gJth/CNvpFZ0RHmKeJ5KkEejrD+D+mO2pBO5idEono7MZpBXzqvOiI8s5GI50uGPKZZuUOtP3ZNYBpk9w5KkOoy4IN2fxe7N9e+mYRMmetEc/lUpNkXbB4V/CfIoOHmpF+Wo4gVQ639Lfbu1QbHaOBiIgFeBFyJTL4vs3sLm9SbXjusdaIe7Iakf2cbTEMMjmOTOmcAbzJ4drQyuoic5TuIXF434pJcBNdXxHWHK6DwUR7vr4JM2Ifb9fFIxOlzuYI4y2lJGUUELnV7yOeZ34b5q7sG2iAcUxzC6pH+Xu9lboIH+xuAMhcMikQ9F/eBQGvHQlcnVGsr4aaqKOzQ8cXhjTx30KjbcT8dRxwVJI75kCHDJw3RugisnuRAb+fUCYhrSZFDT+7//OT6GmSkOAOJiDJEYJYgrmaJve+6mOetjPFIJHohMM/68lM0npHUihD4d6ydjsjzyHA24jrGJN+19oE00/b3RfwqiLB1WX4YGX3kfaPSWIjTUFTvnyXw1q1v3oYs32pIjPo2a6/3k4+Fh4aaaGWci4ze2hGXtzORy6whwpmOV/q/E4n1Ul2bc755ww3neM+3Mm9Dm5Ank3ftsOMGjnUOfYsjZ54Jj7ez8viPNaj3ERvQ5993+3vdl7kAF/Y3AK/2DAzMXR9tSGMFOuiu2aKavAdU/gG1DnofedENlW6oXQ9de/V9ftGZCXL0CNcV4i72L8jk+ekEoSyi9xENqb9LDXFbz1o5rTQSr/WsrfcjzuyjCFE74nXC6EjREd8jybMOYry9jZHSu4UYmeNsq2N3GqN0pxZ9rYY8RyPOYVryjutiniGKL4tEiZUEweYJTHuTb1z0Nt/61Y04/Pk/UQin3QzOhTTGHfT3HgZGWDsfRo7VNbSZGATsY++fkJS/BBmKpOOdii+97CvRwb3LgZ9YHZvl6nfuz+MKpu1L31uB5ks6z+4FDkRE1nVk/p332VTEfTWbu63vhcduhPZuqObXQDsyorgG6u9sPKIlT5x+ZTDNJXc+Fbn1WeZXGD/2NwD/MXkNTvREOoCFyN9pD4RIq8hA4so+FmaKHMN2cON3oH451O6Atiug52To2KI3Qsvn9IwnJ0iLkUVXuhtOy+imkTDVgY9ae75tSO6f9Ha4rCNz7g6igv8Aq9PD9vw3IgbnIwS7H9qp5+FOrbwmASdZ/2VIBOfILW+27AYR3t5mhh9VhHBdzJSKo76BuLQhNBLtgCJFfIpo0eYwXoS44pGIu/NnLqJLOciiCA6pM61b+6XPFxEJxP9Y3x2ArA2XJ2UUzYHfIq4rQ5uF55AFZQ0ZEdSAwTZem9g34wrKaUNGPQHYzt7/fu6dPMFO+24+8MdkjNK5N6YJ7C/mLWHFabDwz9B9A9Qvg3AWTB6+cvGy/56MOOY6WhcjkfhzMbD9S7nuy7zqud8BWOfzGp7oiXb9CwwB+A78PmTV5KKN5xHCbbbI+rpfFHmhr2+r9BapjEFGBSmCvpRo8VchRuROI0xMR1Zx19n1WPtuelLOC4gIu6HDJ61fvmKwfJhoCu07698Cv0Gm5Gnct76QUFcC/31IdHoREaE7d3cNQtIfA3oMlowYuiggEdQ5NJ5d5BzIcbnx9RNs3VCkav15djI2C4EP2/vb2r1BiFMZZ33jYzLN2n8F4jrcoKAvMVjaL/OA6w1+HxeH4yYkQqwDuxo8ZxOJ3800iojHImLiRPWrdu0clUejcBeA5dYHC5FVqM+BschacQqNcPt4p/D3FYWjmvvNv9uXWPY3xLmwAukZ3fLQNwYP0Ps8qvIk31cg9zsA63T+N070REgy3U0+TxT9dNpz5wCW0tynKF1szlmsigGAZ9cvLbH/1xtcvkivQEYHLvZy+P5Oo9I+1c/4WT91oCtp8yAr+0LEGaXhgbqtjLwlmSMYL3tmgtD84L9A44F7NSTuynMNLyDu7lcoqGzaniMNxg3t3nCi7sv1Rl5OK+IWRxB3+w07ZkRgg/VDhgxKUq7SDU5+Z+8fC7Ta/wyJL2tJGV9ABikP09sXrDPp77r1g2GDBrUAACAASURBVBOuKnEDNIFIaCr0Rtx1YvDWlPCtQNzFXYjryxI4/bsWGoO+ep0z7d6faeQCOxCX4nENx9rvP5LyPDbjqsxjb9PKiJnnFhpFrSl8S2k0XukCvvhSrPsyr17udwDW2byKJ3p+GMUJs+v2IB3TSX0sHFdqOwL5KRJjOSJY1cXaLM8gErAjDPlMQsTDxWP/MuRzKI2I+R4kfku5l+nI3Pw04mF4zazQ0mtHJo7Alhd814OCwo5G4qpUt1FFhPWbNB5C+KLvUwgB4JdJWecjhJgi+DpxI7AM6f06aNwgdBFDMR1r9TrRXIAI370NC0v3byP6Y/UgB9nLrJ7HiAhxDtLJTEbE29uZjw4+C3GPvyYi5COTOt+etCkAlyAdY15EtghZ142kWM/pnI+3L9Bbr1RkZddjMC5Bc/ysXF97JPnxyb0WpCfcmLgJOwk5MfvcPN3ut9Pc/DzNK1snvnG5CxGmdL11Nvm+Duy7qus+zUC4GU7rd5y1DuZ+B2CdzP/GqbZd8uNotgi6EHFwzsAXTkD6AUeYY+lbtFOU3W9ocrIYT0WcwmK0A68gxOlRCFpWAwGk3IGbU99vyMCRygzi6bxdwN7I0dYJy/lIt3NrQfmOLJ3YjUbcjnNyTkjv7zXJY2Twecn7lyCfojHIufh+euvNgsF7BXKkfQJ4zMq8mxip2/VG77Vn7jhcTcbyNHu2DBGpk1F08iLOuB1tFs5ASP13iJPzPnKftV8UtNXLa7X2/ARtNg63+x3EMEj5ek8BtkO6NB/TMxD35lZ6TyV1uIjvj0hs6FxTKmJe2bypIYLloZdmEA9SfArp7m4jEsipVu/kgrL6yn6MTSCezuz1p9x5n8fLTISe1V3z9l1HKE/yXe3c7wCsk/nfONW2CuHaxojRRYu4hvQlNyfX6fN5du9RQ3KBeMaPEzpHMI/bN5cSCdwLRISej/LdRW8xW93Kn2H/5wBtL04iIT9H7rOA/e3+McBy+/86hOgdRkdmCxFX4OF/HEE4Yr8aOS3/lsadtyPpVhRGaJsE4RzcMMllQfanpOw/2P0x1od3E/VEE5I6HkLIeiKNlpDev3cD30MI2sfTfYeWIT3Oj5HhQUjqr9l7Y619Hp6oQjwLaTnSY81FJuEZMYTQXIP9tlw7z7BvXYd2upWZRomvoTnxQ2S5OIHorOqEzTmuv9s4VZPnHUiPNczqCMBm9v9LRK5vEYaQiQdJemimyTYm+TnmjrPpPYd7AZGbm23jUqeRe3YYexABqiGOdLaNWX4z4Jan1yCDl74IXB0IU9ZgzQNhEuVJvmuS+x2AdSWjnfQp68HETSEchUxYl0A4CIVc2dT+p8dc7AfhjwlxOgEdIjig78WQIsFgC+thQzbfI1ob3UfUPRSV4QjcF7ovyOloZz3Grp9Gu9ZUfOLhYILVN9D64R1EC7jfoCgR7ZbnE/UGs5BIywnUBsRIEI5E8iKiCXbPY8+5NaE7lZ5jSMiR4DIaDwDsJIrfDkvGzg1APOZcnWilV0eE8SvImnJRUm63XR9s5WREbulxRGA8CG6z/l+ACMtUey8NVvsda5/3cyfi0oYgYuPGHV3Egxv/x+rsBuYmZf2K6BYQaIwO4RsO13u5YYZHzjjArh+zfsw7D6fHZNSBO5N6AxKj+tlO/yAStJnoEMlbkvHaxr4bTtQZzsr14fNWhs/zvyIutzVpl49pkc4yJOU9hzYhAXGfP7f/txE3bKmLRGEeYsSvBZ01dQKE7SBsifyrOpI1/zMIW0EYiUIrgYI1B+hcX5ueLyf9dxTwYH/jt7U19zsA60q2xTTuX/DjRdCxN4RTISyCcDXyuVgB4VPoIMEiAvV7CLtAmAjh2OaLYQ69dQ+eexCxciT9TxrNkv0wu42QscPTiDC4fmc7dExGHUVmyJK6fEG7KOcthgB+TfTFeQNy6AyIUDrX8ChCrDdbX41AnIEToIeS8v13MeIQ/9cQke+U25BV2ECDbwmy9No1+f5WFEmhA+3Od7M+WUAjYX8WEZ+AEPc1RILtop37DebtrJ+moIgLAelFriEq8c9MvnUuNt1QuB/RQho55B7EfToROo0oUv0pOja+B4kt3TfLo2X4abde3gJkLPE8cfMwN6mr1doxB3GV+xqcPyEGge1C82IyFkUBEcUf0cjh/gXpjO5P+tWfdyOxsc+7XxE3EjMR5+IiwGXEzYVHqu9GRPtug9/npZfv8+EpK9OtO0cQDVDG2LMVKATTMKIfnPfHPIOzaveX0Xtj5Hlps2dfsd8WCMdDOBjCYlvz/wXhZFvjtxrRGosiV3yukUB17KA2lwRqFbNb45RpJSnLsunAT4PETZ+/BTnETM299zRiKZba9f7IXOzLyDHlM8hK4s9Q+Xzx2Tc14hlCY9AOfncUTucWe/YxRDBHoQXfgwwYdgNegwjXzoigVOyb9ZFSeAWKCLEhOp/HzymqI+5ngTVhM7TYvc59gM3t+/XRgt8IIdAt0EKciojCUIRsR6Fzijx1IS7mTQgJT06efdDKW4gQTYaQySLgtfZOsLYtt+sP2XtPGnzXW/9tbH22eUHfzkfEtIoIw3qIKO2AENgYFC/wLQjxDrKcniXl1msBER6P9P5BhOhnIW7kNcjHaEt7N3/+Wkp4BqHpswzF5KugabSNtfdu5If0NmtXlivrBcQtdKHxeA8iXh9CBOBRe287YM+kHROQiftGRJ3Z5khE9lRS/rvRnJkH7ETjuHZY/S1W/wFoDgS0wRiG+n1La9OwpPz10Dhkyb2dUP97qlnfzUXO2q9Dcxs0/4ZbWzuR0/c4a8e+Vq5zslsg0eAgRBCfQ1z9aGSp2jRdDpUjYNA0NHmfNSBBSuPDDPgvWSN/as8mWcGT0YLcDeaPg++HEP4PIMuyoxDB2qev+l+tab2Vv1KmJM1EATMZhbaoHcje+DYiUXJ764G5j+cg7ACwuRZvUarbp7PQLncntGucgua4I+vRNC7cEQg5ro84gA3R+AaEhHZF9NI5kzQtRIt8Z4SoFiACtQEieu1EBfLW9k0P2sW7CfZO9v3OBW3qsTK3Qgi2jghCzZ5vZ7D77vVZ1M8jk/Zi7RxmGSQCeyPwLoScRiHkM5rGc+tcpOcRJw5O+q6ewDwU+IDdA/XfEjSkWxNj3YGQ+XP2f5T110bWT9sRjRTmIuS8W65PFiCCuBGRIL+NSMQGIQIX7J1DEng70JhsmZQ3ytrnYlQnmB6odsfk3RWI2IHmRTvCpR0I+Xeh8RlAnNYO4y72fA4ibAMMztdbTtN464OlaH4MsHKmGQw7JuWS3POYgcutX30pbY02Z5nBPJtIWEba7wDiQZ/LrO/WS56/B62tBWjuuHhvqdVVmDbXeA9aaJ20e/LM2UqsU9Jno3LlDCxx7mql8kTd1UvbYSdxzkCr5Ryk4X4Ura777cUi6jMSYS+ARc1PDvW4atugXaQHPh2NjjoYbO9NRzqeDFmNHY84oIlod+gH7j5J3PEORJZ7zyRgviaEsCWKaj0Cmfiuj8RyNyBkcQIyYZ5k3/mpqkPQInfktyJpx91EH6HXhBC2CyEMQjv6FQhxDEbm1ZsjxHORffsl+24QEmc5ktwKIcGFwAkhhDejaBMD0Q5+K4NnCBqSbdEOewBCqnMRIYPIPfoacDxzDkLe9yI902F2PQQN+yfRMA9CiPA465vNgEtDCNtjsQFDCG9DItKdrK7TkEl+KyIuyxBnsgwZU2yLuECHhwRGPxW3zeDw+fOQtXm29eNuBmubfbOVwXeutc+fg7i8uxCBWYAOYNwQMfo/M3huRVN3PURwXh9CGBxCGGXPQdN6AjH5abVvQPPXYe2y34usb/5iZZyKxmkq4nBeA+weQtgccUGebkLzYQAixlsSrfk2QEYfbUi8eJKVlSHjHIdpC+CtRML4FusT5/gK0yL1C1ugSfucFbgMUVE/zTpd46CBStMg9cFrkltbNauzTJQ6qFXNiCCMfcR0UO+GcAo6e+nDyGBiMYRDTPRTKdBBnQ/h9RAmQf3rzXVQrrOYR6Ol01KivukphDzdSTbvWJnmmpVzX678drRYfpO08TKiCfqI5F6NqFB2iynXSaR+RWPR0em/S+puL+jLw+z7R4g6jG8TdT91hHhHIvPuKjJjXoIsA58l7nxT5bYf2TCPRl2Uwz4ZSVtHEfUYcxDx/zGNRiJ5pfkPE/gHWj3uXDrL6nC9kUden2NwTgRm5/rgHnpbT/r/6dYfw4lWeKMQIj2BqEdJYXQjlbyJd0fy7n3I962GdEZ1RNz2IFqB9iBk/GsiR+tHqgwpGEuvpw4cifSHs+zeM0QLxXzYKTd1/7qVczBxbqU6mieIhkC+4akgyVreYTm/hlwnOAnNNYdrqfWX9/8URCCbOvieYO+2QPgmhE9DmG/rehY6QDFAuAXCCAjPIb304Tkd1EHaENyLiNTOaE6WOqhmeLe/AVhXMokV31AIR9oEnG1EaAiE10L4Qx8EqoIUrJvTpxXfIiTOTgOmpr4ZNaSruNIW3Uwklrk8QVjdyPJpui3Kp5PFl5qipwiyQqPBxILcIvb8S7QjPoFoel0FnrV+2tvguItouvs4MCrpS9+tLkeE6DIaQzO54n4QQmy/QTtkj8a9PHk333/P2PuHIgIX7PcAa+MNyPnX9RI3JXClzsr5vBTpv443+O9DG4UPJv36G8RtuR9XxepejgjgKGRo8RjR+ixtQx3439y8m2FwdiAOeipmDYeclFdYuV9GpvSz6A271zEfIftpVvbjwISkrtQisd1gnm8wdOTg2jep63x7ZzFxY5OeDzYTiSiH0js6hFuauuWgm5vnI6nn/ZO6EZFvRxztHUl9HpJqCxuDK9AGZ3yujHQNrCBaWvYaf4trGVrQZvQUCDugE3t3hXCerfGADlQcQbEV32niKO+wNj+ErDVLAtUM7/Y3AOtKRsheR0P/m35Q1zQ3S+6g8TgF3wkGdL5SDxJzXE/0l8kjoQoSmbwLEamaLfBfIBFTAI5I2jUTcQKuZE4RR3rgXZHjpSvV/XqawTEFOX3+iajfqCFC5FxGHSHaTyDE1oX8oVJn5Bfsu7xVYxvaQV+QwOpRG4KVdwlC2j1JW/dK2vEL4pHvw6yvfNd9AdGpcxYSQ52ONgQOi+/O3T/smGTslqJI3X7isCN8h+1OxBmOSL55BHEqFevTnyGCOQOJ5x5MynYfpFF2b19EWNPQWB4d3gnnOBqRfOrc/BjRynAWIr41G7cP2n93FxiOrOLqiDAEa8uZxPiMM5K680eveO4kco157moREv8txixDrW53IN4drQEv91DEEZ+TtC0/T8damWORRMGjs6QR+J3wFc312jWs/GiPPnLpB7UmeLe/AVhXMo0Eas9a7wCRq5TbIOzZKM7JL6Q0pIz74jyfLOauBNHUDXH5cQodlvMHzbUYMnERztUIgafOvS4e+gU6CNAX735I95buNv+JZPeHIFFUjYjsOq2v3By+qJ1pRHFH3otpjLCR5+4eQpZ6s9EO1Dm1GvAGG6OhVtZUIoGpoejc2yAfIhcRuR+Tm3HPQQaWbtl6b1L/Bbm54CGC8g7UKdJP4V9uZQ9JytiAGIcuAD+2+4OIPj/u53MJQtp+Su1+SI9yTVLX84hATEra8IlkbEdaG29DhqXPF8CfEpVFNDoWp+OWz+2Iy5uajOkSa/cxiFi/jUi46jY+Q1DoKJ8L2xD9/XwD4ET+XPvtsn64n0Z/rbq102MPdqKTmOcS/fY6kzYsI/rdpXO7WZSXsBd0rulJvnVFnikjSawu3u1vANaVjBEoZGV6+TFQ6YLK6kzSDqgd0ztyg2ePfTaPqAtxR9L08DhHBu3JdUA78BuTd1sQAQloR5yv00Upz6Jdc8UQzLHJ93WE8PzbY5GOoQ783PrFxWg1tIvvsYU/zJDTcrS7PQ8ZLuSJVpV4rH0+3luKOKsIwY7NPbsWIeytkdJ7D2IYpycQIkzPAvon8K2k3BX4xiOO9f8So0qMMbimYOKr5L1OesOZInp/Njb33TusX/xcrXvtuwNz7301GfdORHBaknKnGPznIcLTioj0QKRX+6CNnyPoOiLMTpTzhNSv/UiVrlw7nNjeg8R/e9u9zYgHJS5BhjkVoNvakVkb/cReJ6xpUOJFiDC+L6n3aXtetKHrsXraEae8nEiYTiP6qTmHupzGc7U6aTxdOaD530bzSC89x0JPO30fEprPnVA9VmWehh1FUuZVxLv9DcC6kBES/j3aofkEbz0Gql1QXRnbX9UkrR3dKDbLZ48CkC4aVySnUZqfRFZueydIZw6NyHKWvTczV/6lwMftm+8gRfh99I4ckIpfAkL0U5L+ONTKuwCF4wnIofPTyBIvf+BgxfouPf/I6xyLds2OFB60MgKymDsQnSvkIs08AWjPIZPUeMAJvu/c0xh0vmvuwM47sra5w+hR6GiOqUisNcnKvZFopOF1jgG+CGxoZWyBrPVSB9pua+e9RGfjgXb/U0gc2kOCwJAS3Y0D0lBFvoEZT6O+Jj3yxPu5KGxV1cb060g3d7k9O5XozNuK9Ih+avLEpO9aaDQqSEWrZ6MNyR127yBEFDuBN1q7Muv/9KyvVLzteToSbZ+FuMg24LN2L23PFKsjlRwsRwTTifNgRCDSoz08ukp6ku8zyRgVrdMVTqRWZd13Q+XoKGb3eTcFOUVv2N+4bW3P/Q7AupCROKVoR9W5O4SbYUUX1Lugmts51Tsg3ANzdi/erXpeiEQY6b1uImKaYovoI2j3mnIatxOV1ansPL9D/il2JIQhm1S272F/HOG7uOSF5H7NEICHjskrnH3xzUREz+PPuZjQidMLCCnmDyK8nSiaWmDfnZvA+EGExNpINgnA0OSdzRAH9VliBAmPZpAXm+Zh97BGweq/Belm6ojLyceIC4hYTQaubzJvXkBRMGblvnOE7Mj/WoTQPe7dInrPt7yIysfFYf474rB2Q2b9DsNOSb3L7PvZwOjknTHWp/fZ9TAaYxIGZJTzdhotSd+QwPKwzYv5BbDXsEC7iNCnestuRPCW0CgirZIE/kXz6kr7vzMxBmXe+MfLrCf1ef+tQOLtdxLFom6RmjfKOLmgHZ4re0D9VljeAaEjx1H1QE8nhOugZ/fesDksi4BN+xu3re253wFYVzIxxllRrm0JC0+AcAVUboD6tXY9fCXRkdGOLeWQ6vbbgpDtF5IJ/igSkRSdttuJuIVJCcxfTRasL3y3VvKwMXcniyYgxDQYmR8fTzQFr9JI9PIGGhOQ4vr7yLdnIOIKUoTsgWhriAupI73ScfZ8uSGQbqQ3WmjtON/evRLtvjdI6q8ho4Odc+P1vqTuVmQtNTD3zhi08065RUdYeR1YkeJ8OZErm4iI8jiE+KYRiV1RvLhqrswavU/TXYAFi83BPRRxd+n3g3PvvBfNrRet8uz+xoi4VNH8GGx1H2fvXkx0c/gAEh+2EuPmLSFa3nnd77RyDkQc1B1E4pv2ayqu9f9XoJOUt0M6qBXJ+wFx+u+3/88nfbQwKXM3a9tANI+mU0zgv4O4cueiWpGByedsLPM6tmWs5By2LWHpCRDuhkk3Qu0yqH8H6sPj+WbNvj2qv3HaupD7HYC1NaMIBm4tdRaNVkt95dvpvTCaErbk/5P2OxmJFN1SygNdPlCwgCrI1PiLaIfq9bkivGr/HTkNQwYV99EbxnlEkVSq8Pco0HUkIhyIRHA1W+weVXwB0hkspvnx4kuRdZ/HbFtAo3GAE96FyAqxjsRz3cCnk7E53Rb/CkScXE82FunJHkvq7AbuaDLGz6GNQMoNuuGFc5Guq/NNQLPziFyM48SsmS5tSTKOTrS9zDQoaw0Rz+XAnwpgH4i40hXJ+zci0aIHX30IzSf313pb8r0f5z7ZyjgoqfvU5L37aeSsfkejkU7aRidsHnS2BxFuD9D7NeRj5cYO3Va396+vh2n0NvSpEU/+3QAR2mB1LUEbOY/CXkHiyQ8QLT3T3I44vpqNfQ2JOUcQo+w/l6t7Zeve85TcddH6f8DK/IT1607Alv2N89bG3O8ArK3ZFtY04hERh9OooM/nZoSoLwLlYYJc3LDcFqwj3lQ3tJhGK6hg751KFI1di0QhvrjccCEgg4mfIqRXROhSP6QORAReZ9f7IifMOtIRVZHIbZB98x7i+T3NRCMpd9CFkOhyg/dQFAXhRiT6eCJpYzu9jRNmooPmjiRG3TiJyAk5gZ2MrOfqiCjfgHb2eU4lP0YecDQltEsRsZ9j9e+FfMJcwf9sDka3hOtBiHk/K/cxhBxTGHoodrbOc6ltiBhfibjgThSZoyP3/hIkDv6Ewbcj4nBvzsG4V9JXdev/KrLM3B4ZH3RYm/Oiu7xpuM/HzxLn6neJ5unfszrutTregKwVlyKx5HnEjYET63x/dCDCc5rlHuKm0edmB3CM1bkxOsk4v2Eam4P/BSvXI/1XERF1q1p3EF7d9d3s3QuJkT3OsnG9tL9x3tqY+x2AtTHb4ulIJtQ3iTqUX9C4u1qT/K+k7A8jHxxHjKkoxLmm1Hl0AjHiwoVEg4aLkOK9BRG2HnofTJf+d73IoUm7h6KddaBRPPEY8nNxDuvj9v6HERLZhuijEmzB/Zzol3KGvb8f8VTUS4jHsqcitiVEsYojltNQbL69bCyCfX8NvaOGF3Eu9dw7c5BoczZCSHUkzhyIEJ6bd7uVYwsiKg7Po5jFHRJvOQewp937kV1PRaFsBiJxmm8wOmxM2xDhrBK51HMQ4b+KeIjfE8TICXmxYL69qeVaQJuAv1p7q4hovQU5FF+f9Emgt7GNt+tRK6Pb2rCdtfMue74vEoGnPny/QGLiQDyqxbmZe5Fv1oTkfd8QfB7zlbNvbiJuEIrmcS3pO7fqOwNtJnzzcJnBPopG5/KH0ZyempSbil6PQwQkWNsm0djXq5svJEpE7qVRbdBOaTTRGxf3NwCvaIYtA5wY4PIAN9rvicGMB17sFC2uop3RXUTEfRXaka/KxHQRQYp8n0yQgVtq1YFrDYaTExj8173w59uC+xMSaxTJuv9lSMAdGl235cTHLZzO6zUp5HW/EIngHkDcXKonq6BQOWNpFH/MRwh1JhKXdAHVXNlDEMGrIqJWQyKgFcSwPIHmYhVvi28g/L3rEDKu0GiU4HmB9dk2CSweYcMPMNyNKC6sIz3OSdZXmY3d4zZ2rjO6FelI3H3g8aQvDiOehNuN5s8cJHobaO/tSOOx6/cSRctPWT2/TmD+MFEkm/ZTnSjam0rU8fk7bfTdr37fDQamokMiAzIYqSNEnyWweBsmovBOrv/J69yuJloKOkxtaIMxnciVH1cwFx8z2GYQ52BqEXc54pIm0nueLEYbjj/YPedoHyso5yBrzxQazfL9OxfdPU6xCLev/C0iEZyBOPsirv2Ta4qz/lNzvwPwimQd0f6PAJ2ht4Nth93/R9AxBAMoFlHVLPt5Ta57WUhUIKc7+Eds4c2it8VaIJ6R44hmRyQ2q9rC/WZuEk8nKqj9XhvSef3Evp2SK98jPcy172aicDk30diu35KI0RAR8V31CESI2hG3UHQcewURsRMRB+bEw8145yDikVofpm1bltxbjhT1pyN9wyO2qIusBvN97oShIXYc4l48rmE38tkZRBSxbkw8vfgRxBFeafAfY/d/gpDIYCtzkI2Rm2UXiU3dbP4TRDGsR3P/tpV/BpGDeycSN1WRyGuwPZtt3x6VtHUc8AG7/yWr/xYaLeSKxGTez/dYey6w/vkxOmzQdXA+v4rWQRci2jOT8urWxycjjvcpeiPhqtX3Cft/GSLidWBGjjC9KRlHL8fnuVuaPm/wdhFPYA6Iu/6k9eG/aDRUcpeHBcm9LiQSdQvD7RH3FKyf/lQwvsHK70GuFtNzdQS0Nj225RIa/QuL5vILa4Kz+h2/voy53wF4BYjT0UFe3CsLTVSrQ/tZUYFZNIGeJSLUX9K4gw+2+PxAv1uJlnMpYtjVFk8dESUnJFMRRza3oF7fkd6NkNgEtDP0aA0nI3FKyok5MXAk44fF+TlLVxBFMK6Qnmnt2pfIhXhg0lTP0cz8NkVi/ltBHNJxKCDorkQjid1pJFhn95qgeifQW6zqfZ/64+RjuLUgwnk22iEfkrzbae9OJir38w67fyBuTCrASYWLKJ7SmkfGn2ryvjvVukXjl3LPv060avSx8TnQhkSdn0MGCw/QGFS4h4iQ3XozL+asoU1Hld6WjQNp3IR0ETm6IUg8+EkUleNXBW3ua16kTup+cKSLPa9DnOhlRJGdnxw8BM35xxM4vB2+UexJyu9Em45L7NlERECuRATOua0iAjwRSUdWELkt13Vui/Sl+W9mEUXDo2jUoVatnouJ6yZvwp/O56NWB2fZe0f3O559mXK/A/CyNQy2Xw+6KhrAvgY5fA3CD+1/B9S+ocnZDAkvIfrWOOI4z/6n0ZWfQr4pvjA/bZP1eoQ0051/6mgZaNyN+RHZx9h7j1udX0XE5sc0IoYOJJoaQQxl5PU8hHbeNSQanEw0kOiL6Lgs/2H7dQOEAJyZ9PkGCMGkRM8X5DlYSKLkfY/67Qs0tR7LkCNxXhnvYW7aiaGdplg/dCOu4282RhcijmgejYF08+27i0R0lYPxbHunK3f/HUSjgoA46XkG31ii+LYD6QdHJd8OTeruLdbROxtTbBHm49SJkPu9SOHegzisPey9TyMi4Dv3BRQ7oC4Gjs31+99pJCQb557vZ21yvWydqO/8FpFbzIgBg/24+ReI1ol9GR74BqsDzR+3cr2TOPecIN2FjgP5Uq7MNuDzBsvViEOaZ/f3tfufScpy0/V8PMxg8N5g7VlIjGsZEPFJxcoL7PcsGo2PptLc7aQCPPNNWNoD3SvDWbn8H0uk+h2Al6whaay8EAhikVdKnPL5HghbQ9ijOcJeQAzE6RPYHWp/gg4FdKSXTr4UwfyLGO7lShqPZqyxHAAAIABJREFUNjgZ7Yj93XFEJfNbiUTwUkRwiiZ8kbgphbfNFvrFSNznVnnfJepX0u9bkvvjiCeael07GHwvhhmy628SORC3SPTYavcl9cxDIi4nhKmYqhWJ9zwUlIu1Xpe0azriKhYiS8ZngesK5shQGs2O85xFB9r9Xow2AKOQSM6f/xIRP0eurls6wMp3pDXKrk+gUfw3D4lTXcRaQ0T9LYgjuYootiqCsY58hjYoaFsL8Edi3MXDk35cTBRN/obIoTxO4/xZiAh/J3Kmdl1LF+L6phHFtqmF6WZWtnOGm9i1Swpqdv1BtHHoQWsob7LfjUJR/TeyvgwogssVaK6nyD7Qm2MrConUijYSTijHEjnCkYhL883iP4mGIx0Gg4sxrySKLfN6UieSPUj6kM7fdA3+i2LrxwDU9oPn2lcBR7XYN5XG+/+Rsf76HYCXkUCtUcTxeyBsA+HqxoXQLN9DRFiuc0ifV20R3oOQ43uszCF2zxFRD5JXT0smsyOWXhO5YFHW0e5uRgLzVBp3vouJ/iVz7L37iQFUHdm6f0gHQp5pHLs6Qv7HIH8mtzBbhHxk2oBbkzo9Ft8/rY37ICutZuf4pP19OzHyuXMcbXb9Fmv3wygMkccafNp+W4H/yc2PQUiM5P16NRGxDEbiIA8U6hZ8RXB1ILFaFRlApGGSPOzTl5J7OyKxbQfRGKCo3a1ErmspInYZcTf+S+K8OLNg/t9MjPwx2X4vtnGZZ3koirYRgNbk2/Tgv6I2+//xSAznpvvOXTqByhBCn2jtrhDPBtsNWTdOz9VzO4ob6NzSM0SfsnZE5McTdU4VNI8fTuB/A40Ovg8T9b55opVfPzWihaC3834kYUgJ0enIQKWGfLm2QRvJvENy2mfT0LpcmbN+HQi3QNvKwif1QaD+I6Ol9zsATYjNyQjBttqk/Hjy7Cs2Yf3Z25Elj8ue20bAmdNsYlYg/BXC7rlB/iWEg+3/FyCciiKNbwghQ+c7rWRSefbQOc8QDQRSZNZKPCI7r5NKy5hPYySJJUiBfhwiBAvQDtWfX0VEdp9Fu32Xx3s07Dbs8Dfrq4Bi+B1IROa+ABch/Zkj5mOIQWTr6KC7A1CUACeK7UisssTqnkmjldcxRD+Y64m6nFTsmJoPB2uTWyVempTl5uH72P/bcnNmHxpjD+6cPDueSHzSmIIeoHZgcu8wol7E+9f7ZA6NsLYiZHwj8AOrYzJCWj9HxG86vXfNjtScIDnheRJ4ZwLLKIN5nF1vQDRAWQB8OHn3DCKyG0NjTL/BVucS4hEfs+zZQKJVW94tIW2794X34/fs+xcJlF2PILo+PI7Wci0p/0pEUK4kzq0nkMiwbv21MZFT87pvQwY+zqX+DhHEX9nzmYg7DAh3bEiMZLKCGBqpG21O/GyxryHJR2q510Ncj0XExA1EfDOQOvBPs+fL0JpKrQ/7zEMgDDWcdBOEt6LzpraF8IMEd22XvD8EwsP2/HMKtebhzEZbuevZ9VEGWyvisg/vbzy/LhOoT6Po1AOQUrIdseOfRruWPW1y7kwUp0wnHodx4lRb/BV0sOBGECYlg7wHhL/kCFQgclAdKGRJHxPKI3A3M6iYi8QJLTbhf45Crbj+qhUhoO8S/W7GET3kJySI18UTNVt8d6IdqXNoleT/Mwni+b19M41IvPzXg87uQvTA98XsiOMOtOP+k91PiY9Hnkh1abcR9Q8DkflvhcixeR23GqxujVVHptYDkVjo70QdliMtb38V+Hsfc+diImI9HSGSqsG6mEQ8hhBoBXFgw5CezA0OXNeWEpRlxIP4bkVc10SKzzzy8XgacT8T0ZxxK7LupI0VohXaBDRPNiCKw9KYhC7anEoU07nDcAA+1KRfBhkM6UGJKcGdYeO1LzH6uCPtHrTBSZX7btgTkPFLOu5pRJJWjIvOEd4a2li9gcajP1KO53K07k8jBgv2Pptu/dsJHJ+U3UbcFDp3tBxJLeYSYzTWbQxHG8we59DFfj5n90Br0OF7lBhoOSBu7VmKDS58bFOxYFHu2QaqqXjvHgjPQqhBeAbClhCutWct9OagEgL1nTyBsravAHaxZyOxwL1re/ZJtVanLMueRjvUY4FbQgjnFbwzHXELd5Fll0+Hz++AZsd6SK70OoSxJiO2az46d/koZJ7zY7QqP49YhyugeoQ+T1NAC/h5tNh7UOw50KIZbf+vI4rzPoS4vl3RRF0P7boWIbHL+wzUG5Fo5M2IOFft3aWIQ9vTypuKFrlbAG5ldXp4mX/a9aYIAfhzENF05bunYUjxXbd6sS54wuDNgI8iRP20PT8EIUfXoXSinTpJOd5X3cD6aCFva/WB9BhPItHnHGtjmoYgg4/1k7KCtXOhwbM4982bUOiYgXbtxGlbRHDbk3cPsLJfY9c99u54RLi3Q2K5d6IdOdYfAxJYPOrEJkTdyvuIFnpDrI+8X0ICmxOw/RHi60RzZHjyTsXa+WwC97ZI59aC5luWPLsHzRdPGRr/7VG0jsE0puVo2tdy9w+wtk1Cy2Ujuz/FYNvB2ugWddivzx9/dg8aj81R/3vZA9FGC9S370KbIS8HZDbfncA0CM217e26TnRB6LR7H0Xj4GM6EfWRi9mGojGegebKkKTtA9Emq0I8gv5mu96dyNH6RnEXxJmB+njvXFkQwzl12PdFqfO1MHhSk4cgEYCzi9NR5ztuA7HPU4Ar4HJCODLLstFofgxC82820u3dEkLoZB1JeeS7VqQsy45EupHRdmsjdITBdmhirCxtmr9xGBJmn45kC4cQZ3CztFnjwn8RPPvd1X59QS5GC6fDij4YLdLlyfvzkQikhgwe5iMu0L95o117HeuhhbTE2jTZvtsJId7N7N3JaEHsiBDQAYgT8+McnDMaijivEYggYO/tRCNyWYB2WR+z72ba+zsSRTwDrL1b2XWK+FJCh8GxDBHeNHUbLFWrr5Xe6Xlrc0YUDXr0itH2jocKWoKQgCOHFQj5bYf62An1cEQkNyByW0OI5yntTyRI+9uv73bdOXUF6jv38xqOiPoIa+tmBrtvSrqsDidSoDH18t+B+rbTYN04gWtnJFFYZG0cZH22g8G8PpEI7m7vbIbmlIej8j7ydmVEv5wixLmxweKE1/tgZ6J5/+ZJWVjbPHlb34s2E0PQ/NnE2j0eEYhtDc4KmndbEon/gfZexe6NRuOD9ecg+340UQqxnr07HuGNXezdTRIY51t/TSYSbm/fvsQ1W0P+UeNR/7vYsQONawBej+bxJkTObFDSloA2DOvRe114qq4XN2CAdisnI5GK+w18uuDDgtQL94UQ2rMsOxRxfhdlWfYQcEIIYcKqFdmPqb9ZuAJRxCg0HvsQrdeeRnHBbge+1eS7FqKI7/I8G9wDYRiEpyDsAuGWhD3+QiLiu9dEfAHCpc1Z8lQ+7tl3Vi428SMMnM2/2fIKpFdyy79rEDJxE9eA4sxtgMyke4hizN1ydT6NCMMoJMNPlbHXIiRzGVpwGVpYVxlcS4jisDrRPLwl6dPDiaGAHiceSJfq0pYjEZOf9TMGIa/f0Cj2qqP1dquVcyJRtHcmsKjJuF5q3/+FaFSSnit0ILLySq3KOpDFlOu8biCK4erWT3NpPN7bYZ1o8PsBePMR8/1XJOLKkJOtB5Kt2f/lCcybEcVbLuKZCHzUnncgxv0gGwuPczeVRj84t4BcQTyI0uGcibgqFw3eRaNVWQvS0+yRwPUDovjSI158rKDPMytjN+s7d5OoISvEvC6uShRVn2ff749Ew6n+ZQaRK5tG1D/tndRdQYTxnqQPfk88DuV+RPi3s2dHoPn/TFLPYnTo5Cb2LHV5qCV1fYyoXx1J1NMGNH82MFhdJ/puq/NH1r9uCj8HzY06vSN8eK7Q3Fii/joaj+zYEenJO+36WxAOt//TC0R8P4Pwcf2/zNr2Tit7vdzYDkai7Af6G9evEj3obwAKFscbbBLvgnarX7QF8GVbJDPRLjGvg3oE+KoRqAYdlA/i0RDeD2F47n5KoMYjQ4k5EE5oTqA8LyGa5VYQor4tt3C7iYYBRYTOka4v5B4aHVvHo93YIKIJrIsY3LghIBZ+MkJav7LF4vHIPmt9NAnJ9YcRTXwrSDThSGl8MpH/m0hQUrjn2XfTyfkPoV1j2tY6cnQ8jajLaTM4vk8kjrWCuXCmvT/FrocTNwefsXtvQsjfrdZqCFmnvmr57MrtRUjk+Vor6ykao1pcR9StfA2zIkzgO5ioh6khYpmhza/X9SB2HIR94yIxN3XeDDkhu36tBzkI++nAywzWPFHI5xaEpFN/p38m9XiMxUXEjZ8buxyca9deBs+9Bs9XbR44MXf95alJ24PBOiBX1nCi3jO1Sq2jOXsVEst5P3diFobEaA4BbW6cM/qrPb8YEehxaD5+ARGTR9GcdWOSuvWxE/f/QlxXDzKYcMdj1yv5xsA3M0W6aDcSmkeck27Z2Jp8u4DmpuUv5k2gvjTBScMhXGL/H7VrJ1DtEAZAmJi8fwfagD+kcR5KxBXrIa7+Y2juDUDr6r7+xvXrJIGyifcThPwXIaR/H9Ea7WiiYnocdoSADcAMYNkIOCO14vNBvN8mw7HJvTyBChC+CGFzZM3XZELVbBKm/ip53yPnpr6HdrI9aJflzoROkBYiWXsaM68oNFC+7JA8uxtJL59ARGgk0ik50tvZ+ugaWzD+/TXEM4OeJO7Qneh1IqT9U8RFbE8jp3IWjRZw5+TgfYJGzuJptEv9KdFSLnXYTBG5R3uuA/sn988mIta5RJHNYzSaxKd5vuVUROPtuzMpe2TyzRdyc9KfbZC776KePCfm+sk8Af8kdhR67r5HZaik3xjcqfm7I770qI18no44JPcdW5CM7T658XJjgsOS+2cmfeV1jkNOuG3ABcm7Xp/PqQq2IbLnHkbKYXsUIcq9kU/aJCKXmY6f+xzdRORs/LDKr1rZB9l9NzgabfD+GSFq/y7vt5RfSx66ybnXDkS47kv62K119yJaUn4WEdE/0/t4EF/f7UQfsmb4pL4+hM2MyAQIf4ewPTLuOgjC1xMCFSCcBmELZPX3L7t3NFQzracpyNrZCdRIa4ufX3YvOaf5tTX3OwAvW15DP6iAjmq+prkFX7sN9sTcJHSl+C127060q013TymL7z5LQ22x7WvXbcCFyQIfQhTH+MQfSzyF1YnJCpr72PgiTp/fb4vwGXr7/NwO7Fg4YeT/FIi+Pd2I8HjbZiD90CesrA777hC7fl1S1ggkEvKddQ8iTFdaX15rC2onZLJ+GdEKK80LEYJxBOTEaEckKpph9WXWzxcRCaQjvnuT/qkCvyhoew9wUO6eRyVPw/i4eK8KfCP3/h9IY67F+w8TLcFaaTz/ywnEOUTx5hDr8zOI0RCcy+qxeZHnfOvIP+j3SHzrp+26m8YZSJTp7Zhm471hAueXrJ9HJgSqYn397qQNy4hRSxYSxd3vajKv9qKR0AWbF2MQAUgjiXcT11UR0fGDMQNag7cgQjeZxnV9IY2bAY9cXyOG43JjiZQjTx3fVyBc4EGBH7H7bsAU0AbjaSQ2bBoO6hrDPWuCs0LpB7WO5TWMJBGQP9QezZH9w8SIz+ni8B2zL4BrkIn8YBqJlItsUsfC5cRjKK62cs+k0azXF0VRvLpziKbjTyFEvAgh3W8kdRXtJJ1ry7d1icE5xhb4n5BZtb/bQwyo6t88S6P/kUfT2AUhm77Mw6cm8Hh7U9hakUjICeFitAnw59MRx+YhdfwwuGE0j5Q9BiG+fH84wcxziLOA39j/tyDxnY/p2dYnnfb9r5MxuxoLxos2G6kz82C00ajTexfeiZD7uQWwX4c4IxeN/ZLo3HoDUVTZjcRoTuzarR/TYzVcpJXOjcv7GKspaB0cZu9+Pnn2XhoDsbajCBd1ogjzPKQ3vAPN2WkUR+TPz8167vcFFJNwNvCg1b+cGE/z0ALYi2I6PoI2RnV0enOVKL5P18PEXL9NQRzi12ncOLkfWLpWfJ3fW9DOAFT3MNyzhgSqjCSxzuUYdHGVB7oNwtHNuSeXaz9PI9GpII7ko0i0FuiN+IIt6vSsm9/ZxPeQOEWn0boi/vykrtEFC+90ez4L7YR7kkWTh+MGGh05z0dIZTTxvKVg9+5CiNwJSDPC3U4jR3k/URfSQ/QxGYSsDH+IuI/J9EZO7tjsZbcQwz+lZxaNM3jPJoru6uS4HyQ+60j73u4/ZbB2EcU7c9DmIo0uMBv5Zj2DkJQfyeGRxw+0378T/Y1W2P2QwP4wQlLXIS41jajRgsS83hfPEv3Wfp2D+w1W3oG5+x4FxAOdHoXEqd62xURjnKk0itRa6E2onQu6Ghm1vB1tfj5AnJcB6YJSLjrlcIrmShVtvh4mitGcGO1H9JM6MGnbWwzGtCwnrHfbt/sQ9b0fKVgjztVXiI67+bXeQzzl9xgrK3WIHkpvzrSC5nEVraGv03jcSbAxcJF0UUimOhCOg/Y1IFJlLL51NsPRXVCt5axk8rmKAsV+rXhRee4k6oocWU9CVkB+xlEqlvHdvCNp16nMQ0TC4405AulCu0rnPByBTKB3FIqZiACch/QaO9AY1drzJLRrrCKTWddrnZIsusXAr5LrebaA3RDjAhoDrlaJx19cZwvfo1u7E27R4s8vyApCyC8QEdrVVk874oryx9N3I3FNJ0k4I+LGoErvoLQewf3y5N6hRJHqO4jWXA8QreKetu9+T6OIpwsRmBaigcNFSdkDEfHrIJ5J5PPD2z4FWcRVECc6wcq5FG0Y2omBX7uAtyTlj8cie+fa+amk/K2S+zcjLuVxGjcYdaKFnOth7rFn8xGxzkeIbzaWPl+vJhr5fIwYQunO5F03wLjLrh9Am77UmXcM8K9k/K4izvtT7Lv7KTaGuQ0Rii8h5+NHaNSdBhoPq/RNwUXWR054p9sYftvgTee1z9cuYsR/J5qdaH58gEZn/i40d4vEkk4cwykwqY2Vi/sMp/3HEqcQ/sMJFPL5uGsPqD4ELd05b+0golTvgHAnLNy970XYSqMV0klE66Q2m3hTk+epeOJ4JOefjHydrqf3oW5L0c5rgk1UPxLg+9aWSxBSG2n1zUdIp5np6iJboPdbeXtbOfcanFUkE9+fRBdm75xtiy4jHjQXDH7fKbdb/c8hxP4H4gmlKZFO++NuZLr8HrS7Px3pAVyZ78S6SNxTJSrJvdwaImCnEy0EPahtPhbfoXZ/FDFa9zwbt3HA08m7+xJP+/UdeRdxc/IbGk2S2+zZYUik+M3/Z++84/Sqq/z/HmI2YIwQERABYVlFFPmhUtxlYVFExHWxLqJiW1AUEUUBZSmrIioIiBUR6QiLSO+dhBYIvSQkkN57MpPpM89zf398zuGce+c+Q3BXA67zet3XPLd977ec7+kFedBlF+uJNla38Tmz4lLfw4RHapvNyXnIVuZpdY5BKq0G5ezovk5NIuvIOFtHzwLu89VN2Eurc7wKSfQ+v3OQlH+0ffddyIN2fuV9/91j63AnUjt/2NbSqxEPoHCB9vTet5BUUnXg2Nfuv9G+5yrwE+z+ofbOmSi+p65eU2HjnWT/n0Uq1V5M/UcQZM/neDPa29X0RCsQ47Cpvf9polqBOyM5nD+CYN0zvMymXHG6TrtSmsu9YPzlUPRCsw8GMs7qgeYADFwOxX4VO9pf27HWO/BnJE67ErnZOoBiE5h1BBQT4LGboPf3MHAkNDcqe3YNdzhScRVPgRBDrpV0h/32ZJUeL9OX/g8ihHUAkXBzXvqOl0hoAsfbeCbbRruPco2ne5C67IH0vnsMZUS+HCGvSTYf/0Z4d02rzJ0nFO1OG29cam+AcvnxPsRt3o6qEe+LjPg/tPv/neboQSJf2Cjr6yqkwsl5+JwozUcZxf8NceU/sTl/eg3XazlCEO4M0rD+n4EIrqfZ+fs0/jcgqSMjkespxwfdadfPRsit6miykvAEKxBC/R4W71VZ7z4kpY5Offgs4YF3I0JwzvQstvHPpT5/Xj783j2I4J2I3MbfhxK/9lJG7quRtLGPzdN1qU/bE1L9IsIhaFPk8HKAzet9SPpy+PDcdv4NV4l7/zpt/qoej+5h+BjKztG0+XyGsvenz8v8dH4lgkWvSlwgmL3Z/p9r37iTYBQ60/s+zl0QwfI4J3/G4xid2F5m1zJcLrVxur16VmWuh5NMB4DVr4Xi+zDvQmg8CtMugOJYaN+o7Mn7AAl+/5qOtd6BPxNxOokywalu4KpBfk02+HcToP8YqeL8mUnIWHogZTXbfITYuivfWIFUOf9AJLRsIhXDNgip9lMGQt8U9yCO8SC77vEpMwlvwhEEkboFeVcdj7j82anN3KdnbVxeYttVcK3mZxqysVxu/R3CxdkGHUznN6f2Jtj9bsqIpSBsWJ6WZgkpIWlqb4E9lx0M+hGC9IS5PQh5PJL6vox6xJ7j1fptjDcRyGo5QlA+t45gVtt4jkWqxtOQDcJtDf6d+YgwOZKabOuTJZJBhiKuQfv26nR+OxHnNse++VRqxwnENODWmrnbCRHGuwmnGm/fYWOGtfE05ZLqLtV7xodratr3DCGHohiyZQyFpSqB6bP5+AmhkstMVoEYjUuRVmIHwlZ5BcFATCcCrcfZur4XwfeTDN33zjw+hgKAJ6R7H0e2rRsZuo+XI6KftRiPGwxsT9lZ5CQim/4qQqIaLpFszkdZEGrS6uEM9qfWNu79G4F6YeK0lQF4T81CVhHSmhAmt4UUBnCTElD0IjtSF5JKPk9ZzeeI7nxkPO9DQas/oZx52zm271E2nHsiWOc+fUzLETJy4D7e2m0iZDLd3vW8ZLdRTuZZULa3tDTaEmq1LqSyKRDnvcC+/xO7/+PKOhxkc7S6cv1+yhu6QSR07SKSoP6TPb8eIr6rMNfm1NaZxOZ/CCHszxO2lh7KiPFEZFtbhpDZQvv+gYhwNO1b5yLEcyNCWsuoV8m4bbHH5qkDIcaONG+tuOQB+/6DyB55FrKz+Fo/ZWMpkDTg3oiHUVYXLbTrbr/cExE8L3deYMX50rztYd+/IV07yea4l0i5VM1q0QROqrTl87BN5fq9NpdXEKrHL9p8efHNdsqqseocubRyBlKrrkjtj0ZwsxzZQAcQA+ZEZGtCwvOYSSfAXk3ZpaYs/U5BknomHE207w83uJhh63F3DVxcZnPpSZL3rdy/hWBQFtM6li0fw81RxhWHU8PIvZyPtd6BPwOButAWNMfF/CnHeIJj8aDBAhGSQwn703uQG/W0yvuPoiDOmwhV00rgTOvnN2r65984i4iP8jiMQTvfm0C6WcJxLrsfcZ8bIqnucAIZzCGIXCMdhW2m5WjTHkDZ3tZPecPOtfm5z+65KtD7PAJt/LvtuyOR1OdqEv/+qand1UTtnBWVNX0FkjY6sfgshKBcvbKX9ff49M4IlJImE4hV9mwfUabdJZa7EAI9HiHXrMLM0mQ+nEnpJMqzryAyaHdRH2zt65Ul4wWIwH4TwYaXQ3EGYhySfg+hrCbqIgXZ2thvR8TpV9bHJ9K9fe3bl1TeyXkCz6+sSy5p77E+nyQkqKewkiDW1h5pXJ2IcD5IWc2X00wVSMI93vo7m7IXZQ7fcAm2z/q2L4KvXQi1mydnzcxAgbQE/4DswL5+r7M+b0kQ9Op+7AH2sOeetP65K/nZwJsYajcdh1Tlng1jFsqcks0Brpb/LvUOTmt6NAmG9+C1jYP/RqCGJ1A3VxavGk/T6nBE/RzitqZQDpB9CsVuOMc9izL34xyZE48bsYKBiFD8d+pDRtIdiOC5Tt25uVkIwY4hcrW5HeL5ej9EYuMqwGb1xSKCSH7Gzl2KuY9A+m0oKDK3sTzN7dWIyP0SEai5lKUv55SnEE4FTgS7kH2uh7KK9XxETM9LfXqaVAgw9e0he387hCgW2Rrda229w57bknBImI2Ylmm8MAzkcbg0dy1yFHD1mY/3ZiL7+XtbwOJXDSbarb8FYZO81fp4m82jZ4tYk/41EEL/PSK6/ShUYRMb/4mISVlm89qPEOmn7d0zavq6BQF7XotsE8JW5OVBliBEPkAQyh9a3+9lKKPmzjS3IBVbD3BIWtOqV+FRqU+7Eyq8AWv/20T+1CWUmYjq/lpCME27ItgdJLJ2XIlg5WxCpegS2GPI+SbHO2b17w8IrcTWBIFwm2bek51IkjzT7v2UsqbEwyCup+xAsib4KhPGAeDbaxsH/41ADU+gHqxZzLlpIX9PcGTu8eRc1yBSTf268n4fIjgbUeayVhFOEQMIQfqGWWpAtJCQWjJX5yqTQxCn1Q/02xi8oJvbSRzYOxBxuIhQPVU3uGchmEI5qn8iwbW7O/zimvn7TWrPkYMTwynAxTXvrEc4e9QdHYTnkqfO8aj+z6d2dk6bbtA27FuRveSTyFjtiKBp/ckbvbqpvRDkNERgWlXy7UPc7DeADSpja0NEpB8hOc+kcRZiHm4kuOHX2zvvImKkfmFtOIe8OeGB9sOaudwSOb1MrqxtXuMFCHZn2JiqDEqehzmU0+w8iKS0jyI7yR6IyLujQEE5OPlcghAPEi7pz1KvRvfjQ7md1N7zMIT2mmch9/c+Vnl+JLFX/H8X0lB4f1cgBmB8zZx53yfYml2UnvH+L0CBuhshWC6QyreNKLeR2/R91U7Elc0mmMCPE3jFGRl/91zkHDSX2Au+rgchNa9X6PZ1dEZ5BRHoO416D96TqnP+cj7Wegf+1wfUGgm5B5c7FOTgwJWIi/dAuk4CoVxNODI44XEi+CuE/GcSNoyByjedaJyVNsZoxNVn4/Ai+/2PCFEfjaSb6oZrIKTrVVxdbebODzMr87ELYeCucpjjMI86e9alp6cQ9+wxLOPsfhdwUM2cf4SwZ2XEcIXN5VnpeuZ0fWzLKWc36GfouD0De17fu4iKqr1INdlhz3hKoAMIY3XV4O7IyT3qFmGeUGikAAAgAElEQVSuxPZuGyI8z8chEZJUk0jCuwNRZt2Dcp+fW2vTYe8cu/YZu/arNI//DzmdOGxOpiyhVm0R04Bd7N2PITiaQQQILyOkP29jESF15fkdSO2vopzrr85ZZgCprArCxT2v/TXUZDZArtmT7Lcj95sQvPn3vpiefyP1iXIbCOY2qbSfveuaNp+nEHFMVSnVJf2vIVX9D+1dj8Xz5/dDoQpOoM6zORhgqENWE0n7q5HHoc//vURW/QZSe/vYHknzPB6VAMnxUwVlZreVtH392sbB/6v4fG134E86YOMCjirgogKus/9HTVTNliatiVSBuFNHvANEFoL8nnsg+TsrDeCayAi9f7o/QL2Twbq2YVxF5wDWRLEks5D6ZEbNuzmuYp6dfwUhoJmE/WuWPfMkQqauXtkGeR657Ww6sJ/15+8p53wriMj5AniksuGde/NASUe6e9lG6rO+zCNKHswZZg3cpuB2m+yqnp9ZgJDKBYjA+rUGUmFdYL8/Rjmj9DhUasFzHfoa3YekVbfJvS3d/xBylf414bE3DyHQLiLZrsfrzCOCn99MufR409bnS/bOpwkbwRRgVZrbD1ufZxDxR9OQLWY9ojxGB2GL/BfkLfgMZdXTU0glNT3N5eWE+u2rNjcekDtoa3s+kq6rgaw5K0Q7kW+w1ZrOI+DpQSKL+GrEqDjDcKhdc1j5UZqPNkLFtZJAxv7/GZsXzxjvzis3I7j21Eu/RVoJh/NZ1pcVNs5ZNjerKWf1r0P6t9rYrkZwlfs1kVDzfa7m3WqV3VGEne1+yhLoUkQEnUF2BuIz6ZlLkAdjK9zWjcNXCxxZJIb05XCs9Q68SMK0c6EksD0FdBdQpKN7AAbuhMXvGuqZk48cu+BST4GQhSMvf3a6AdNGSB3i0oojhszxzEfG6U+lTbGXAXQXknhyue0itfOMAd6s9O1l9s2t7HwEUkENIu7qiPTsAFJhPEjEjrjaqY6L9dicbhQTUw2s9VRDT9hGqhLP6nkXoW504/eVlDnLx0jplVJfvES4u1//BiGNs208OYNFXsOJiPPO65HHUbLV2bdGWp88n14u/30BgWy2JaSYAhGQ7yP16nKErHxuOonS41+38fyGsnfiqQgZfsva/ASSKrMb8nOEvWS09d3H8TO7/ggwpTJ/uzE07s3h0h00LmBoolRf55yRpInsq04U76zbh/bN7OE3l/CYc+/JFdTb1bIUOIikisk2F1UCuBJ5MnobJ9T05eOU7YsDSEqbnK45Y7mX/d4BZUJ3lfr+lJnMCdZGLl2f/y8lVH+TkcTqsP5me+92Inwg24MLhAs2Q7FmVYa6w8Z0dvqm98M9Q+vwWj/wxK6wYB7c3QpH2vUrC9h5rePzvyoCFXn1hs1Q3oBmDwwe0tq916WPAgXpHUcguW7Ce6gLISS3VzmgOCAuRZzc3gkIDyUK/2VE3mfAeg9lNcPzRmGkYvC2nya4qEdIrtoIGTyWvjnWNlgO/GwiCWJPpNc+AcVYXU+oJ+uMsO4E0ENIRnVzeDtDVStHW383pMyN9iOJcQZakzOp2CaQXaZpc7QnKXYqPfOMrc3F1GeayBv1RmqysSNpoZ0gRBNtnrqJLAjbWD/cE3I7xDw4p73S5nE1kU1iOUPz/G1JIFz3Bj2bIKILEVHYhPBAG4ccAzoJG+YygkBtadeqXnttiGi4LanV3Cy1PgxQSQZL5PdrI9RW1XVan0hse6/1PcPcIutHE1M9pv69jXJG8irR6rVvZo+93PYgIQlfjeDoOBQu4QHHGV6fQPF/exHIfQkqeQ5RTdnzFy4i1OyLgffZc9dTVnFOR9qUKkH1fp9O1DHrtDUbTeCjJZQ9FFci5xFvZxbhWdqDsod48G8OTq4ei46Aeb0wMDgMfrSjUbxMUiSt9Q68SOL0QhP//JGSvrZaUAcCN/reTQTS+eFqpgIZk/dOgNaBCMXySps5ENaDAq8ikOKvKRt8FxMqqtNtQ0xFm/prhBrgUcSB35Tan0c5cWXdWAcRNzvNNtYzlHMGujT5fJAf0vtfSKid/Jm8IVchT6YtkdqlD6laulPbt1KuB3WwbdrVwBfSda9A2yTq2Gxl97YjuGHnQBcTiUPzmB2BuKTaIAzonqboaMoE6gZ7/hwie/wKzAMz9XFHe/+WBAPdSDrqx8IH0vOPIYJwMYE4u5FTzSBDkf92aX6vtvcnIQLqBGqErX8v8i6cQtmW2k45oNbnoGnPrkr9KBBcumPHd+y+h2lkpqgNqcIHEPHeI90rCEl0EMHyk8Asu/9OlM3c1y5LvN6fuTaPY1O7F6V7BQHn99n3ZlBOkVUds/9fTJkpPBV5KLon6UKi0GQP8hJ0b9kZhM3IYXqfRKy9UOIz6dvzKcPkQqTJyHhjS6R69fCGahb7futzO+UsN3X7uwE0D4P2bobPN1pzdL1CsFRbWuelcKz1DqwBcfpzlM3wzXQLYd/IAFUgbvQYtIm7WgDHFNuMq5EHX5Y6tjVA3sOAfBFC/CfY/SWUXXJXIs8s35A5O3j1yH3pQZLgkUg6moLsXv9O2D7cFnAVIqjOmfvmm0QEOrraaQ4KOHS1UxNJTmNQcG6O48kI4VG7vhRt8Pa8ngjJnm3PPofULb+y/pxg193zKtul7kf6+NdTzjwx155321E7YiTWQQ4nvoYuOTsy77Briwh3+HYiY/3zTIX1+yabGy893kAELkvkv0XE+qHUv5UIga0mHD16KbtTjyWQXS+hev6dte9ZERxuC4L4fR2pJNuQndLvu1t7lqzvRVLh4ZTVyZ3WT0fkngdvH6Ru8j4fOQSBBOy612LOUZhhpBMxHyMQDHbZ+1sjhwNn9GYQqtunCJXWagK+PL/kAMHcbWlj60ROTccjQpb70IpZ9WDrAkl5T1B2RniAIFBjEUHrQXvEPVx/QJko/j9790YbW923F9oaHkaZCcwq68donUGiAPrfBav/Wst0rPUOrAGBKhUeXAPxtcjPXjl8vqt5SEKalAAiP+/ct6fJmUiUwz6YQHbHIcknb4J7EWKaT2vR3PXPs4gkqK7eG4eIWw+yC7j9Ittj7qGMRK8CHkvnTkjejtyjXTXWT7jHP0PZS+oahqrvPIXMgnRtPZsPf88RvrfvXGVXLeBJB+/F3VzSyWNbgKSdBuJ42wj1aYGI8tZog0+1Nmel+wuRY0Aboet3Q/22CCksI4iSv9dNpAlypOrZAAaR+qYNIcaTrb2PMLQEwzNEtef7rL+jCCN3H3KfP4lQcVWJsserPYWS625m7R1q77wW2dXcxuUw9t/2XA9Sge1OSFYN4Bt2/3Z77wjKto0uyuq2S6lUEq4QqEHKCVCzl2ATMSRtCZYaQEdNWx7f5u9NSfN6RxrfXBvX5+zah1Ib7ViclZ2PouwA4tk99kBZH6YB77c2ZxME3QOCq1UE/HDNxFxbH4d3d3D6BbLDOk7xGLQRqc3xlMNBfO/4OG8jUoK1wmErroW+F4MXK8dLutDhX+5DQh5HGgC2I6+wddHGu7cG6N9YwMafhcGvQPEBKF4JxW1Q3ADFW1A55NdDcUqa8Oug2AGVQv4nKCai0sjDLHBG+l7JdQohYTlxcmmjysnmYzUhAXWj4NxTEJe7L2X1yyU1c3QIUaW2k4jJyKUKOm1TuTrKbWXrUSFQ1uY9Nt9usF9BFNrzNqciff05Nv79Km04N1ggp43stebz4MHAn0R2r5vSPa8oegryVDqDyNqeudsrgL9Lc7AaIfbsov8MSf2G1Suy3x8kEMQswluwmxr3W0Rg70ZI6wnrw4/texMpE8yCsCP+2vrxHGEP7CZUVk7g+qz/7TYf5yHpqkrMuhBD8jNknPcg1IW2bnUFC+cixDWAmAPPAtIEXm3PTAfOS+9sS6iTupA34AIClp5ERC9nQHdYfhSppfZDBOc5QsJfhfbzh5AqsElIRIP2rS8lGO8HGjVjOoVg/v6DoamWLiUCvx+1tk+utNFu39gUqUodbv5AOG94m9cZfIyxc08/tWelzTbKtaj+gDQUR9qa5RIac6h3ZMhzlfHJTcReyGP1PTEc7irWgcENoPiN4brtDfcdmnDic1D8CxSvRiXlP5HuAcWz0HuobISd6egGijQHBxJ24FtI2fT/mgjURKSieY0N9isMT6CO+iwMvhqKe6FoQNEDxeuguNsmeAUUj9jvR6HYCIoHkPR0PhRvgOIb9YvboDVn0kdZ1J6OkH8fQgbPIV21S1CDhM1kQ3tnKUJoY5ABenb6bsPa2qFmnkYS8UjZC85VXE2kKnSXXK9ZNGib66lKewelDZK9DwcM2CYQKrNFhGv6QamNYxHCmYuI2yrr27nWphO8TpK6AAWE+vxVXW5XIWntfkRk/pNQn3ncVCMdA1QK9Nk3VpOq5dq5OwLchAiKj/n5yrb2rNu2brFzT9w6AhETV4UWtu6/QNJ21SV+JRHH4qrCpxjqSdpFVG/1cbnq7yiDlX6bi1ciZNcAzkp93gFJZI68TkGw5YzHk+nZa0hpjtL14wnJNSPT/yLUkB+x/p5n7Vazta8g1KrZdvRZAs6usN9u15pBlGAvMDW4vXeWPTuB0DhMQbDjqnBH8plJ3KUytk77hjsKzceYNkT0GsiRI6s+nTgc3QJ3ee2w7JwzHqn7tkOE7zmgL72zH4FLPBzCvYuXIuKYvfdaZRLpZJiwmaNg9bXQHAXFh6FYDMU8w4PjDC9+EooTE/68p0Kgpsqz78jKmC8mJPEP2xq8BaUdOw64/y9BN1zs/rP/tbW1zUJZkH9v5z8BXo027BeLotgtPVsAbyrgu1+AzzQRhPvfGxDG/JQ14H+HIPbvB+nam5ES/cdDu1QgIN0CAcwGdn0dBDzvtE89gQAepJbaGQH+5vbsPOB1CEndhwjMB62N9yGJYB17v4GIyDrIA2oEQnjPpX79HbJnbZqurUKc/gcQZ/2QXd8cqYluRq7wb7c2VyJu+B2VKQIxClshzv7taJPfj1Qh2yIGYpQ960HJOyMJbRQRBf+IjbNpfb4bAfDG1t+RCMm2WVvz7b0lNo+boQ3u3lSr0UbcADEw+a+DKFtQ/fsIIiw9dr4jAoMHkBpnHcQM9Vr/1kXIdY6Nv9fmD7T5/o1gPJ5ASPiD1kfPPbcaIbXX2ndHEWuc/5wAjEaSygY2JxDI6DrEwLyJQLo3pjY8Qe8TyMNwtM3Hk2i9NrS+3I4KUj6CYBKUd+4tiMOv/o1Bqr+RxBqB4HPAxuZ99cKLc+z/5oTNx8eyGsHeqxAy60Tz6wGvE4B/Qmu+AsHlFJuXf7Zx+bdmE27x+9h4vL1ZCH4h1mS+HdtZOz3Ihrwugt1brV3Qnuyz+3sQsF6gff60fdf/dkaw6mvwRspwDYLbaSiDyLV2zcuVNNC6d9n10Qjeeu38n9HefQjFcm1AOVxjrI15a4b+FZfA4Kdg5IZILbG/3fg4WtzDkR50XcR9bF5poA1N9BvhIoricwBtbW3fQaEQuxVF0dPW1nYT0lKcY/fXQevxlqIoZtf063/t7xV/zsZr/hal390IGQ73twGIguS/K1Ck5tFI93QSgvzZKODjl+nZfjta/G2BgPmVCBjWQ8C2O9p8TQI4898b7P9MIohxW2SUX2L3dkbA74DcjxB5B9qcvrG2QxvOieUriVxjoxAgvwqpT1wy2yr1pQ8Fbz5r/Xglmrc97L7Humxu49sKIRqPVB+b2mu34+/s2kYIYfufEwFP+zPS+tSOluDv7L5vsuX2ncLGsG6au0VoA2+Olm4T6jchCGlW9xYIyWLv+l8HmsdNEIy9ysYyFSHE0Qi5vt2eb9r99RCCABGbGYhw7Fj5Zhtav4adu5rylUTw7hhEHFbbmL2PI+y3I9Z1EAHqREjo79Ecvg/Blhv4RyBmowtJff1ojscSpTJ2sLZfQazniMp59W8ZQ/fgpgQRnmXvzrJvjiYIyRjrz2jkFPA6Atb/gdA2vNrG9GZCahxj/doOqZZARMudQyDgZN3U7oo0Fg+i3pxgeFxyW2bf2B4xaxun8c1Fe3Vvwu7rCHdLBIPtCD63RnM43cb0CsR8+Rw4w/Iae7bN+vdGG7Or9LegTMxfl/rzKrQPdrY567Y567Gxr0PrfdE2VmMeuR7lTbCeDQhkiD4eGc/HIoPjgUPb2gCgra3tAyje8l1FUfh+3xL4eVtb22n522jO/6wE6i+t4tsrnX8P5cXbD3g0XX8doeK76PNQHJtE0nz0Q/FTKDa384NNlK0+d0EL8Ziyy6sfrrNeiDZeP3J5PZlQvRXAH2pUaU1EO11t8F9E4btBzMkAAWyBEPzhqQ9TEXH03G8F4qbXT20sJam60CZvIinBOf+CqGXkdidHDtlZ4PlgxRZrdkUar29mt4/59Yat7UWIa2xDG2phetazIGxbaX8qYuBcRXmntdeTvuXfWYJsP1mF+FNgYU2/ZxOeYJ8l1LIjiTIgZxP2AF9zrx+V4aOJuPyzUNDocURG7w6DiSYV9SPKOuBuxJOJ8umfR+EDF1Ff2bYz9cEJlBeqdMeKLa2vU5G6sMfWu64u0yBWbsPW5gPI1ppVdj5+d454kPCQXM7QzOcH2zubpPcnIsQ9yub8WkIF6HM5mL6bbS5DvAPtO1va/RF27kHEjsg9Zs/zVjbS/Zn2f2Rq75v2/ACw0q4tJZiMH1rfXaXWhaoVfIgI6vUEt/6tYxIMNJEgMwhsXxlLnoufIBuuV+SdRtglq7bt4Zy8it+bA9lmUNyVcN4BUPygBhfeA8UoZJcqrA37fSFiIpaQKhtb328BDvhL0YrSt18CBGobA6S3I47Ba/w8b4PKBKpPi1KssvOzkZ2pgOIhI1YPQNFEbuZXQHFY6wVuGGDkMteDCHmdRejKPb1+HYHzVDDu7ur3ziaCGg8m7Cg7kDyZ7P9NSEXjyLDXnisQkrjFro9Ght8mQrgT0uYaILyDjknzPJoIzm0iwtFOIJOCoXWW9iaIs9uX3M0368m9/ENdUtA2wtj+Myyux+6NJTZnN5Go0zMfPGhjfhhxsNnm4E4tU20sj9R8+2h77iE738jW2dvxQNw6mPAifk2kKWlirvap/W2JVD5N4MQWMD8uPfMoQ2OrPNWRE80J1COkhvXZ4bQfqfhGUE4PtF3NGqxEBNLtWdntero9swopIjwV1xxEfC4nEPGPEYH3UipnEp6gQ5Le2vfHEOmB8nhcsVEQORyfwlJKpfe/jnn7IWmoQbne2k8pewdelsbl8L4a2Yu8IsAphK3vaLu2GuVsdGargYK6H7E22olKyO45uJKIaTyfsHUWaP8uJlTBdXC2CivoiRjBXyPCmJ2GFjF8UHrxbRjsfgECdRkUc+3301CsC8X0RKCmQs8cWU2mYM4slXX4KFJ9OmO7PhVHqr9aAmW/j7XFmUvknnrei69KoN4PxQZQjIFiJ8pGv5vs2vrImeKjULym9QJPQoThyXTNAWQ1oTbz+BSPb9kHIbIzEZJxjtNrABWE63a17k11oz6JNvqlCRjnI+eRDKx72OH6a2/jXqL+ziDwTM3c99i3jiUQ82qkiujEasggyc7VEt2Upa4m4uI9Tuo2RIDdbfchxH1mt3f3Suwn0tFMItIjdSDE4imICqLE/fFYqQ+ELHuJSP9JiFBnp4/nEFPxBeQ9WRCBzFlKaTI0U/gM5D7+PSJOzIsOPt0Cnsekdn9Sc38EUfqlfZh94RlDPHHqDOQe/1vKqZB6KAeE+novIZKsHo8cjy4j1GXueXo5kXpnFdprbYSaymOpvBji1Mo4HYam29p53auCspPEKIT4n7Vvr0ISjuejvI+wc/l69BMwfTtWasX6/DRyA/e5nkY430ytmU/3ZPTqArPSfK1C0s6bkabDnYUayObiMDjLvj2eoXGS+fD3izS+pcSeuBKpaydam54xwgl/A+GJWdQngfU6a63SthUbIceH4QjUUcjbeTQUW0Px2/QcyIvvnWHrzJ58nWleP4uYiA6DnXP/qgjUn3xU4qBezDEIxeX1C+sAfD/lTBBNtEH3REgi56HLCG2WbRQ3grdj6WcI9V0DUwMi46ojSpd4FqCNegXivvx6q7gLP/rs+zci9ZH3y4MsH69sWC8dktWCro50IrmIstpuNpIAd0ei/3PIAO9ztASYm9r7COXgyXsR13WIzw3BSTaQmmkF4mbfke7tmdrcwb7lHPJ77N2fEFyuE8351Ac196G8eJcQklne/AsxD77KnO1PeF0OUXnYM5Nszt3j8ykiqHk7IsbK5/j8mjY+m/o9kZr8aAYnz6Z1HkScfQOppP5IPQLtRcTkWSL11P1I9dYDbGzt70bF5ZtQUy9CthPP83hMan8lIeVviOLK3MGgEzEQOyViPUDA2CCSansNNj5u48jeakuIFFQ+hzumPl5CBa4TwW+m9zqRk8idDCWM1TlzxqSwtbuLcE+flvrRa8cZRJLmBtC0PrShoPOB1N4TlOMNvR+DyEP4c4gQZ/f9AsH2eIb29fnjSl5cfGjl+Fsc1P+QQP2PMknsWL+oru7IhMfrszgn5/cn2DW3P12KOKNqrEOfAWBGgpcT0tl84OMGwOcQ3OUYUu43u/9WhtpCrkQc2HlIJfhw6kNVQnPCmRHCeOvP9danulLT19UgyKcQMWwniLmPqaqSaUOqEq89leM57rIxFEhKaCKHS0cU761pq4m45VMRZ51VRY30/z4UvLoN8pHpRQipbu29PEonQhq15QmsPVfLutrrC3bvQlvvt1mbbyKKDv7O3rsbIZ1eIlD4IISsTyDsbt6vrVr0Yw5yF3e7V1Vl5PPgBTH3RXaq71COu8sIea7BwfeRLWxJzXd/RMD19wmX+AIxPU9X+tCDJMY9ato6nMiiXqB9NsXg4JzKs6OIuKx8LEEwfxUifvelex3UVy72+W1H+/haJGWfXAMfM4lM/btYe+MICdNV9W0oJMClrduIchpNxDwsZeiebEdSsY/tDiI42OfQGedcwyurZBczdF6KnaDo4kWnOfLjb5kk/heI1J+Ui69Fwljn8n2j5HseyLfMftdFcbvqZBySHJrIVfTDwGkMzauVAfdCJFW8kVBDrbY2t6Oc8sVVD46APp428R6k9El2bUOCY38aZZ5w4G6n7HCQA2yrCMz1648ghOP9mI+QzFWEWuqPNchoFEqC2pW+7yq9G4h8h369iRD2AUjauQ8Reu+vx3fdjzb4E9bWcQjJn2vtPIu8GQcR8vUN7pvfa2752N3p40mSWjKNYyoizL0oFdNV9p6rkD5gz/WhGJdRBAJ5IDEiM+z3KYQ6y4OrJyApZw71gdsjCaLg6bbyOLLq8nFbs2UILpdb34+2+27vuhoRpUcp2zx6kGR6O2IIPpTWPyP+k20tBihni1iMCFHdXM6052cRDiPddm0FQtZPMdT5pkps8rVuQgXWgexVvncmEo4VQ9IS2fVLCccNd+suiBRfu1AO7P0k8ug8Ckl7VXWrz9EUBM97okDeQSSleqaNxZV35hIM3TNEot5sE/dxVW15z+O0L0PR/eI1TS/5hLFrvQN/ApF6wWzmwxAnBwonMtkt9Z8pR4vPQRLL+gTnnzdOL0JuCxAXNJKyKH45oYaYjJDzcwTCyZs+67FPRcSmYYDq9pcm8ng8nZCoSsgAIZ86TvIZzAMOOSi4Q8Z4xJUXyJ38IEQgXN3oXm7ZqF1texUiXs8SOcec+zwfIZ1NEcGcRv16OIKcjTjb0xCSfZiKCg5xsHOs3cft2lZ2zdvM0l5BuTx3G0Ie51GWwmYjp50PIPVTE6mwjrD1WR/FuTiR6UdS9Tykpm1HcPR1G/90hHTvIrwTPTHxGMRkNJGK8zgbzyjk5XcdZUTWbeu+fZqHUQjG/Jm5lJmQB7D0RAjm+qjPprEA2VI/a//vRwxBK+N+A9kb9ye87L6NGAd3wliE4KGqOs7rnfNbtqdrVyNm5Wd27TeECu0sJIHuY2u1kKH9GyClOrIx5lyHnydUmLOQx6nvvW2sXd+jdZqJVdbW5cT+OIZyDsJTbX36EI6YRjBref/sj2D3PMrE53Kk7qvuObch1uK1r0JvNzQbLyxN/S2b+Z+JSO1UwBWD0N8Lg3nSe6DRB4OXQ7FTa6NmA7l++33nouZVntvWALXKsfQl5OCutNUo77lAYc9tQxDAkWmz/C71IRM8/+2S3ZOIG/uvyvOfsbZGos2cXXoz5+Uc4qcRYj7NzudidhW04QtgdoUInJLGNNG+e0Aa/+4EYVmYfs+2jdQ5zGaqHp5N/HBEmN6GQjl+TbJ1pf45AVlOVLh1lUtWXTrx3bYOnmxcC4ng2IxIGojB2NfGcwdCXLfZ/BxG2XV4PMG1b0QZpp5BLv0jEEFwKeJW5Dp9Y3rWGZ/xBJMwJNWRfedIm/O56X1PdutE9G6irHqddDOAEP76iIHZH0kJV9M6WXEdUehACN/tgU3E8DnR2jrNz/nWn6zG+kTqk3vZ9aRrBxLG+wMJO4/PWYb9hn37IuSR6szJzyvPfQbtCSc02b2/ShwGbF58H29FSHVtCE6ync2P2ZRDMNapmb/51o/9KeOueTZPbjbosudarsXO0HUbLO2FZleFMDWUMaKngCuKl7Ba7+VLoOx4HTz3I5h2CfTcC4sugOJ0eOwFcu51EBJND+WqlZ5Hzat8LkEeb84VZTXHZCQN7EC4IztBmUdwvu5+3o4Qm+uVm+kbkxFxOdAA/XQUEOhqRddnV73QGumbvQhJn46Q6GLCUN0gpCW3Sx1eg6QcSV9k5x+w8ylIxeHIfyqwrz3zKcolFOYTnKdzmo6svksEnrYTJSK6KXuqOUGtMhgdBFc+jUAohfX9W/Y/ZzkfQIzBQloYgRGBup5UndeudxgsuME6S3kfRYRmDFKrZU/HyUgduDC900SMUBsiukdWxud5CudjaYmQ/aeJ1MHPl9uo6f8MRMw8HsjnvgNJsKdQVmMvszWcbvNeVXFnlbL3fSZlxsnLSdxPEGOd+qgAACAASURBVIjpNWufJcClSGXWhmyKfm8uIb2uJvI8em66qvQ8Akl53s9+hLxdVel78XQklUwn4KlaLLSw9xcQjMytSGNxL4LtHa1v1xOaiQaCjQ5Csst7dEaaC/d8/ANS659CMC4e69dETM9pxH56kCi5UaT5conuSFozfwNAsRVccgQU18P8O2HFJdDzM3i4+FtF3T9zhyOjgnPtTmSWt1iwgtCBNxA3209w3Q4ks2wTLalstibiLJ1oTKGMsH5ObNrfErn4PK4iA5J/d4KN5VzCRvFxIiaqsP6sR8RXZKeOqoNHv82H9/tMwtY2iZA2msjgProyp7en535LBLLena5fwVD7ms9Dr33vGJLjBEKwXYRk9c40lg9a/w9FMVmPU/ZiXIA47QIREC8b0kzj7KQ8N364QT7nMrzB1uooxKnuiBiA662vvyA845qUE9LeTFkq8X50I3VR0/rq/eghquqeThkOsn3h+PSNfewZf8+l1Tm2Jrsiqf1Y5D2WnXG8dHlW8eV5cc82R9IDiFB5bamrCC9QR4qXIGJxFSIAR9n9Xso20AJzM7fnd0SqOGeS6vbjPISUHbmfiGxi3YgY+PodX4HTrYhckU6U3NlhLrFX5lF2VMj/fU6c8TjT+u178ynrh5fL6UN7czRRpDATOWcWjyE0LC7V32HjGUzffM7O77a1zG2tRirhAyjX+coepV7qZeUw85vxoo/Z/2+ztnH4Xy2BMgB1FU4VKbVS63lQpxOoiQTnVyBnhY0o248GiBQ0Wdx3QLsD2WvuSN+dZe172379WpQC6So7d07MubdB4B8rxNftON6fGQhJFcAb7NmT7Lmbra+OFNyjyt8dRDrzi5DKbLm1nZHjiQb4nrndM2lXN6FLOO7E4FnHT0acfJdvALTp3dPqQjv3kiRH2jM/tzZcbfI9+9Y9Nq5sB5mPkn22IYN1L2IccmaCZcC7ka3uSGv/ptSGx+/kGk6+3q4+8jW6GREdJ04/sHfusvPnKJdRbyIJ72LKqr8BhFTdseFhIh6naXNzMSHpFkT9p2wD8XgYd37wdbmMyJj/IRTP05/enYm8Qgsin+KJlKXQLuvHDBu7S5JvsT590M49RuoLRFn659fTnvmY9esCJJm4t2IHId1XY3tcinKG4jf2nJcr8QTKDevjqchOM5OhCN7n/Ac2H8/afHzS2jrY7p9FqOdd4va+bIXg7FME8XOPToebw2ysj6VvNol96ZJSL5Kgvkh4+/Wl5xsIP30r9cUZnpGU47h2IAjvCgQnrWyFGUf6nBdI6ltvbePyvzoChWw3mftck6OBuM4CiftbU84a7JH5DxCR5k4Y7kmAXOVMt08IdRBxm9WI7/nA3fbcfpSlt62RreWaYfreNGDaMM3BPFKpbsRVdxjQeuDk/NTfVYSK4tmaPjqhrCLrQSRR+GZpoM38HUKfP8LG79+6DhG4XuC9CJEMAN32vEuWU1P/PQvChXa+G+VSIM/Y2GYStpXnELft6+iS7SCVTA3pO+OtjVX23tft+kREGN6D1KzfJ2BjDoEcXELM81a1IdbBZd1zRWrTkdQzhAagB3HjB6D8hvOQw4CrtwaJSq8ntBjvruk7SwgJ7gaC8fDMFJMJgtrA1FHWzmTKacjarN99CQ78vdsJVfWlBPFaAnw2tTEaEaAeAnY6CIePbPcaZKgdzKWHp5A3XSehAvOSI91EguAZpEwpdv2mdP4OWmdr8PX7HXKi+hih9sxS1lLKTjo+bo+/8/Lxh1aeWYlUwk2CQXZJdjLCWVUHpTlIveyOU2+jvu/D4cQ5axuf/zUSqNfzwkZ3R6Z5wRtEyqFb0jMFQrherLDPgPU9le90EjE4exMIx7kvl7DGIyN/VcftKrLTDLDm2PWsLpuBNptzaRmJnYPcp9sQQVhdmZf1CKnS1U+TkJTzS7vuyNn75in/66LXXWr0OWgH9h9mXTYjEJ6rvQpEGI+099sIxPjKyvvuBu0F9yYhbnI8oXLxFEnvQITW++ptLrbvXtWij++2tkdTdhSZRPJss754hhCHk4VEAOpw8OdceM4uMNyzg8jN3N2tBwnpe7vUpzkIflcTBRT3tr4M4YSJ2kVzCQKYYe1OovBhF/LIaxLxQe5w4SUv3lVp/6N2/9pEoN6d1t3fmwf0DgM3bUjFlQl8tqVluPRM99mJYSmC8Qayaw0QRNH7UgohIBJAvxMR/P2Iasp5z3ah/eh7pklIf7cgycadT3x/e3/7kTT7JLGffQ18fzhRuZyQbLyNNltjP7+OcKJxpma8fcc9Ewukkvf+vxCRaqfGYealeKz1DrxIInXlC0y8e8F0EFHbnQaYTlCOIlRNBeLId0TIokFwQg2komrYBpmEEIPnYMuL/Xz8B6ET93pH1T72EAhySyxY0QC/ke4fRMSauA55hn37P4kUQdfXfMMB3vXU3Tbud1QBE6n+/B1PDZTH5xvAY0nuQnaVj2ESC1Ev6/ZKH26x+XEk4G7h2yKVTfY4m08F4RIE3z0RvfS4F4PM0omn1jnC3h2BDNP7oPQ/7kxyE1LLOBLL6h0/zyrAOlfjAhnh32TfOoMKV0qqX4Ukw3uoX6dsR6zWmcrIph1J+tchiWc+kljeg7KXOPzdYO/NTPMzQKSpaiDJ3FPbdJNSkFkbr7O+OEGYb2vugeRu0zrS7t+V+lwgQjcDc05BYQafR0RwAmU39r7Ku/MoSzRfqvRtJCKIbitcyvD56vJeudW+504svrc6kGo0r8c3ELxchTKN9xCJZqsENAecf9v6OQpJR1WC8QgiUF1Err670/051tY5SEWZVbG7Evv9QaRV8u/eSMDyN4aZj35qHKVeqsda78CLJFCfpiwl1S3A1en3zYSaYCba4FlN6EG5X2KoPvtc+6Y7LzhAeoDeFAMQzyrQhTbgSZQRWS+R/HQaYeR3+8P1hCTwHFKtedbzzYjMBz9AYn8dF98gpCYfg9u1xtocfLNmPs+zd6+gjJCzK+uhyEb3aYSI76WsRuynrPs/CXGos1Mbvma3Eyq52chbbX1kI2lSX5DwgNTORGT/yPP7sLVVV8XUkVAHYdNxJH82kRC3QQROn8bQ4OymreE/WJ+eBR5MffwIKSs8IVW6ba0Nwdqldr4n5UBiPyYjrrqbSDTrNqzzETw/TDBAnieybh90IXhyibPf2j2Yslv662vm/HR7diSyx5xDMHizkENLNRPJOGQjuZyAwZUEwu8mAoFPQXF3o5FK0WHnstReN5GR/pCaPl4HLE7n30tr9Thl+Ksi6IlIjdppc+7r5IyQMy+dSDrzAFrPseeMztMMVfEehuzOg7ZGJyIYd5V0VgV2A2+zb89J7TRtnQ8imA3HJQ2ENzy2bTGhou0aZtzZWeJl4yix1jvwIgmU2zEmt1iEDChF2iwNpEOele61I5WKA9jtSKKZltpxx4AmcojI7qy/R0XdjiLKhlf7clTNGEbbZnK9uX9/MeI0n0WEpsppFkQMTUEQCEe87sgwQGQe/75981TbIKNSP64nEGAn2nQzEYH08U+xdobkiEtj+WUaw1LKmcer0kc3InKbocwZH0LqkFMR0uxHyHimzXVdsHVGyKts3S62Nr5BeFm69DkeIcNN7HoOdn2MSLmTEVje1OdbW9lusb21tZedr2fPe6qcQygn2jw5z789Pxsh8Osojy+rfB5CdpVfpbaOsXUeYd9xVZLD3/UIJn+OEP44ysHbdQxOH5KupiH7nttlf4Skz31Q+MN21t+6dfV9sZpgWPoQg/OKFvBzGOHJ52M/xN67jyAKTeA/a+Z/b0T8r2GoROQqw2rWmMcoe/y6ZuBOe+YKpIr3Mfp8TUbEvS5z/80MndN2JM0eaOs9Pa2rl5kZSM/6vYuRU5WnROqizKQVCDfsm/p3VXq/FV707wzyMlHvvRwJ1Chb1BMrm6S6MFl62Z9AOuPTYvrG+g1Czj2EDjtzMh6wOgal969+qwNJFUcjJJgDd5dTKUlt43gzUlG0ynbRTnCa+1q7rn6bRSBSLyfRhgJn/V6WMFcA/27vXoKQjAcgL0UIIXvGfYZybM8y2yTVkhMjEBfZQEZ851p/ZL+rjiV1h6s0lxD1rCYTZSc6UZaCRYir/2l6t5fWNpjpNhdfQCqVQfuOV2t9gLLUN47I8uDt35HafKfdu49QpV0PLErPdAEH2e/LgMn2e6x941t2vr6NZxGRXHYEEcPlSHMJUe7BYek26/dyIl3TbUjd2oOqntbtm3dShlvPwH83cgDw9Dq32nr3IUZpBVGy/IXWshOpMY+zvpxHeECeXNOndxHEyZnBBvLMLBBSH4n2gK/LZCTNP4skDLcZ+th8P90GvDXhjBUEMXgC7Z07rK0DCM1AK/tNJ9onm9aMY3/KTNPtSIV9CtoHVY/aAkncn7c5v7Hme7cSHprXUsYThxNM9CKb7xdaH+/DYdQEvr+Uj7XegRc8YOMCjirgoqUw/gpY+VN48i1hUK5NoGjHnYRBtUlkJ3dAPAJt0KzSmWbA2EPUFXJi5v8nEETsQcLonAv8fbsC4HMpOyp4W35+nG0Uv+fR5Ruiapdu3xkkMqOPynOFuK5B+70bUlvVEcEmlTQniAAXCAG6nvssymlfnkTcZl3G9Ubld4Ny3kOXOk8ggnN7CWmvqDx7HypVMAoREJfmfL37gfe3QMhjEcK7DUmrExmai/ApRBQfpxwsPEgQy2uxmDHEVHQjBOeBuv3AsXb/WeDitA7/bb9vBebZ700QspxNlJTYmogBczd/h4mnbLy32zsZdlYjJHuYrck0WnDGlIODF9o4ViCvxfWRWis7n/hceYYI3y/TgC8T2dSzlFEXH1SFj2VIOn6q0p+rDRaew5I214zhfIbCXGHr/FOUtWWsXTu58u6b7fqk9N6Zto6e3igXDDwBwY7PRdWzcBkRauHj+Fb67XaofyRsSq6q9Tlbldp3x5PtKMfuzUKqfVc/O9N4n/330IhqXsB8rLJvn/MW+PF3oH8QLi7gugIuMtz6kg3eXesdGIYw7Vyo1EZPAd0FFH50QdEHjSuhsYsAu5qqKB+/IrJv91cWuEDI6WdEgGSjsuAZsd2agP5hAziPhelgqHTlQJUzJyy0a99CUoHrpucTqZG2QkG8WRrrQBke8rW/r2zEHqyGkp2PQNJmlTPMHKd7TuVN6MhlhW2whem9Z9L/nxH1chyRnYdUG102xn5bn/+wdn9mfdueKFa3iNjIrrv3oMjcn7mENPQHe/7MNN4t0YZ+mOAauxCC/xpCxgP2blbz+Hf6CMKxD2Gj+h6SzN6AiPNziHB+39/BahfZu11IHbST9XF3W1OvgjvS3r/ExvY0ET/mEkQV9mYguJ2DnCK+i5ijjPDGIQ7bA2dH2Rw3UJmSAiFitx/lcbtd0QnNVHvObUMbEbFtHoPm7+9l3zgaSTkdqa2HkDSxnAgQdljytFhZxeXj7Sdc0avqtox8/4Oyp14Dk17Tta+lvo5HzGMmqLNR3s2P2HXX1LgzyD7I7uM2JFepZweG6t53B6BxNvZfpv5kpjpL7bn21JcYWvn4MkKz42aHSyrfLSrtdb0LOh+Eab3Q7GZInr5uw7FXFi1U+X8jUEOJ0xolhh1EGXy/LmCoswE54EwgjLGFAV4vEbfzWLrnRGwQqYieIGo+NZGOfhytK7L6ZtsKccdN5M76HsqeSxfbNw4j9P73AqsMiD9OeK09WtkMjmBmI5XlPYTY70Snys3m82ftnclE4J/fX4a4x1mVTf7+9L4HLHYhaWtXe+Z3hES0C5JEC8IW5kGc5yUCemH6trv3Pm8rQpKlb0hfp2qGgGb6vQJJLYcQKWH2sLY2JDy4PBNC5lhXIAL792ncxxM2Gk8kuxwRy9E2f1fZOq4mVSlGROUe5ATSbf0fQZQeX0k5S717aWVYdskiI1g36h9j4/tXm6cJlLNF+Nw4Yvf/noViENg7IfFBG8tAun9kWivPJO/EdJw9N5awJzUQom9DcO+Zz3OQawN4bQW++hGTmCV+97jz3ISZqcgw621mddqT1r/L0pxkyX8BkfFhGnLwGJnebSANxjX2XfeyLex8U0LDMCLNXyvTwwyk8pxJ2Md6UQD+ZQQcZq/GiYQzleMtv5dLd9Qdy4FlX4dVXaxRvaiXZALZv8xHKtV00/XdqVbFbFFa4y5UNbJucrug+a36zMa+wZtEHMIAZS8mR24rgVejeKtxdm9Vpa1scP0d4UXTRRTYcyD2Ym3/TXjf3GDAfBRBrB4nvAsLwp3V++5lK+qAvkCI8w7E2Q0QLtIdyM61PtqMS5AKwYF838paPJLez+qXa2wD5T6488hVJDsQoar4oZ3/ys6zc8be1tYj1pfViAseiZwJXNrsIEpUHG7tuLdS1u3njAu5RMdywi42QDABLoHuTjne61+Ro4VL44uRynVTRIiuJtRw26U59dCGd9t7H7G1Pdja38v6Nd6+Oc/6cwpC4v8PSV8+7meQ5PnDNN8rUG2pBpFD0uOXHrd3nIFqUvaw9N/thIPNJGvTmaRvEEzMfKLu0SnW1/loz7RTLlmxu7W3hZ17poSN0jNbprnPzIB7vd5JxGm5k0OD5Bhi7Xh9qmeQ6szndH2kovOxOuF6BDEHw7mgu5TkezHHPTUR3LsDzXLENG5E5ApsAAda/36S2uxGDkCDCI4WIPjxvuW9NJi+/xsbzzZpHvKzrgJ26XI4b+anj4D5nYYjv4uq7L4AkSpeakRqrRKoIccwxQmHI1AFFN3QbFGc0IE6q0s8AeqRRAxHViV48KQ//19YgCnyQJtFqMYadu03CMn8Z3rPHQyc2LgazIGqSRnA/HcHik86FmXC2AMR0AlpTvesec/H4dnON7I+zCNsKeulcZ6PENFu1sbP0CZdgewvVdXpKkQk3JnCDerHEmVBVmOFD9HmHqzAwjtSu/Oo94rK9ruVDC28dyPmHZXeOcPuXWrjOR4R/mpskSPyTNCeqHx/M2vPVTHzEPH6JwRLDYTcZ9qcPImQ/wBCRJ6F2qWRWwh15m0o7u4igmmajmCsmiPRmQmXqDyINK9JJ3JW+SZC3G7nODi1s5HNaS7y56rs7nT+3pq12Cx9/wGsEm+6XwC/sN+ziZLgI22M/h0PPp1OmVA5UX6ESNr6Y5u3VyN4dYkvq6+9qOejCIY9s4mvbVYHDhBS30gEt4+jPXQQUtXeSLnqc93ecm/JXGxyBaHRaKZx7kvsUQ8oHkdIv69N4y1sXTNjkePtnEFzlaPjiFbezAO7wqKsznsRBMqJ1Esi2/lLjUC1LO/+QgRqEIorWnMTHiezmrIawHXsjrROJgzgN6TnxiMV4TjCUaDuW+6NVFBWAXoU+hFITTQfGJ/m5+fpWVedZanDs5qPrMzrs9RLV/cj9+B2pL6oOlO4Dr7X1mYBstN8LrU1SHhWfQlJDafZ+LP9zvvQQIjpa9buSLvmqY7WJzJ53Ic2r8eIuWfcdsgulzMfZEahB3hmGDjbGyG/lUSG6TuRvaTT5sXXrmrP6ELI82aEIN+P1HVbUy4kOZOo/9Rtc+fztNLO5xOSnc/PXBRo7g4ZXhhvfeQEs5/BwXh7Nqv5Mqx5vaapNl7PcjHH3lkAbD3MHHVT5rwz/MxAhP319uxY6+8CxCTNtOfPoJzqaAlSaRdIk+Cefz2EdOdreT9iaDYlMnbkmKomIj4LCW9LJ+Q7Vcayld2fgvaWI+xsc22Q0oPZe1tbX7L7/nY1cJFTNC1BzOxHUE5LJ07eZ4ePTHyLSj9mIAbiNiT1zUj3P4fg7ZVII+KSVVelzZV2fSX1zkoF0H49dGe13nAEamDotZdMGfi/JIE6EiG3doTs10VqkXlGnDZ+CHrfDsWroPh3KD4BxbEVAnUqFBtB8Toozk2T2gvFN6BYp37BfBMuIjxvmoRb9hJau0X7Rl6MCNQ1tvFWIWTqQNlEnOqG6b1LkLrGJa0nCV31hsgutZBydVLPPuGGbc9MvrXN5QjEba9Im8nVVo8iJOK2GjfS1gXALkREyTdWrkflCHs+NaXIre89CPk+XnnPucsriZIjJ9kYF5BKgqOM5v1EfjfXzTftnS6kJvS6Xd7PoxiaESNX8vX1PSzd/5K9vxrZx3yOe5Aa6ovIMWUikZfRvzcXEY7fI+LqDI2vgSNkDwr377sXo3uILSHith5E8Ji/Mw+5fv8SqQ43Sus4YPPlxGXbNLacDWQxkqaq8+P1tTKx+yxifqYSnpvZucRV3zlD/ZftXgfB0DRtXKsZaj+7l/BcrKtH9T7KZVeq6toJCA5vq7w3Aqkin059uIlwdy8IguV7/YD0vufVO5gIEG4gJmowrfHjiKB5mrPtCDvrUqTWfszOB6ztTxHqvH5CVboUSdj30joTebbH+t7MqusnibRide8XG0PvM1D8i+HSvaA4NBGomfbc2VBsAcXudn0CFP8ExfpQbA/Nd8KH03x9gSglMpPItv9GtDe8YOcfXq4EaiKy77wGcVVfIRGoDjh6C2j+DIp+KK6AYmSFQI2A4ni7fwMU60Gxwu4fDsW/QvHVFouWFnoRgTAaSB//fSTqu11kKYHkH7fFaaCI+pJUgrhtb//LREDefhXk6W6rrtueQGyqMQg5NdKiZ/3zfQQn76l+zkC2Ec9KcQ6BoDxm5yBEtBwR34TsBtsTdjZHSk1EHJ1Q/oFhAvpQ4HMDSRwNFFtzNmUE5f0fQFzzjojgTCZUne4ptxhxqP2E08WXCensrDQXrl49EwWQun5/FfLiG2Hr4na/0+xbTxDOKE5Epg4zxg0RofglIhzz0vhcAn2hGJTBynNeqO9WRIQ/wDDZpSvvuqF+us3TVwh7yCcZmtnkVwiWPLh2BWFY/yMBO5709BA7P9va6CSkBM/bdwKSXDwRakaqzqB8lyjPcS+C19cMM0avrVUQZeHddd3bPQulYPo04b3Yh6SNjxKxZe6McSXh2r0FEU+0EqkCtyds0R7cPpJgCn6PbEGeLeISRMh9LX5I2XvQnbBuIuxRBbCj3d+KkPYm2Lq+w+am0+bNmb2LbC1OIjwiC4IRrap5S8dxsGoXKL6JGPfxRqiqBOqzUHRC0Q3FPCheY3i1AcXN0LuuvrMRwjMdwJttLJti+SKRjf1YVIhxXawQ6suRQH0mnbtk8G6MQP0Bbnk9FE2bxAKKf64QqHUpi6MbGdVvQvFKKKZBccHwyKKRjrpr1eu++dbknSpHU32mDpHle82aa3XtNod5r9nimeGMqQX1/avrf6s2W/Wp1VHXvzVdj1bfaNXH6v0qUq3ry5oeVRtiq7H+T76R28n9rdpGXmi96+Dlxa7FC4211fMvdqytYHJN91dW1a3pGOr2Xau+tVrPNelnq/Gt6XoUw3yrAIqfw8AIRHwcV36KoQRqerp/EhSfSecFFO+QpPZ5RKBWISeRar7MCxHzsPmfg3asw1/ub1H63Q28qnLztZsha6f/bVFpYEPgFen8lYS4043Y868O34fpiKOaZecFkhoOQBnDvebMAJIMQBzPp4lkrQ0k0u5kxwn2XJtd77XzD6dndkIphPxvIpLEGkjtshNhp9oJuRu700YbUo2dgQVjIk7+CHt2lbV5ov3371+KuPMr7Zk2+8ZFyANxQerPPOuLqw5A9ridhjn2sXlahDj4PZG6ZAHlZQRJO+fYdydZH9dBy/cIUeNoOeIy97JvfJXYeL9M41pov6cgRme+ffNZJDXuhDzTvH+T7fk/2v8VNrcgF+7hxrkrcqG+AHHunn7K907B8H+F9a2NSKtzN3KqORCpG4f7vktsTWKub7O27yHUzR+z549H6wya699an9uQ5uIhu3cOgbD3tHe/YOeeeHc64WDk6/UwUqXeYO/nv3WQJ99HkUoVIt/lcGPcCUnBIE7d7S7tqe3JKCD3OntmHRvX75BTzFOExDWAtA0Fgq+dkDre98pDdr/LzhtIbbWTtQVRx+0GO59Nec98qdL/39r1xWje/e8DBBy588wqJMm9m8hw4l6SIHPIngg+Fqa2HNZWMMzfIDTHIqrif1vWPJfx62y0OTZIx2Q5Pm1aFEUX0iR8BVjY1tZ2Q1tb27b26rcRbE1sa2ub1NbWduBwfXvRf39BCWqvdP49JEK/G5OgLl0DCarqJLElFLchkXQ9JKa2kKAccB9H4rWrsXzzZ45kkLJR0m0K85FYfzKRHXorQqXQQKq8LVI7hyAxfrpdW4kIkudSO9Hez9VwD039uxcR1FVEAOk2RDVSV794poyZSEd9ber/QoTcX1dZk8fRRsqxJV6t04/bqDhm2Lsj0GadhdRBVaN0AxGP1QiZ3E3YKTyIdAukWs0qQa/X1IN0/QsRo3Ab5fiQG4AtK33amYgbcdXVBYQx3zOPLEUbz9dotd0fy/DqvMXW/j2IyDSsT9njqln53SSy03v6oLuQqnF5etbVfrcgtc4+wLrWryYRX7MFAY9eRNALBzYJx4xrSe7g9tyuhJuyj/1YzL6DmLAGgtGsQlqKYNIz17+VyOpxQVrz2Qy1465CnqGdpCD3Sr82Rfsj55bM359g7/+28t7bkFotB/n+xM7dEcYdSLqt/QsJ+NvO5uByYn8uJ/JzzrJ+tCMGdhtry5nVwmBhY0ScfO2zV2yBmIpN7H8Hil073dquBtBX6935d9xetZwoldNSgvol9FYlqE/XSFBZG/UjKL5Ywa+F1WmrzPt6SGV+T8293dCeeWPdWv9JtOOlQqA64OjNofkLm7irGWqDakWgCii+DsXHoDikftFcd5t12k2EzPsRwnCuy11Pc+qX5XbuQDZQad8BrZ3Q8WfXaC9XvqsBcAcRrDqGcrJJR2xvt/sjkQH58cqcrk85M3KT2NjLiEzuQ7xxkCdSExEZJwgXImnE2/Mg4ZXIM6vN1utcyq7JOdj1ORT/0SCIQGcC3nnWp1Nt3PPRZtsQIapjibIn3o/sSNBfnYcaZDebIJKrEMIZYdfnEF5Q2Xutb1wCmgAAIABJREFUziHiF6gEw1hk33mAIDQTiMz0nlnAnSGW2/x7TI2v/WMIzhpEDrbdUHaKA5FEU3XQcLhypDWIZQdAksT6KFTACWUTEcAhTEWao5zxpIcysutFtrGvIMK3k7U9zs6vtXcfxhgEe281kjaatu45MDZnPlmMkJvbLzxPZgfl2mRHob3pWVfc9rNpZSwj0b7sRES2K30n78+bSOEMCMmuwgqK2rXDKu8UiFlxBudpW8M2VEcuM4hZ6pxCOM1UVXF99t35SDLNHsUXI6eVf7axryKYxxmVdtz5olbV9x0Y2BmKI6Dog+IeKMa8AIGaA8UmUNzM8wkQun8hKX9zW9MPGwysgxjL8TYv+2HqPUT0exjGi/RlS6AK2PhB6N0BitHIi++jUJywhgSqB4ojaenFlz1m+onKm1dRTtLZtM0yhbKrc/aYGoGQg3NMg0i98jDlyqCOUOoMyQ5YXq+pqDzzDEL0b0VS2ZbW77NTP9ZtAaAdRODkroRKyDdaGxFP8TRCwF4J9UiiTs29hDOJ98057CYRV9NEnP977f4c4H771kLMsyn1+5uEp95y4FU18PLW9F13Rshz+m0q+cMQEem3tdsQIde7KROiLN1kwnQ/JrGk9v6NKAznEvP+hBp4EiIGA4iTXooQ0J2EO/aV9q3zCU/R76G4Fi90t9qee2fl+2MJGMxqPp8DH1MPUntugbTcK+29HSrtbUPZeO/vepv9VBICp7VwqW0JsE/lfoF5btl6zkAwtIuNsYEM6edZGysZCrNzbd08V94B9uwbkTODS5tPpO+OtveWE1nkd6F+Tyyn7NDwBCJiHlLyDhQsnp2n/H87od2oa9vVpP7/Qvt9HtpXvn4N4MTK3J2e2plmc1d1U3e4cnjNfehhaH+KjWFwMjR3M1zayouv6l7+APL8GwvFa6FYV3jjDYjxc089T6nlyXh/gnBoJ5JED67C0EueQK3xUYmD2oWyK/lwxyAUl9cTJ1eRPJ7OfWE92WIDcS1vQeUfPIAuIwLXE/fZcYQBfp8B1hvtOc8+4SlivJhZA+Uq2xgZHh3oOglX8yohy8TN73dTRtjLkYphKpG7zLncf0LSj8cbjSAkw9Mrm2WaAWA75XRPcyhzwQWBgKdjXkrWxhGVsV9s5xk5bGHf6LA5yXno3oOQh2/8x6wvuxEqtYJQ63Qg24NnmPA8cy6FthPuy64+8US/x1XWeDPEMN1ChAU8guwybcim5fae/dGG7UMcZNP+F0j67iFqKrln3CkoPKDf+vR+xMkfS0hjK23O3owYFF/7fiR59BNqK5cW5qffztU7AzIdIX6Hl5Xp/2IE603Co7OJkJKXfN/H+upZGqaQpDPgTfaep6Xyud0lPbO/zVkvQyWbebZ2DucriUwMf6wwhZ7o+UIiPmshkex2JrHP+pGE+kfKe8X3hyP76j6rMjAzCSl2KXKJX59y2MNu1r9b7XyJfXNrW4cetPfOsnc6KddTy8dViFBuTjCuy9Kzvn79hMRWh/OK22HpGqQ3anX834qDWtPjQDh4AXQPQHG+KHixYA0ntROKFpkkOhGHmqPoZyKuaBzBxRYGWDn78EFIbeU6aX82q51cnHdgWQ4UtrFGGzAdh+KhGgTX6okh97F+DNo3lhnAn2nPz0CqhufSNxwh9SAktIQW3BSBtHIfn0DE5IdoI7tNxRH5NLuWCcu7K+26OuNIIjr++0S6pI2IXHxfsvvbWN+ftDFujYiQ2wmaRFqcD9k7f0x981Ip1Rx6fgzY2v4chTS8x979mrX1QYK4TSLctfOYnkKxaiPsnV2JJLWnIWL1qI3xrUQWhM/btQ2srTEIMV+EkHQDIamsKnuIkHbXR67x1YJzfUhS8wrRUwmC8Mc019sjSWUR9Zx+B4H03EW8STATv0bw74jP1+Ma6/PrGJqJ/Y/W3tg01wXhWr2hzVnWUvg6rZ9ga6SN09M3ZSR8Bdo/x1C2l/ZTzgpSRda+T5+lDNsFkWrrF4Td8xu2ti7ZDyKHk1OJhMdNotLw+xAx90z1+9pzWS2bVckeRO1JdD157u+JfIHurOHvOEHyQP6ce+8GyvkBS8e7oLOLIYlh1/T4v5VJYo07AwevBx2jodgeiuvXcEK7oPnleuTshnI3mjvAZNvNkwZsNxDShxMAz0rgxOja1NdbELLIMTV5kwwkYHyIkBj6EaLYFSEqz9rdhfT4bmj9BOUN63WdXFpoYvFCqU/9dn8DpOI6BHk/VTfvIPXzlZFZE9lEbiRKtheEt+FFSNXpG+sJm6uLEDGdY98fsD68HW3S+4kS3B7n4bYd19EvJRwVMqfpG7YHqc0GEbEYg5Dk4YjYZzXScqRWc0LQZX3MJTZ8bHPTXG6EpKQm5txhfXbnj61QQGYDqWAvAKbZu72IKO2f7n/A+uvqsO0IZ4PzCSI/Ko05I5+piMnyjOk7IMSVA88XIZg6wPq6rY2zy551xiszM91EXJZnNXc7WdPOD05E9PlaVgTC38bG/Acb0zQiCH4ZUnf5XOZSK39ESN61G5kA1R1V2+8g2n/7IwLtabxmV/bFSWlOb0aSbE9q40a0H3eysb8hjX0QwdVBRCb9AsGoS4y+B3Lf3Ab1e7t/tPWljXL13IWUU6y5FHg4kqYynD6UfrsN+Xet5usomJ0dJV4Ecfq/lYvvRR8vIpt5FxQtiFNBSENORFxV5Wq4O4jgOAf+u4mEn3VFBacawDuxWYFsUKOIjOIF8nxyQM0qOn+vIEpXFMhA+q/22+0e9yNCtQfhAt7ASk/XEPhZRDT7CKR2qquXNUA5SrwNSTO+Ye+237eluSooJ9b0wwM6q4RkECGynvTNmxFX7UGT7sGYS2v4HD2OkOq/I2ZgMeEt5hv0By0YHXfFX2xr7LacvJZeRmRB5foyJOm4RLuLtelVcFcgo3Eb4uAvtPtPAlfb75mYvRAR50ft9+4Mrc57gLXTxVB46yDUmNkm6jC1kMh+fnOLuRhBZCcpKDtqnIDsJGcTZcXzui4gUkb1IIljjM1DRsru5JDtqX5elVA9iNivLUCIdxAxNh64ujFhNx1hfa1TaT2NBYcS2cjPqszBRulb3o/HkFr+mwQT50zSDkQhwW6CKczMgMOrB5J/O83HVfbdY1Kby5FWxAmOj6UTaVcK6//y1K47qLhaPqdR6ydsfC0J+2GwuhOKxgtLU/93s5n/iURqpwKuKGrqQfXBQA+yOe00jB4WIaH9CCPvjxGxysTmQ4iDdYOoEwj3knFkPYbY3NlY/QSyL2xrG6nqov5fhLqoDRlyj6bs8ls9Fth7u1ubbYiLd6B1QnVeBRF5my7lODK7DCHs+4kSDYWNze0/Htd0IqG29Aj7BqlUuj3/ZiJ11G8Jb6onaZ3GpY5QtiMO/g6iBMrD1v8HkapuFiIav0zvttO6QN/N1u9tKCd99awOjuzd0+8khpZKuCC1V1cF92e21i75tBNVc28AHrPf29pY3C28rjpvGyFFFIT35BmEYdwlJM/R5jam44msHse2mI9NKHP4H7d1uoYoqtewfi+x79yHVJudvLDE7Qh3JeGFeTNSrTdsPauJcA+0d3oJRq+J7EkFsWf2JBi/LuRNthzt46y2HyQQ9U8pq6c9G4Sr6JvAjxPj8UHC9bsOnzSQJ+DXKDtMjSEyuPh7T9m93RDhn5buOVG7ADG1/YSaORO9JXb+FURAsxR1IuGItQI5B7VS7xdAsTP0XQHNAegfph7UFcVLRK338iBQQag2KuDIAi7shxsugOJHMO21a4YAJxOedD+0zeCbzcuq+7NnIUSxhPJm9o1zeNoQTYRQ/4PY1BlI/di7BcLYEXl7VYnpN2yTuXuu99U31wSkMsjfepqoceTPuc3pdsSVuZfepvb9UUSG5aa1cQWS0j6U2nYJ7wstxtFG2DPcs20qrYs39tnm+jxCrGciJHkfgVzaKVcq9cM58exM8hhKq/MFFGj9esSEODJ3Yn0GSonUwKrwIrvKTFu7rBLMktwSpP6t2l42tLU5NDEHBfAPdn40sDLN01W2nk6QcnXeKoLJEoc7QFxHIOxfIQ56B8plOhwG9iFyxx2M4N6lT0fk1XE6jPUSTNpMpJa7HDEG30EZ7KthAPn9FYjweUBxAziqJfJRyIITp8Lm4W1EKELOOXdnes9hdBvkYTaTKEHhsOGB9W7/cib1DMKW43PRb/PXh/bCVypr0rS+btFiHL+qzIXvh9lof3yQsibjYoMJD/HIDO84RDQnpGsXpHcHESydnL71aM16VPtTbALzjoLBAi4s4Fr7f2Txt4q6/0ud1UJnRFy3GOele6cQqqdriJIMvmj3GnC6/SkTsN8jzvnkCgC5t87t1qfPEA4U0xNg+TEDcdsfRkZsj49pIO7qj4TofikilLMS4ptK6PfdYaM6ZtfnP0IguK+lNuoCHUcgZNJN6LZdfXFvOj/HnndO82Sk+ptJOR7K+3EFiilxFUm+7wbrLsqFDsfa3B5e6eNI5AXp7UwhCLQfXtKkOi+DNtezbZwzEZI6HalUPo4Iqkub7uLskmoV+R6e+jWOZOdADgKD6dwzY7eluesjCje2UUY6fnQiYu3u+oOI478USRb/ibjyAiGlSUgydvtmdfyrCcLvSPscyrFPH6MiiSKCuDj1dT+i8GC/zWlVdfcz699dqS8OTx0Ijq8laQasfc+O72rdPOc+FxfU4ILHCXtcm82RE6KLCcej6hz3Icbgamv7stRmv8HFtNTWN21dnKAvsLF+FGleqh6u9xDMwqNIkmojwjMmpmd7kSflSKIEjQd2r0R7pMlQpvRAIkxhFuH56vNchxedUX+eUX05HGu9Ay+SQJ3dYvL9cALkaVIcCN5DcESHUlYN3IwkitMJR4rC2vgW2uTOjXm2bH93wO5dQwSruci/gOBsMjFdijjGJYhIPIM2l6sCnJMbY88OEAjF44duQy7kH0v3qgS7HalWnMANCZ6zjZM54h6GFi3MLrHOaT6ACOkXkHv2o0TwarbnzSHiygas3yNtXl0660BEYlFN/95FEMFL0zp0J1gYSO24KmZnJJkcjbj/y20MK5HaKju+FJV23IvRN3R1Xl2i/mTq58nV/ltf/jHN8+/sWrWAnrffY2Nzm0YVubodY5at5zLE8PwUIdGPouKHXqrDU2U5UrrV1sO91rxK6wxgk0rfRxMekd7OBOSM4CmzBokwAkfSDjc3Ibj8NVJxHmbPPkZZM9BNOFMMEM5CmcD9vAXc/qvdvx/Z6FpVI3gYxejtbd/yPdsgXNwfJALL3eZ6MWKoLkIwPkDUu8pEtCBMBAXwEevjTtauMwb+znxEyD2UYS5Kiuxw4I5cn0fS4XPp3U8QnqzzkdbA7+UwmuHKwPcB/7G2cflfHYFCnmm9w0x8QRjns6TTROqkQcJjxonXzylLNJ9BQXtVgnIBIijrExs2c6yzUcXPNzKUk/WYlC+hDZ49eFz3PI5I/1/daE8jdcdbkSdRP2X9uktIGdE+RzgIOGHzDbUSbejrCQeRLCFWid0CpMocEsSZkFkTEf1sx3kEefG1I2LryCv3fT2GZq+4CMXXnGh9m0d4x3lanLzZCiQBLgAmt+jjNvbcFsibzoMf52Fpk1D+P4+Bujat4wqGzx6dnWDc6WJRujacjXQl5VLfXjbjh4jzfh2CFy+SOBkh/C9W4SCNdQwigrcTcJ4R5F2E3Sy7Pn8CSTY3pbVq2hrmNdvO2nkWEQqf1+yI4urLFcPs583Qvqyq0qrw5yU7brZvrk7PuBrxCqLCbn7/6WHg4F8QM5mrazvR7EIEcE76hs/hvYjhPce+05W+6er1Le1+lq6cUM1De7aJwjLy+L2awWnpneeIOLWZdm2JfefcNFYPwq6WPKk7ZtDCfvtSO9Z6B14EgdrsBSa9ejjAfD8t7ijK6UkmEcGXA2izOgHxUu5V184CcUBtRG2oq2uAogO40fp+qgH5hdbG3gix/9YBs8UYSiUvEIK6OZ2P5f+zd95hepVl/v9MQoghdAhIk4AuFhZdFFwLCLqu4lpQ0cW1oD+sLKioYEFhsa2Lig11RaRIkxZAWuiEDqEEAqElpPeEJJNMn3nf5/fH976573PmnYnosgks73Wda+ac85ynP3cvQmBOfXrASe/n/fbsq8hQwq2khgOarps6pDaPd2Dhl1L7rycso2Yj4NZAolU30V9tZfe0clfW6piNjDU2Q8YKGWDcYmO+H1HCbubrVHg/1TxCmwyxdy6ydXUfq0uJGHZXJET7xrQeMwmH7ZmETqu+XsPNZas96d/PJRDZFcQe7EBIyJHIXCRS2jn1/xIEkFt67SOTaW/zybSG7s/Vg7jZxciQwZG+6yNd7LQMs0xMdbsC32PzOVBfiERRDSRudk7kNgbHBTyQqqNpF+IEC0Lyw83fGkRgnY6kIT2EPteNiDy2pc9VPkMLqBrAjCXEu/W1dFPxYxBS+a61McGeH5zqXFH7dhU6cw3kc/giIkZm1gdeTyCdgjhVD5brzs3ZkvZuWxuHNwcN0ffhru51Dc+fdwgqHbyuIRZkqAVamDbranTofpveX46QzU9rB2YHIsJ2rnvANuomyL/GRUXO+mcZcxMZV4wi8hktJygrF9n9CFF3n2vR/37Esh9gZfeyuXgXAtZzEPDyKA5ZbOJ9WIUA/RUEFXYdETjzxwSQKYQc3OvoICjXhcjc2Ovpszk8HR28D1qdmxLI5lB7NtXu32H3h1k7Hqrm7daPRURuKO/DfCzDqc1Tp43Z+zmAZedFwGS81eeRHHwcDxEcZl2XmRHIPKoGKH+0tVhl709FCDSLClfY3GcxXjcC2FcSe3A54tYKAtSPpbnP/fC/bvL9GBFdvGnj+STifLazsW+JgJ3Pi6dJKMi4w1OjZ8OQDgQ8u0hJKtEeaxJhbTzg7k/s3q0qn0YCiMhoIkODeQQ3di8Sc7pRx6ra+mYnVycM9rY63SG6ifbQ1UgMmznbJsFpNpBYbz8b22IiOsb3rA8fIBz160hpP2Td6Lofl3QsoIoYtqAKSwoR6cM5o1VIerIZEWG9kf6uxiKypL70IbeTMVR9Nj+G1n4FIaqsc4D1sfjcZOffF3RQzyKS+rpNdFb+OiAYaqHOI4DGGelbB1CdtvG+QQRQzd+/hQBMmfLrQ0DfrXGuIzLqvpPBOoxs3j0eHXD3RPcoC+6xPg2JBOuxyxam/q+h6h3vwPIKQhTp4ZxuJHQ49dTZl6OD1UeI/T6NdAZuSTWZwebjDiBmW90frtV7GGGyfiXBRfUipNoFnGRlPdvrktR3B1SXUA3T4/L2h9LYW617fr6IcDw+DRlfzLI2dkaHv2Fr2E81uOgDBAD14K/F+u+J6ryvLg66w/66z5f3xzmaAeB3qY09CMV6DzJHfwUiqG5Eeq6zkOj6foIgcvF0q/G7iNEB98NpDRegfdiFEJ8jTxeTe/SP++z7A1Jb2yDdpiOULWrr/hMrO8v+1qULy9MarkJBSX9vdc9CCLadwUYzH7HyXYQxixsXLCB0vtkKdA2BJN2vK5/FGUgS4GXaqVpsDhDpX+pRPryeHqRH6rVvHIYssn2Q4cUse3cPskDNdXXb+p5OIK98zm5P8/YoVf1UvvKeaKb7BvDWdQ3Dn+8Iqg1REPfbwjnHctEQi+VXBxEyxI0oVqZFvDe970fUuPtFeNZX3yyXIbGLm6Q7FXMlkW7aN6+H7HeHPd84ExE7fz0C3E/7xli/Gog6u43BivW8EZciBPw2BMg9A6cDp2NT35cCe7aYUz807lB6HCHWm0corrsQItmFEE14X1wfNICAwH0IaHchatO51+U2Hz0Ed9Ffq2cNkXI8z2U3g7PYrkCI3h2G/aC7OGspKWp1bdyTETJ3Luu/iLxNpxL+Y5mTOZmIOuDA0qOGdBIGFi6W6UPAfFtkHn1Bqs/1hafYOrte5wb7zn2UfjFE/5cRSMCz0Pr+XmxjyDpNB6b1OeyzOa871nYigs71aasQkutBiPafrdyl9nxlqre3Vv/pyPzdEbbrV/qQ7mgAyy5MpPxoFYn/w1RjC55CNQJDD/JXOpjBjub1axnax67zuZ7B0TIWIfHjbALYu0HOTAYbtPyZsKRzw6fHCR9M3+ue+drhi2fwHkCIK7sGOFLy+88ytMTIn19hfx+0fqwGbl/X8Pt5j6BKKbwOjvgF3H8ZLHkMpp4J5UR4aIdq2KGhNuUl6X/fSJNqZTZBLPY9tbp60kHZAul2Hqp96yz5jkhUUGwjevSBgxD14+VdZ9RuByZv9sWIes5WOQPIuq0N6bLOpBq/KwehddHEqUhk6dTc1VRjobm4aHR6ls27HZHnIKAet6xBBNx9AgEFjwb/lzh4+qGag4DXyQhBfhKJWJYx2Ax6pLXnyMzFLw4MvN4f27OWVCNCUC46OofwBfM+PYaIhMcJxPNT+/YVVPfN5YSYa3Tqh9e9pb27gzC8uNLW1ilzdz1w8eEAAlRDIaifI276htQP15m6Lsd1RoMMFtI8HoHEpj9AiORaBptPD3f12DpNRUDekbrrZVbW2j2OIDh83O9M7925eUV6Vk+/vh2he/IxZpFtL5JC/BIjypC1Y0llf4+IxOxc7+JUdyDO0pQbCMvMtxOwZozNYTbA8msS0ueOTucmv+9H4t632hpkMfNN1gf3oXKDkeHWohOY8xKYeBSUe+De++DJi6D9YTixrMc+T899BAV7F7i4Ab11j+hOKH3Qfy2s2CsAeStA6BRkIairTkSVzEybow8BiQMJ8U2xQ+ebcA06yF22wfKhPowId3Q1Cqp6IUFptpJ9P4mQ3iiCO2ojFPT/bQeqlc9QG+Le6k6uTQTo32Vl3kpET/8R4ogKEimNsXlwgOkcyXwiosBTDBaBuuLdKbxOxIHebf05AgE/d5yur8saRE1fifR7b7U5mETrxGiTrP+T7X5fqqF6pqc+DkJQVvdHamX6rH+/RuGVCgKC77X3ryDSQLiY71GE0DxUz51Wzh1s5yBOclH6bsDKfNb+3xYh4z4kRjsbUesZ2M5DHHIdUTvALQiIZWfQOwm9ixNBP2gxl8uQXmYs4vBORIB4Tot1diB5F9rPHqLnYps7577q+8Hn2JPtNW1OctDjJuIU3mHr49adH0GEg7/fFREAn6SKmPNe/H6Ls3GG1XEaAvwFIeKxhDhtUyQZyRHWc73LkKTlDKRjc2Lm9TYHXvZ+BDdcuuLwxjNGe7njUdboqVQ5sbPQ2fQ94AjUA+QOhZy63wjLp8Aj3VC6EnwsUJoRNeLiYvq99f1a5x14BsjpL4rP14BmF08Hj637ujjQdUTiylo/UA6oBhDw+Q/CmbekTdJAbP9L7ZuD7FDtT+t8N44MpyFR5NcJq6WGHbTDETfmHIn3xzfkCenAebiiSYS1VxuBYFcgA4Isksi6uuUMv9H9myze8riGHuC2AdzdAuBl/w+fW5f7O5eUgV0TIYUzETCvc5HtNj8nokP/U3v/Oyv7USvThUz9m0hckkVMLk66hgA+3sZSarmTbByLEDBbjgDkEQSiaQL/bOW6kBn+KwidwROICz8PeNTKZY70UUQwzET762HguhZA9e3E3nMdwiyEyHM4q0L4m/Uirm+ASGPuot4mAvgft/m7nWriuz7EPdxORPmo67j+RJyf2cTervgD1sbhBNkUhMi6qYYVc2RVJ6zy/yuIJJp+tjsJ8eo3ifN+emp/B7TverBwU/bc9VXO6bvRhhOmBxMERbE1/h7SYU5nsD7W52gqEtm3IWlAByL8riLgS97bExGcccLPx531T96HB6iuRaXtL8NTPdA/MAx8tGu9jLu3zhAUAsz7/w8gp7VN/NNXB8MGkZ1H5J3xZ6ch4FWnch62jeNx7j6LgKnL0bMYYIAq8ltGUInXYg6RhAy5BynHu5G4qQ1Rh1+vbeQG0kfshQDMN5CYw40J3DS8fljy/RKEcDy6dSsx6FUMTgToYjs3sPBxNbEgnbXy2yArv0zN9yIE8//svouqr9oAg02Rt7XnlyOAuZBqgM7sLnArsL19d5Wt4zHpfR7jbATgB2xsVwyxZ39ERBZxcek5yAjmPlvnTa3OcYSJvRvRnIAU5x2I6u9HiPgVSDTVICxEm5joqEU/5iKOpYlEUnWdilPZH7I2PH3GP1JVoi9Pc+HWnzcgQL2cmok+ihri8/dHAlENIA43R7gv1ta3qCV+tLp+ggCu7+kFCDhPB86rld0Dnbk6ohogUmjMo3Xa80zYOLJ1Quk3iON8v7Xx6dq3yxCH3GZrvMy+3yzV5xa0ryGQ8xKqrhur0v8ufnYu6NuEbmg6Qsw5FYknUr2DsHh1JOpttYRpX4Q1XWsh3ltc6z2SWucdqG3O2aTMu4ac9n6myCkjqSFyRPnBnsjgoK1LkehiN6ry3mwGnOOHFXTYffNulZ57iH2P6t0klNqzkL7hcCS6cx+Y+xjeMdSpvTmIEvdD6Uh1LkptMdba9qgZO6d53oFqZOaMDLuQuOIgpKOYgDi7A1KZR6jqB9yHyfVvSxEwPQYBopNSP5vWfjcCGJ4zqp8EJJGVYyFC4uxs/TybANh+aLOorpn+70cRJ6bY/xkYzLL5G4SgkGjPAUkvcpzNVn3uT+eWbM6Run7iUCLNiAOee2ptbEE45hakKxndoi9zkYg4c7xOKFyADBTq+zoDyPb0/HwE/BeneX0v0Fdrc7+0JvvaXDyBnHm9nQ7kQlEQwrqEAMgPo+gRG1h9GxN6Qt/D46wOTy//BUJ06/3NY1kIbJj66Na8307n7g0EV+4cdLet83wi8PFQZ2tF2iurEUf+GevnE4RZt5+ZG63t673PqX8fp4qsPMeUt+XIpoH26O5IDJ6/aSIDkuuG6XPZG5qd0PwTSvK6EZRx9v9voDSh/AzKLij1+3ZQjuTpjLrrTe6n5yqCqmTZHepqQmnUng0wZJZdvzz6ctYx5ejN3QTwXkU4NO5qm/kChMhc0X4HAZTNm/0iAAAgAElEQVScCpxNUL11/YvLvhchiipzZVekw9RESOcnVGN2uSL/eAKYPlCb0zarZxWi/G5LbR9uz4+2skcTB9z70k5wm+5ZXxAHdyxhNu1RIHZLbR+GgMJSImSUA4llWDw7Inbd02awyFnUnXw3QAd6kbW/DAF0F/P4fEy38c22Zy4OcmTl486IqoG4gb2QL4ojP/cFG8oCcDcCkNxGDbkghOrWnZ3196ncslp7lyIE+X6qFqFNqmbOOWeUj+1+6/8cworTffn+QDildwI3J2RbiEDCX0p93gjtn8yNdBI5vFxMtkUaz35ENIqGlTmUIF6WEb5JjhQcGTogf5fVNRHpfLJD+0U29w1aG374/mqg/eyBk31fP4II0E9RRWSXU+V2ZqA918qCNsfl83E4tzqRSEZ6HSLeZiA4c4iVcStNn2f/P1uiep+mMDzBWv4MfT+Gsg2UC6GsNnh4P5SPQumBMgPKSoOLT0F5K5QTdb/eZM9dZwgKQzwIkF6ARGRrkOjPOY+ziIPaAXy9wDa3Q88boWwG5dVQbkoIaD8ox0B5E8q+O92efceebQzln6BsOTySKi02Ya9ttK9Yn9zkPAO1OsKpI58pRHqEX6fN/y2UWt6pw2VIZHc0YZzhqaz9QE8j0jccSZhbO+JbY4fGFdP71ebfnUILEYXhZYQlXk4h0IbEUx6lIYePcQDlda1GAGOQDsfqusDK3YUQ7lJ7/m4C2T+BuDNHUgUhnwlEMNDs6e9iWQ9JMwMzBkjtfszKXGLj2ZOqPq0j/V9q/68h9DXH21yNqtU/miry+PAQ48/pQX7X4r07Z/7C1u5XDA7CmvdXBl63YvEVCURYj6n3BhvPSsKiM4u1T7ZynYhYcaODbqQ3+Xdij92CIo17JPfvEVKF9w0x/ncjriiPqa67ucL6tIZa1HwkEnvK/j/Y+p9zH32+Vt6NkhxprEnfuMvBAbYnVhOiyw4k9vMYht3IYOS3hDh6NYqW70FpO9Deb6I9PKu2bplTy9IDf7YU7d01iJD7ILI6zfrA7tr3g65toLkYcU0XJdg43LXc4OJh8ay7rKfWfesCQfWgOFcjkZz/rno5v58M398SypWIO7rWkM3ShKB2gvIwYlf77NmuUB5HViz7Qtl7aMTkXIhv5DqS8vfTEEDIFOzNCABslYBBgxCpzUfx1J4gdD7OMWxhZbYlrIsKorLG2beuM7gRpWdwZDaApR63ul5LFXE44vs1QnqecC4jmQ/bt28miSVqa7Y51cjsM1Pd+fAtQZTeJwhT2lHWb6fgHfkcUGtjKWF4sIgIBTVAAJRMDNyPjA3G2/dPR36v1TuViJLtmWs9C+4frM5/ZXCywrpRjZvP34IiD2yCLLY8Hcoqm6N+BiNJB6hTEUXdiwDStijG3iGIq8hWpb5Gdf3jdUS4rM+SsvMi02Sfq1NazEUfhkAQQvw11cSa7r/lQYIH0J50vdsqjNtKdf7MynqCwCbmcJ3291cQcsrBVfP4co6jU0ki1FTPa6y87/VfEdyw78XpSP92K9Uz7ASM5+8aiwx0moTeaAPEYbmk5HoUp8+lFD2Ey0N36peLx/uR0cihBDJ50NrdAiHCHRBXmRHrA0RwZd9n9b3novwFLd49fR0DHZdCcyRPi+yGvM5BIj6gbA3lgXjXVUx3ub5dvvDP6q+trW02kuXugxTrb7fnrwLuK6WMyeVKKdcDHNHWdn877HlWquudyGzrk8hk7i2IlPPf/ggTfsfuf4vMkR4a3K2CNlAPQjzbIqA8GVnnbUb4d1ybvvt8+v5ctLk2RAB6ATJh/RTa5DtaHS9CVOwWyBhjPNq0fwbeQxy03ayu+YhS3cjqHIHEPq6w7kv9eY/19RxEsY6zOtrs/TJkOPB+pNRvQ5zFUhT1+5UIMdR/r0NiL58nbJ52RKLMndFhdrHdZojo6EWAq4HEnLsTAODiWhuvt/cXIvPirVNb+Xc/MuWu/3ZBIW3+kJ6NQsYYf0b6ky3QAT+fAKoHErrCTnu3u81HEwGuBWhPjLJ68zw0EILaFu2XXdDcOoLdFBmzPIHWcHsEbLZMdeT6etE67UiI7G4iEtKNtHIzEJcD2g/vsToesW9eiZCL/8YgRHiK9av+OxAhy/qvibiySeio+X7Jv08hBPt3iCh7E4E0NiT0OD7m220etrJyW9Tauw4RqPXf59C+WY64HI8zOB0d63ehOcbaHIH2yjS0D+5Hel3/bYe4I69jZyI/2ih0ftYgjmh3wl/t3wh/wnei/Twane/R6Mxuj/aiuyP8ydrcBe1v7+NpqT//ZnWMQ/tlBYoJ+STiBP+eOLeDfmdDs8CIo6yw/96ENkUvEp+8Jb2bjjbq4VQW/yxKOaRVG+vyty4Q1MtKKR+35+PRRhhVShmoI6gPt7XNvhx2flGqqx/JX76JkNHHEEnpv/2RdvIzdn8G4psfHb6LqxGF5QB2IdpsowlqC3TwdkCbdqw9W44QG+hQj0Cbflaqw3/F2hph349AQDPvrbHoUG+QnrkV3nZEBAys7u0JTmAHIlJyl9VV39hdRIwvdx5eaO82QojGp9y5Nkc+G1h5t07c2cbkY9+cEE1Sa9v9x7Bxb2z987YKOvQjCKTgvw5rs/5rQ8h+DgGAt0SczlyEQEfatz0IAY4hdA9j0Jaan+rcxf66g+RINMf+c8q5De2HjGgY5r7b6hpFOGu2IaA8Glk/uuPs3PT9eCvn6+bz8ZR9sxERIHW8/e2xcpsghDCb1r/6/sTqakv9hOBGOux+hNW7cW2cEPujH+1Xp/bnovnfxMboYrYGgcRcquEiSdAea7fn29v/vscywdBD6Mx6iEgQmxOSD6xPzjmNo7pWntKkN41ze+Is+lkYZe/85ynvt0FzPQJ4ic1bg7D2bFhds9K326BzsNra3hHtgz77fksEY7amxe9q6G7CmPfaoDeovd8RWRXtX3t+HpK/J4rxckp5X6s21uVvxNqL/K/+KthyHKz4BCI9/epEyMl/LcmK2m9g6FcN+7spkRV1Q3vmYrlxiB3/mZXxCOffsPdO9W9fStkVcR4Q5uavRjLmPuLg++H/XCllo1LKrunaFokM8nyMQIzgMWgzv8ba6kfigsUIGPcikdfIUsom1t8/Wh0OwB0ob4QAweWIWtvexuIGBKdaPS9GopUViChbQOghfksA1B0RwPpiKWVEKWUE8u/A2tkEAf/xCOiMReKQhYgLeSNCCCOQ1ZqPvxCIbN/aXO2CzuV/+DPC0m8z6+sN6JDvaG19yMa1kfV7FHC2fXutre/LkVhuR0TVOuD0yCEjiH3iejwH3lnE5O993l1ceLQ960Xb+sWI0H2N1X2U9cdDH11YSnEC6pv23UuszhMIY5sHgCfSXFyX79PzQwhAnc/cRBvXFETsrLT/5yCOJ6/fyDTGv7P1Hom4g62t/lus/8chRLGh9dOFIu8qpWxt300hfOa2tzW7G3HhU9E57ETr2iQQ2ldKKRuUUjYmCMwmOg93oX37UBr3aCTavosq+GggV5gd0lyNL6VsiKxs27BoK7ZGy5Dz9MfQvr8P6PfvEHc61t4VJC1wP64lSNQ6B53DYnPtsKGg/b6l9S1zm5Xfchj9RuvUn4cq1OI3gA59+q16Bp//7/3+N+SIVHVQZ6fn49FiuDnqXaQUAnfCD7aFcjWyyOtGRhLzkg7qlJqctf7sd9DcfmgdlIvwXAcwA23ua4j0FS5Dr+dpqivZB9Ahm5jKH0VEmh5JNZXAyS3maQ/CAsijDBTCJNUDv05EzKG3PRkdgEswY4pU528JZ0aPWP4g1bHMRkj0c+ig/rRWx36EVd/rEfByazd3bHZLv92s39+k6rTcjhD7DQiQurLeleZNpEcYhxCu+9z8ifAJaVJLYGdrdob9/w9Wbo3NtZsEN0j7Ln07mXB+9GClX0QKaw8jlOfJDUfusTXpIJygHyfCG91EGBA4N/EIQm5uVZbrnY5MjN+OCNsVSIzaY+MYFOoIIS83fHjE+uL5ztyEfBpCbv5NG1Wjl2sQYHVT9IsQkpxF1aCkIAB2N9ItPkQ1ysQDSJAxBunqfO0GCELvPARHf0jofNta9K0X7bcfUXXlaKJ930cYCDWBf6vNSzvaywcTjuuFEMFnveMMq+sfUhs3YnrlWr05hNZVCEmNtzU7idDvXmnrmXW1rQxyHOYsIujvaTY37h+4Eu2zIXVQx8LKLmieQNWKrwFlCpTNDWaeAmWJwcRpUF4F5SsBJ9dbHdT6hqAOJLJ+HlVgmzug5y1QtkCKvX+BMucZIKjfQ9lgaATVjzgdt4KqA457EUXoh/o6ZAXUj2THvtnOJDz7ffN7KvIhLXDsMN2MdDAzCYug+cBlNie9CGBeWKvfEcsOaT4rCAqp65r2/T5UkYGP2VOLOwJqlV7bucsVtecXpr7cTVUh7kCvn9YHvo3IMuvl8sF9Eklqm0h/Np6wlOpF3GQbMsueanXeYu8dYU5A3MmX7NlWqf3tbM7nMDhNdg62ew3iMn1N97Pv5wG/sf/fStVPriDdx3h7fz+WBwsBckcspxHI19vMgMzn9HJg09R3jwO3B7JQvJ0qQHT/uzVI1TCWqjHOzYij/ZnNy/4EwVC3HOsFvjTEuX5ZKputA+dSTQPx1fSNi1jXDLEnnHhqWJ1uRZiJu5mIy/L+ZyvUduCwdL8XVctTNzf/GtIpzbRycxHB4kZFDyCEPZkqMq5fHmjXLYGdkH0MIRo3FHLH55EEwfknIpt2rtPX6XakB5w2RNvlxbC422Dd2cggbIzBytdDORlKL5RPGQLbCMrOUI5CBL/Byf/bVnx/0/UX+kG1ugagTBhsnecboN8OQ05X0USUqx+Ia+wwZMDRmco/hLimJ9GBzwDaLdrGEQ6dxxAclkcz9vIe4qjOqfkh6K6V9XE8jhTW40kICinMB2xzOyU6k7AQ87QODSIz5zwCMFxFihRh729NdXuAzjwG5zr7ifTe7S0A0cFE6vEvIUQ3iwiW2Ujz4CKb7yFRxwFpDB4XbjUSC/rc/CzN/ytQmKJOInbiQCrbRQRYbSCA1GNztV3q871EyB4PEXQ2QS0XxMkcQnCOA/bdJES9z7Rn3SSuCOm0PU+U+5r5XnTiwfeB+4LNtXV3/6VtqQZC/pX9vZHYL6uQ6GtngrjoSXV7ZJJX2/3nibBJM4HXtVhLX6Orrc9OaHk/3BL2QSLJ4SoMQSGk9BFbxwGryzno01LfHUivJCwA5yFibiUWwQIhqCNs3b9DlXDI5t4eob8OH7oJrs/frUrz9FnCEvQqgls9NNXxEGFAcQ46f57JoO4U/ygisnyfP0GVUF5FNcdc/eq+FAYG/gr4aNcLflB/I4J6NiJJ5M030zbBVKrJwXqR2anHvTsU6YH8uwGq2WsdQLuZeQfSU/UiBPZi28j/VTskDoCWISrrVASERiLKvD3VdzdVU3c/ZPXx1UO+zEUi6l8TOYCcy5ht5Y6w/o1CvlbTCBN8H9NKqhGob0cI5lUEcPdvChJXPY2gENDwtBBnEXEE/y7Nn/uM7Ym4nG9T9VtahUSBjkzr47/E2vC1aSLg7/P2eyR2HUmk21iIiJGrrMwCBgdmfR+B/BzI9CHgUkllQGSjnUFE2C/Wpx/Y/Wa1+rcikPJx1odriIy6G6H9cbN9P4cqcbUcEQU+hrz+TQQ072cwp3MY0pP8gEiT4oSSG1FtS5hxX0s1Er4jQR9fv7XVjhDhdxkcvd/Heav1z8/NQalNR7YepcFFqacix93rqCZmrO/7/CwTUfda/3wNH8cyGFvZg5Cu0X2iXL/oROrrrX9vIFLIHJfazlFOvH0nEH2/fhYhrlX27d21b+5DBOwsqub4LWHZ3jDQ+dcjqBciSfwPIKm/KhbfF1pzT3kT/zeRcM43q4sS1iAA5gBwpR1ENyJ4zDbqTgQQ8FAyub0GQoLXIoV2Dk9UEOAbgwxqXIT3ZyIz5zIsooK1dy8Rh6wHAdo3UxWpDFifTkEihBsRBTuHoYPZelsz7QA/yeDwLH49nVoi9et0O7CvIA5zk7A+u8Tu7yeFXbJvRxOm23syWD/hoYXuQwCqVRw2v7ptvF9FnECuZxLmi2b3kwkT7bn2racR99iIbUjU6Pl6+hAx00f4SbUz2Jl3PEGh+57x9C2u6zggIaelCFEeS4iYmiQdlM1jL/C91M5IhHCPs7kdSl/RRMTICbQOveRm4r4WqwYBC+mG3C/nBAanjliGzKIbDNYNjaV1gr1+BIyfQPtuDoPTp9fbmGljvRYROp5jKxMrUxCBswtxDnuIBJne9+X27iYklnOE6eP7ZarTI304oZcjkjisuYsQCS8iiDBHkpNtH7gkpsfabqT3zvX1I6LYnYNbcXwFKIdDd/cLsfjWCyS11mjmHVCOGOyxnjeSU9PdBDX3PiJ6s5f16Bb/jADA3Fpdi21DP4TaOwXpBbzuUwnq9l4Gy7I9yOTpSMxxXnq3BCmsnwbYRFbT3RF3VlL/XTzUm+r+Ba3ju7kT6wCh23JxnQM4jw04XMI3N8V+lBBHLSKMDrIobTkpmnStP3cT0QSOQwDqIAQcJlFNWeDXaoSw6gfW21zF4JQk4+z94XbvCMN9traw57tae85p9iKEMsnm+BxEzLgOrYG4pdfY9wfZGJxK/7it/Sfs2akIgDbsuccJdDHVHgRxMEAgqEtsHjPSHYkILZ+fOkfZoPXe60REyERkuPAOm4s3EXtjqDxUzmF4WysIH7CVBLc/3cYxHNL0SPRZ5+WOuH1ENJIGrR2Rt0W6Iv/eOTiXLngf34K4xQHbB/tgTupIV5s5y8MRl+PGCvmd763J9rwLifz8DI1G5+tRwvihDovutTa+bv25Pb3rtfp+bO8ahESiFYL366mvwJIuaDSopiJqcb0QzfxZQlJ7FZhQpNTrqk16VxN6JkDzjRHqpNWBOJIQm61EyMEPgOuf6pGO7yPysxyJrPPq4Uu6Uj2FKrDL+aMayBhkNeF1XgeyHuvMrdiayHx9GaL4n7BnjowutLbuJZDtD63dAQTYXFfRRphOz6AaaLUg0cmYBABOsef+zRbIJPxSAnl1IaTvFKWHYqrPv8eFm4M4hNuISBLuve/96LVnt9k8ZeT5KapZcD1ZYQNZUd1JVdY/HxnobIeQXhcS6XiZE22s7yFC8/Tb3C0h9FnjrfzjhDjzvUhhfych3mwSVmsnWD1zrUwPIcbazcbZk76dgIiSkURCxCeJJH7vRgTKb6hyw51IvJfFRecgEVkfEkl5aB8Xe95t41hKlet1vzsfq6cEmUf4KNXXNhsEPYG4ABeRLUb7aDeko+shzkROVLgtsYf8HLUTHNXJqeyeRALOeejMnmDlvov2uROibml6D0LwN6Pz8VSqo5dqyDOfwwVojz5kc+5nqqCz6Knhj0ht+Rmoz9EOtr7d1n5GOP1IzNpHSE8KIm6yHu7YFvX6tfD10NEcBkba8wllPRbrPXcRVCCqcUVWfmcWuMz+HlXCCzxvML88eGXTNmhX2hhL0MH3Q/MwIao7j6BwXEHuB2GybdaJRJBYb9cR0Eor83h6NxLJ0bPl1hwixtkPkdK5FffSQAD350iE5cj4Uqt/cWWBpTtzsWUdGU6z92MQMHVrot9h/jd2P8vq6qEWdw6JOH9PVVbugGmqzcNva+96av1o0JrKbqa/TyGAOosAshOQL4orto+0ObzTxrQtAvKZq+istTcH6Xo8zM99CAG2URW9uYJ/LkHh/5c9G0PEZeu364cEIrqViJV4uT3z2G8uShyD9G1udLESIRgnQjwUUTai6UFIZxRCnp4o8Z/s/WMorJiLC4v9P4WqQU/9rPg4mrVnXQQAbiJL1oKAd13ENhMhirG1/XI62mejCCLm21Qt+JzgGod8xjIidiMTP6f71ep3i0K3jrwaiUDPZbDbwNOAHRFxxyDx3CLCuMPPzDsRYenjfMz+X4jOUd246UobZ3tq9ztU40pOIODG6YRo7zG0P9xwy9fhljTHLUV9uC50GBi5zuH38x5BDTegwekzCtUI5Z7UroG4BV/oB63cSQyOIr3ENttHqCp8pyKqxw/Op1J9T6bnBSFOp0pzYrIf2mZsIkc/0GE9FMnZs9x5tW38bARQR16LEJX6c3SAHQjkFOxr7EBNbzF/DxCIxOXknjJ7Ni38t+zdS1vMe0YwDlSy+XY/8PNUx3utjHOIS5All6civ4o4xM5xZB2ZW//5oZ5JmPFPYnBcOL/canF8i3GNJWIjjiMSD95l779OcIYH2rNv2rPlNqZRhJhnD8SBeuy3thZt7oe44VacSkFI80pElHiurPr4vWydM3gMxaT7PeIqLyE4did2shHEZEL/lxFX5hDqiKyPFrH1rL4HgT/b/8cTObd8rL+ulffIEI4gfUy+lmuQ3ue7SIRaH7+fQTciuZIqMdRj6/AOa2+7VO8mBHGa+ziTOFeORKYRBhlfS+++TjXT9k0oPFMnVaK1IGL4Fqs/m9W7FKOLMNQaykDqp63m/bl6rfMO/I8PaHDuFJentwqbXz9UBQGzS21T1QNaOrv9e3RovU6XdT9C6KnmEPmBtkQAtm5w4AFEd0htPUo41k5CnN0sq3+21XESQZ2/gtam6X5AltmhcFn4W6hyZk9T+lbf59K7Pqp5n65ESPMAJPK8GiHi3H6HzUs3YbjRtHGdbuPwILQDwCes7oMJi8IFwJtarK1Hrvikrc9khOicKn030v8cRZg7z0GEhKeLGGqe/J0bPUxFwOBwJDKdbc9dbHQsAiL9iOipG41kruoBYl9eZe1dj/y7voI4pRxU2PsxlA9dv/XDkw02kajqCIQQf0IEXvk22qt3IfFmg1rUcOvvx4gzcA9yyG5DuhDfCzMQkDyVEMkWAnktq+2FTsKM+ttIzLgK84tCXLufTx9Xdm0YSzWt+0NonwygM3En1fxJ9WsO5ieIkI1ntXWz7ZPRfndnbY+OX2y99kD7uZX+7jhC9PsJAnlejvZJX638ZJunbqT/uj+9v9vG9C1iL9f1XvXL4UVGUgPAt9Y1DH4BQQ2PoM4jTKiHzaOylutUpKwstUPwOEIozrYfh0RLdb3Xr9AB/wZhKdSJDuvmVBWjdWT5LQIBeSRw564OI0Q9C2ob1PUf91gfP4WQQisfDI+D5sj5HDuUDQSUMrB0x8tsQLGE8Hz3crtZH7cgDDDakeL9ilqd/YTzqCP4B+27uS3W9Rx755G5PdSTuwCsZrBRxNepAsAGAkorCf1OsT5ehbi1BwixbF6TVmvVg/bDvUhPc5Ndk2x970eAdDgjgYx0liGge7LVl/NZPWn1ZYDtHNPbauP2PELn2/vlhDXZ96y+L9W+aUv9nJLWpQ+Z1uf+PoX21a52PxFYkur6IsHFLid88XyvdiECIgPZJwgjpZMJy7amzUNfat+5pFXIEOiDyFrPc2t11Mp6xuklSCS8k73rphqBP/uR1dfpdCvjlrBNhMwOsu/rzrZTEbHkYvLbkdFTTjzpXNK7GMYZdy2Xn183WjnsL4WVz4VrnXfgWUBQ+yDqdvkQizncfavLOZliG9wRUQMBxVchsUnDNmOd2vLnmyAAfSjiYq4mqE0XX1xPAMZOBPieIpSp04mDmpHvYYiKb9hG77C/29o8zCcy7Hoss3OJIKt1UUGeFxdHLkMy+iZCQNfb/7cg4HNRbR1+an1trz2/vFa/cyxN6+cse75r7Zt+LJlhev47K3swEmdOQCKa36Z1yhll/wXpD7sR9b7S1uAs62snAo77WJ3ftnF6CKG/ZD+1MgVutc96EVd2KSJw3k+kUh9AwPdEK3sTIQZ+OxJneb0OjGegfT8GAf/TkCNxIXElNm9H2TfHpmdHEKk/5tr7DEz9OqpWl/fjHbXnTyJkP4MgDPazsX2H0PkNRwg4wrwTIewVBEJ5ldVxHyLGVtm4fT7eilw3fO39vHjiTRd9Po64vL70/jKqZ6IdS8aJkP4jSOdX918qCBneYs+vR2cll3mYOPdu/fuXIKG17aeptv4vXdcw+AUEtXYkdRlVUUMdAC+vPR8KUeXnOd7WvVQpphnIKXFnBEz8+VJCJOSKXuciHkZOrtekvnwSAZg1VL3X/fIkaS8jlKguurrMDtlKIuq5mz+PRpzYkYQj51O1eelmsFVf1iFlk90GQgD7IBFhD8nyz9bAI3D3pWdfSd+7GfVqwmrO21yFOOEcn22Q1REiDGZaPdMJgLYYiWE9AOnvCGDStO/uTfPbx2Aup2FzOYcqUeLiVy/XgwDNiQhotiFAfJrNZwbAHg/R7+cQqcjre7AvfbsSieiWEAjwQkJkdZntC3dmbRKOtctpHYfws1bXT+1+OtWU7I783Ny+Oz3fK9VTUMqcev2ep2x/ZGHnpuSN1EZ9vouNoT5v/YQl36lW304ISThCmkY1GrxzgcuIbAX7IWKz3na7jfNR6/uJ6d23kKWoi+5d9+N/M8IpSF+0MTobWZQ+iWqkFj/3wxHJWYVQaK1f92sAOGZdw94XENRfhqBGEOFtfANmJPEjwnCiQRX4ro2SaRI5mYodjKwvmGV/3Zn0WgZb4y1CIsBtEPdTqFoBZj3EdXa4FiNklDnDNYgjOJSIBVio6rua6WpHSLMPRRp3S6cna/O3V/rGk9lNoIrQ88Fahric4xEQ8PAvv7f3H0VIoSBA8357v6utw8MIOQ/U6p2PAKMDfvdf+QTSszhizXPrCC+vWZ89z/H+rrG58NBHByBKvM364tylI7EFCAAOIKuw2fb8J0j/U+9HO0Icf7T7sxDgW0Q1aoRzqFOICBe7IuLFkcQ0JCLK3HWrJHc9DNa1OmL5BuIK9yKS+B1sazorlXfJwIcIrmSMzcvXUps3Ik6sALtbfWMJHdj1hJ9fHSH1ED5/XqYLZQTIBI7P6VXozBaEKHJyTq/TiTdf84dtfH+gehbm2/huIQxg8r7zve1uFZsh4u48BnOUHom/oLPUj+oX0yYAACAASURBVAgvJ+jcVaLYXF7I8AipDmtcjeCiSzfIeiqVKWgPH/BswtQXENSzg6jGE/LnW18KZxwNzYug/WZYNhHmfgu6t9YiX1db9PrlZsJ+/xM7KK6fWkqYUPum8bK/soMyB1F+lzMYoK2xPnQjZOM6qg/YWC4iUqA/ageybqBQB0wXWP9WYualSF/wKBH77at2QEemefuijWM7quniHybMoJ+wPrYjUZLH3nNg0EVVfDEAPJjaGIMO/v+zvjoSv42qAr6+JnUR2grCv8gBZxPparI/lyPd99oYeu3vFTbfl6eym9m4s/hzJuJAC0JmE2p9momcVX+IxHbZCquJOHB3FHXDBZ+XAeRjk41V5hCOwXdbGxORDqm3NradbC/0EwYoy5BOLyOsVuLHfE0noim47ubdiGN3omMUg40APIdSIeIF3kkQSqcQ58SdjX1e/huJzgpVsa5zYIcQ4cIURFrv32zjnYGMd4YiMLuIaCF/tG87gc/a/65n/i3ah8sRcq0j/1VWZqy9O4SIq+nnvUkYZDmiWo0IjBtT2aGiuWREfiFQXgwLz4arzoW+K6H/XOj7Pswbr71WrK+bPhO4+Fy61nkHnvUBQtt+MG0KPNINpR4OpBf6u6FcDM23hAn5UNZTTaqx8AribPZA4U/qQOk2ArmtoCrmmEUohf2wz0D6BbfwcfHVGUQqkGzxtBgBr9PsoNU3fpe9m4OAyk6EvicHuuwi6RYIwD3Byj5IiJxcvOFI5D8QtbsTUkB/jvAP+0uoxX5CNFWQKMvFoEsZzH32IzFqO+Zga332CB5N4JMt9sFMLNit3btv1gIiisLuSJzTT8Tn6wW+TDWsTQMBzDWE2OaztfY8fcdKwtkz742HCV3QmTbm1Yg7PMLmb2fCIOBzvp8R8rmkxRh/nuoflZ7fjXQt2QHUr7mpjaUEp+tRuoc6C/WrGxFen0BGMZ4M0F02PCahI/2nbI77bc7nAOenPl9FhBPzgMsuXn+79esWK1dHJo8gnde5aO9mcbYj0knE/nZpx/mIUHDjDXc4n4jOWpPwXZxl9/1UDRya6Px8G7kT9NbeuVN5qzl0IuvmvaF5KQz0QX8PDGSY1Sm4NXA7zC6w97qGsc8q/F7XHXjWL/jCAHQPpAVudQ2g+H2HDZN7xQ64K9OzSKDYpv4dAnIF6U7qdT2KgLhb6Hkg1FdaPc7t1dttIMDt7X2wBXB6l71baQfNQwflw+H/n1/7dgKRdsBzUDmAOg2ZJzuHOZz5a54T19841d6NOJDvIj3CTfa+HsnCAcNNBLGQIztkzvRcRN2/ixBlFeBjtfF9zt6Nqz1/zJ67cUbDgMTphOjGgdAM669HYXcn2y6knG4CX7N6z7d6P2p13EqEe+pD3OZ8qhzaRYRyvmnz7dEZTqv1+91W5hW1578niJjFNjf7UI3W7uIit/pyAHoJ1b3n69FhY78acTvfTuP1sm6Y02qv1S8n0i5AhjSuO+0Cum0cbdavz6ex5cgQ9TZcN/eZtIbjW5yRn6c5f4DW/esjMka/zMrvYd+PQr6QdX+6LlvjXhThYgKD9Yq+h3NkjPoZL8CKzxsSajB8yKLmcyhk0V97rfMOPKvXXxlk9vPVDVO/rkTUc958Awh5zUnfzUDA/bBaXdchefZUqiKXBgJ8jogepioGdEX9mS0O3r8SXNkUZBTRjcSQHyNk9nVRmYvWnEupW1R122Gaa/3NjoVODea5WYxSNDgCdv3Lhwxo/GfLTaiD71HOW835SpuPxwmgupJqgNzHEVJzi7tXWd2eBfk3LdqdjKjwus7KCYt7Sf5Ytu6nIWB5NEF5P4gceJu27v3If8p1Bdkc+WHM4tH6diBBnbfS7c2jdcLC+4FH0v0hhM7qJAIIugFBPyG6exCJaFv5BjoAP50WTsTW1q1or3lElX9M63gC1f3WTOvkXPUUIjtzK2Mg34crCAOGeh+Lfb8XEifOsD60o/3a5Xug1ndPYpnrXGlr20CcXtYpF5uzyTYnORh0ByJ+JqS17kOEj0s88nl7gIjv12reiyGntcXSq1/PWyS1zjvwrF1/Y5qOvYamArvtgE6kGkbfN72bAB+LqMPHaJ2N90n7/wMIUH3UgMh84Pp0oA6gqpxdgowRPoao2ZsIZOEGAqtYu2hmLuLY6uGPmsC5Q26YMF1+nEBYmXo+FwHw3W1eeu27o2189TQT7yRiIBYipUgnouqXIgX/RQjQ1hXVTSK7rd/PtzkYgxDKagJpjgQ+jEyQs9FH1gmtAa5tMfZuBnNn04hIB77OPqcDwB9azN8TtWcjbS2eTP3J4+xBXOd+aRw7W/8Ptrl2Ism/X0lrSn0ZQm5nEaLEE62ue6hap16EcQ+pr/ta/XvZfbG+erisKakPbgBxG8F5f3aIfdXGYOs91+HMY+j4mhmJrCDOwiob03lo7/0bkgRkom86ZuBhfVhEJH8ch9Kun0QE86236Q7d5yERey8S19aNrmYjwvZeWsOVJoI5pRPKn1CywY2gjLP/fwOlCeXHUHaHsjGU8XafkNRzIr7eM7nWeQeG7Zxl2v2rrr8x0eFFaeMMsaGeIMx8m0RmWi+zElFM59jmdMX9KVRFO1cg5PRlAtFNsoM+i1C+tuqH17EQcWZzEbD+Eop64Tqjg20+z6MqovsIkXzNw934+PptjCdjXAQCInem76+n6pzrAHKAMELI+aAWIcS+j81dF1Uuct/Uzh0EwN8t1fElhIRdJ+a6E8+EWl+nJtIlXUYEph1I7zIX8ub03Stre3ETe1dHsG+liuD8ct+4uvn9oUBni73+qbSmObCvm1H7PA0gfdqfUhv1fdGHAHomoFwslQ0RvmplG8Av03Pfo276vwJxwy9FgPq6VLYg7iEj1EsI/ebbCAvVQsQvHIl8v84n9Hj5/LiJ+9eQftfXZQD4qNXxQ3u+AiHqo2zP3Ec1tNVQnHk/2jszEPJwse4q4AvInN33u+vpPk0gy5/bWB+liviy9Z77N7ZCcBUEdRn0/phq6vYmlPuhfBRKD5QToNwHpR/KY1BeYgjNYN16m3jwr8YB67oDLQ7qbEQxT7WF3sM22ypErb4vlT0DKbsn2oa+HXjxZnDy5lBebovriOdHUHY16uOVUC5O706H8mYoX4OyOUqLvOnQGypfA7bRu5DeKRsqPImQ0yqq1HWrA+OHs4E4tEtsbG6g8G7ioHcSaeidc7vWvpuCPOv9kGRqd6rV+waqscQ+iSjgYoe1DYX3OY8AHln8tNKe3dBi/fZN5RzI7YBk87nP0xDl7uKTwuBwQZ4ld1IC2C6W+rU9+3eb+3bEVTaRBVQ2SsiAwMdyv63Ptam9cwhxX4NqqvKPAD21/m1FNZ/Y75GxyM3p2QlUDRa2tudj0pg8VYX38Qup/EJCX5f3Sf163NpdhgC3W9O9DYneJlm5qaluJ36+VxvXOwhDkWXICjWLU0+zNd0r9bkgIL1dra6RBGFQNw/vt35nIsh1vG1U/ZEWWJszMa4UxTx0I5tlyB2hHcGFewkLySba9z53JyLE10QE4h+QgUfW87Y6ox6w18cwC53v7CfnUdUd4famuoZ0ZdkGehcjrumiBJfWdn0RyhFxv96mbn++IagH0EHfBAHMY4AN7bCtAV6eENRylIriRciUc9aRcF4/dH0byv5pMS+AsgBKA8p5thkWJgS1AZTfIw7qF9AcOzRS8mjH+XDmTVyIZHYNQi49F1GbxxOA8mgGi/H+LQHfJhbKBgH/vNEnIgDwBSIStF8X2ze3ENHIVyPF9Ew7PA50uhBwL5jllJV3k3gXe7lPVB7zA4iLcNHT6QQVW5+buXY9ad/0IwDxNVokyLP6fFyfIpDOp9P70chUvZ6wcWZanx9TjWTdSdWyy8dXEKJfTiDmm62NPxCGJLsTmW0bNo4c6sezszqQ6kOIzE39exH3sBNhKLIGcaBuEZaV7H2IS8h6k+nI0MAB8Kz0zvU876Jqun6Fvb+TAP4ntJjzUfZuNyLj9Hhr/z4GW4s2bd6aWJw/tC8PRhxKHkfT5tedln9L1WS+B52d76X1cZ3v7YgbnoIIqh6EjEdTDeI8gBDye60f7mTr/fC9eqy18TqEEM9KdTxm63xhmuNP2FrfYeuRAyB3pLbz3PgY2hk6t1rzWFh5KZSRiDv6S5BTE8o/QPnveNZVapE+nuvXOu/AEAjq0ASQFwMj0vs/AccnBHVKevdF4NECZxUoU6FsNswCvwbKpQlBvTS96xwaOTmAeyht+kyZu/PryXY5gO1GCMXl6osJ01s3/83OmO7AOTnVdXKtDw07KKcj3YYjDwe2fYQ5rOt5mgjQn2n13ZPq9LFcShgArEYiCg9RdFE6yItr3/mBXUJV5NFFIIg1qfx068Nse39yi8st03w8XYR/kLcxYN+7v4s/m5vG4bmWLqMqju0ldGluMFK3vnT90hpCbORhkjoJp9Pzrc93WR0O3FYTeqmnbBytCJxee++iz0fS+14kcj0PiXO9j49Y3c75rUHANae8X0NY4rmYzOez1ZyfbO9uIwIj+3eub/NAud731USMuvrYPGbkyUTerD+ivef1zSE45rynrrLvJqQxOQHSl+rO+74dERQXEqnbs3tGHqcTIj4WjzLel95davW4BeekNLZJVs9EIrJLL9pnvg+nEYYqreBJ71lQzoKybQ1GvdFg2Iug3Fx7dxyUVyPRX3o+yIjquXxtwPr5m2d/twfmlVKa6d0cxOr7b0n6351cNweRWB3p5Zko6uVsu+9ApJz/Xpz+32j4/o1BzpwgCnFbRIGtQiJJkK8GiIIDbcQD0Ib3uHzbIQDqAWFfjA7ZpogjBIlRdkNAYpvU9Y2JlACftPZdpOY6mc2BHe3dG+3bZQgAvjmNZzkSPWFlDySC0faiQJw7o0O6Z/rOAeLGVs777P1sWn2j7VqMIru32futbJ52QGvnc5Z/2xHpuUfZ9RZCrLrM/sfa6LL+jCSCgvajJd0z3bvT9AZofv17z9+zHeLa220eN7TLf5sTFPI7ECB6N9p/L7H7t1tbm6TvtrS/zmW12zNHjptan0Yg94OC1nupjWlv68cGaM5fWZuvDqt7uV0vsvZfgnRIEHtyFLIAXUHr3z+gMzgKrcFItAdWo73asHLN2hhfhNZhI7Q+G1sZX98GMlRZhPbYWGvnJamONgTsX24XNp42a2sXe7Y1Wq81dj/W3h9KpOrw+vptLB+yMYyz8vm3Es3tKHR25qMz3Y/OfAMZq3TbOHeyayObn+2sjvdafXOs3Birt9WvfWvYtMBo93T3gnfY3x3RBPrv1wie3Yo2bfptPkQbz8nfiHXdgSF+xf4uBHZqa2vL/XwJ2tTD/VbVH8xBwcd+jXbzKuDvU0PP8OeOqRDAGLQ5+uz/GYjSvR8BkncgBOU4sx35WXh6CxDFuj1al6Von/6nvRtPBAK9CQGWjZGIzX9zkdL/paWUXW3InvvJKb5t0AG+DgH63dBe93E559aGcj+9lAhZtJ3V/VIEXNxarg0dwoVofUYgEd7dBJU4AgG1DRFF2o+AsYcX+qrV+2pkHr8YAaExNqeO1AoSY4wupWxRSnmJ98m+/4G11UTiqAHrYzsyTshlH7Dxzrf5nIKQ5o72boTN1YC13UCUcJeNfwJCSA58xqIwRRsguDEK0UN/sm8WEtlrB1L9vbZ2O1td16Lsvr3Wv1NTn19t8+bWdu2Im/ffa/MYSyk7lFI2LaVsiDi8DOcaaM/uhLj4ryODlJcijqAt9WkJ4tY2QoCdNC+/A/6+lNKG0nT0EjSeO1vfmMbwNrRfDkbpYq4nYJGfnwJ808p/yNZjG7TnITivecA/p7rXIKmB1/EQQhwvtzWbggiC8QiRfBrtS+dSd7a18jPzehTVwvffNEQk/Ks9uwhxUX6u+5DBkCPvXdCe2I6hwc1mT8GoN6JN8+chCvnvNOTZfAPaqLXfINj3nP6taxZuCBHf2+3/DRFA/iY67PujDfiKJOL7Qfr2M8CkAkcX6JqOZLoFyjQoo5HlywCU0+zdKUnE9+a/XMTnlHNBREwXOsDZkmwAHRQ/SK2U2y7GWEUEt3QR2OOIchuDxHyu9C5W50GERd7ByDDCfYTeiA6mm9lmGf1rkemsJ0/0sXQiQwDv/8+tTY/d90Pry7EEcOxH4j73+ZhUW8vDMCs+5ORZH3shIhe4yCn/nUtYCbrzqceo+26trc2ohp7J8eE8YkcTuCB943qobdKzHajm6ilITNtkcKoLF1/OtncNhOzcYMfFPb4ffNyuC3qsVt9lVPUxb7V3N6ZnMwmR1E8IfcqvUj1P5wtL3020b/a1viy05+8h9pyLiHPWXNfl1ff+/7PvC7BFamck4Qw8Ma2JG6Y4Z3Orzb/74p1KGFK8Ja2X7xOPzuC60pMQ0HeR4VUI8TlX/yoi0d9MwoCm2Ho1UNSIzZCUIkc770Ni2iwarF8urmsn4jI6kjuViB4yicGpOAbV9Q3o70RWetmKrwFlCjLcugnK2UgM+EhN3GfXCzqo/00EZfeukG5HFNwH0rszaI2gtinQnRFUgXIMlC2gbAXlK1DeMgyC6lo7gvLrcaqy5axTWIaQ1ATrn8ccex8CXg4IOhC1X4+ynq1++u3g1WOrtduh+gIRs80PYTYpdrPjqYhibVh7d9UOrvs5zUPA3bO1Ome1GlGNpyDkdaYdyrfaYf6vOoJCgT6bdr8n1VxUdcDn/5+c6hlpZVfa/aes7C8QsPshIdpsIgT+6dq+ehWhLL+diKx+h71/PaKOHUg6d1hf714iGeKvam3caP36j/TMLegaiEhoZcm1HAEzt/i6AdjJvvcYdgcQRMUtJKRg5S4gYuJ1ElaNbTZeF8+OsvpyIso2qtxuvY++VjMQQD/Dnn2QwQjqFuvz+wiz7CbyubqJAPqOuK9DBJRHWjiX0Bv2IT2VczRnWt3LSBakCDG5kYIjnu8SkTPyWJ4iInTUx+v7cSYi8Px89iMY5HEQnSiYaWuez3/WvXrKnAdr9dev5jgo3QZ7zoayN5QxULZGflAnQ+mFMh4Zc41N1+cDbr1gxfecuf5GP6iLh6acHiEMJPwQrbDN+AEi3XPdL6abSFvwcyROKYjDmUxQiydZmQYhjXwDEhH+OwFkZzG0VVA+eDnigluBrUGGFQejiAcNBLzOrdXZm/66P84ZCMi9KfXZ02V/0u7fa/eHEZZbj6a6PZimt7MIIR0H0H6QGwhxn5kOvnNGzjF6DqNvWBv3IaLhnJYbvhrk1RO9ZcCxxtb3AuRX87X07mEiHb0bqTwJbGV1e3Rvv9/T6usnUpvfQ9W45CBk1XmF1Z0BnLsw5EgXhw8xrsfS+p1NZC/2dBnOiX6T4Ly/TxjRZES0AiHs7F6wJM3VEgJJP42gkHl1juLxvVSHR6N368ecBDSftYWI8DiQ0Ft9gmqq9h7CebadoQG/z/FiwvLwLrTX9rO570dIbMDquzn1+QKCO/o0YbCzqrYmvoc9vNVHU7sZ2U9mMML0a8VVsGZtIdmGuV7wg3pOXX9DJIlOKHsN3oBPbySqToWzCR8TF5MUZNlzdvrmCCLIZD1g5OJU5wARqbgA29phfy0RHaAX+Ig99zh0Xn45En18zg6yI8ZCpH13679WyRVbUfgrEaV7BgLMXUg3VjB/pAQoz7IxeBrrYuXvQCKa7VLZj9fankg4eP4zkagxA+yFSMw3lQCef0aRAhpIh3ARIiQ+iIiBSVSt/urXdODNQwB+5xK607NNEHCfm+pwfeMA8qdykWrT+npYGtvI9G63Fm22IaCckXi+PD7eVUjn9nZbl88SYtqXor3mJtITEbLtprq3G0hM5ghsHNXcWgPA61Lfdkcm6lPT+nYQURIOrY3FTcRPsfJLkCr4blrHdazvQScGcuoJX8d26+dym4PdqHIzCzEpAjpTiwiXj63S+Sn23K1sXVSe598tYpfb82NSP/8DcY7ef5+XM6nmdHJiodWadr4RlnXxjMMc+fVCJInn3PVXxOLrgsbhQwMyNzG+j4gcXdJGPZAAuh7zKx+0BlKg72n3HyBMjrtpnR68j9CB3Im4l1kICLr8fSIhm/cDPotq6msPxDnW6jzW6nLFv4tY3JR5VaqvPo46QHFT3iwq8asXRY7YhsF6ETdA8Hre0wJY70IguXrbyxBF7D5HS6lmv3VLxNuQv82nqfoX9RNpESZhQCu1/ak0n00sj1WtzFZUOep8XYplgK198xuCe+sB9qy938PWwXWDGWA/gZDdGQgpuNFFIbLh+jce27GetdVFtX31NbH2j0lln2qxbm488bYW9TpS87HV16xJdY84snwY6eD83HQg4m+5tXkDsuiF8NPya3n6zvW9b7M6G4izdCf17WxP9BN631xXF5EFuAMh1B/ZWMYgA4n6OnuIsgEkWfgjVa5wwOpwHfJQko+On8INzWdOWL8Qi+85ewWSWpu4r1Ggc4nCv7TaPAVRbG6m7cDXY7HlDV6Q6OxkO+CZEnMuawnS27hjppurtiMK9tZa2z3pez9UHUg5/noENJ5MB8KB2lQiCoE79p6R+lGQ2OkThGzdQ+wUBIhWIYT3fiJPVb4G7IB6iJl6JtD6YfXIDz4OjzTQRAD1/YSByBkIoPxnrZ66w7DPzQzC6nCUjedliMPx8schQNW0fr/B5mgAxWtrI4wxfmbfXEQNWCPrrDMYmtOZhwwZso5mWyL/Uy/y/elHSHwTIizW3chgw51AxxApzAesXU88+F4E6M+idQSNTNUXhMA+Zs8OJgwFPk6key9U4xN60Na6XjQjnUVIRDaVMJSo75NLkGGGJ6p8i727L83RaCSK9P3sBgwfRRyeJxO8mQgh5LrHYus/GonFJ1DlBn0+6hKS8wkfslFIp1vQfjiYsKjM37gh0scQwegIN8/H6jRvHjljKOnMGqDtmcKs5yty+r+BoEqhwF4FJhQpEbtqi9xlzyeUCAl0zhAb6Na0+Y4hkMtwMvCSDtGstDkXpe8adt+DuKWRdnDzRu5Biv0u2+j9dkBc5FgHBCcgStsddF057Yig355dUJ8vFG3Coxv8FFF/dYTwGIOjYV+CxFzdCAg1sGCxVu9WyMLwGqvLqd5bqSrOh5tT50QdGLkl3sVUudGCgKDnmnKAuU/qj4vhPm/3h9v8OuX/NZvDDgTwepCO8HdEzqB5NtfOmTmX/QiBvJpIZ3M8QvAzidBS4wjz734k7n172odN4Pepz0fbM3eKzfoQz9+0CnFXbjDRTHPSSoRbn98BQsHftLG68cFrgU1Sf4q193ErczqDo5osoCoOLQjhHY6QjDtKv6PFXpyR1tW/zbrKyxBnfBaDxzJgfc/pRhYh/W9dxL4DIh5WEERodu513dFSYn+sTQfsbfwJObp731ulbh/Acn79NTDr+Xqt8w78r14wrsBRBc4scJn9PaoMzhO0L1Xge2ICAh9PG3NtiKnV1SDy1zQRsLur9t65izvToff39yDuook4jen2/yOIslxOILDhTNvdWOCdtbHvYW3OISjPlQiQOHfoWX4/Yoe1HmpoGkKw3m9P8b41QUH+gMih8ySBcLJIzSnnRxFy/k+kdykIEXpa9rUBiSzSm4MiDHzc+jnd2nut9fFn6ZvHrL3HbQ94OKrFSI+yHeI+Hrfnng/qeru/GHFjeyCFu3Ols5FBRRcyCHD/zCYR2NfTtCyzbw9H/lGemqM+tqGufiJTbhOZQB8PfIfg/FansiU989iPvyIs964kdGkeLun1iIvIubqmIP+lQsQddLPvechgwNe3w8a5msHBeC+wMjnKRP1yn6V2+/9WQs97B9LNDSACyQkLN+tfTsSszIYbBe3fm6gGzV0bgh8OUT1AqAaWEu4GBe3/Lf5amPV8vdZ5B9bHi3CUXWOHuxuJQBxp9Q6zCev3rQCIcwEOaPvtcM0gfDSWUEUwywgAmaM0X0Mogi9BQPei9J2L905Hupg61+PWS8uoIonrERArSES2F2Ga3o4ccj3C97bWzgoi3YLPkyMcp5IzsuhDwOl0ZNQxDYl9fpvKtTNYB7ICAUe3JPwnpJDO8+Uc2sn2fIWtaau1y8DcEdDDVEW3HTbXy4BrrB9uoNIJzE/986gFPYhif5k9X4C4i3uocserkVjM13eokDjOCS2wvx1IhOguAF6uE5lZf9va2RgRQXfX5tEjIxQr9yWr8wIk6vbEi11UAXODahim7OPlYZiuJkVuR0YvbiTRYW0faXN0HBEwuBB+ha0igC9HiHoS4VrgfTrR5vIwa3M/IjGhGz74mcjGCj0Eh32DvfPQTS7yHEos90wQlnOBDyFxZYNwDbl6XcO99fFa5x1YXy+Up2gPBGxusA36/WE2nx/We2vPhzJXr5fxupcjB3FPPe4iqpkMRnZLkYhoB8SJOUCdh6j3JqJ+s5htV8IvKffdTXf90D1lh7sDAdVOQof1oB3il9t37Xb9EQHCpbRGzO1IbOYWVPOppnlwk+5ugoNwIJ99ulzENhdRy+cTGYBvo8rhzre5qyO58UTeoAysVlON8ZfXttVa+py5Qn2x/fU0FnUAlokT/98R95paX1zvczOD/Z62srJZXDTP5t/n4qfWn6usnfem7w8gkNxSIttsXQx6LzAxzdkvaQ2sGzbXk5BRgmemfQKZc7skoM36OYdI+/4og/Nf1QmJbsR1+t7xyAynWVsXUrX0Ox4ZBL2+RX+dq3RkuBolfNyX4JZ+0GKMrc57fZ/PW8t3xepuEqlH3kkt79YLl+2/dd2B58pFmJEOtVHX1DaoK6uHorxKKpcBYI5cfTQyUnCjBw+oWpfn+wFeDDyU+nwCAfg8rE8hAPjhVMUyfYhz2AdRtZcRFKqPtRsBkzqw7kI6tmut3XdT1Qs4R+FI+DR79i+IEHBR2wzr3zEIKY4lQhY5ZTyu1qcFZPm9ymQA8zDwhhZrOhmJrHK+Ige2ee0WMzhRYn393YTYzfB7a3XVlZYEgQAAIABJREFUv1lOEB8DQ5Q939a6VUbdj1D1qcmZfz2zbc7x1ZXeuz/P2Yj7bSKrtwZCFk3MqhERA4uRDsWtA+v6wn+x/vyGEK/V973H8KsjnseR1ehXbB+socq53oo46kJIDtqJzLdNqoi3g6p4Ol93IO7QjRaeQlKAU+1+NlUu3yOuD3d+/SzdTjUdh4vYhxLBfu+ZwqD/i9c678Bz4ULWda2yk/ohuHt7mPYTeOQ86LoMyqWw/Duwamu9v2KYDe4H4ZrasycJ8YLHtXX/m9mp3CwCgWVkWXcmdJHLGCRWuMvGNopqrD73kboCmUk7UMiAeDWDRYWd1uerEYf2k9SvH6e5ugyJYnLYpiZCQjtYfxwJnpfWwCMDTCGsuHqBf2qxXiMRB+oAxPvejbipGWlcrZBNJwKy7tzpfa8D3aOQuO72IfbNcsRxXE+V4q7Xt5AqUG8lLl6BgPk8qpZi3scP0tpk/HOpnsn2fS/wwVTmCavf06a/wfqU+/qYjXcWgSTc2Xw1cpT+JeL25jE4TXs7ERcyP5+PjAhutvXJyHpO6uMKpB8chwi3zDl6uCRHJo4YPP6hz6H7o21AmJE7UZStH5tITD6coU4BHhwH5dvQfhG0XwblAuj4I1wzTnX7XLUyYpoKvGhdw7b1/VrnHXguXCiu7BQ7XA5EuoDGfjDtYhQaqRf6CxS/Ou35BCj/LA4sH/h8OcB5jKq1kouAFqX7VXagP0+IkK6iikDdAi+LqNoRlTvWvjkQiRZ+xGAOon4QFyL9kJuhv8rauJ+Q8f8ScUWTGQyc5iGK10P1PEkYEnQSQNbNsk+w+z2JCAQnUU3r7gj5EUJUlxGtXz1pfjM13EjXvmls7oC5t7U53/pwmdWxiMF+RX9gsA/Vy2xucn+eIhTtx1u5fuRkeoQ938+eb4A43DpizMYj2dIsj6vb2plLGHDkawnSMW1ibZ1kz9+NxLRueegGNwUhfCdKnkBETj0Nezva5ycRiRxPJAwRxlp7I62uOQzuW76uR3q0NyAuainSXX2NqlVpH+HX5d/OJdLZFJvjawmCqEk10aB/t9CeZyKgfla794E5E+x8d6YzXwwO+Lk/QG3mNXQr0RuAzdc1bFvfr3XegefShazQDsEc+X4Id3RBY23hSQagdED5TsR6ayX2eQwBWY8mUQjFbUY0UxB3U0cqD9g7Nyu+DCGjJtU4a3VKbmlq53Gru4lk+i7Sc2SwKzKW6EUUbxsRX7CBlN5LDPjsS1iqnYQA1/wWY19k8zmDyNQ7FMByE+oMrO9EupYvo/hv+1qfL0K6sjuJWHgFifs8uZ4DpLrl2FUICU1GQG6BzWuOEblti756SKg6EfIU1SCwR9pY7rTxn2N1fbPWj5dRDTf1ZwKo9toYFyCx75MI4L8ORdo+GhENc1M/ncAZbo7dpPpxxG3WQ/P02fNrCC51O8SZDCDO5lfWVsPGsQmRnXcrK+v96UYA20V2vyQIgLkMbTnXsPVdgXRj7nTbjyxUF9bKTyYsMjOR008QSO7fOJfWIt1+oO+rsLjDzvXazn0nNH8c+aNuRnt0o3UNy54r1zrvwPp0IaX/AwjAfmm4stfAz+uU09quDihHViOe168OQhfjh2gh0hksrZV16v8kRGW6QcByhOxyGz2ErqnHyj1CldNxK7v9kF+J92ccMiHO9d1q8+VpRxqEOGO1AYK7ELJqMpi6bYV48phLrb0VSJy0oQGiaYRDdSfV9OlzbXyeQK4QUbQ9A+0yIvqAWxpeT4T78XociE0h+f5Ymd1pbUyRuTQf8ywGp0J/pa2pj/3y9O6tRCinfqSH6SPCBn2TalT5DsSxdNXq2ZmwQHunzdVvEZL4NYO59cwx5DVpdfUSxMJj1sdrCOQ83ebZswiPIfypfN32QQjV9/JrENJzaz9va42twRoiOorHz8vI427CbWEAGYI8QOu99zA6V07Q+dmYPkT5AjS+DCs7nuG574TmQ3DsuoZvz8VrnXdgfbqQsvTnay37N8T564CyV1VflK98UOrvbrdD/EWkUL57iDoGCOrWHSaXIpPo39j9O9KYH0IU/tQEPBpE9tqGHWYX7wwll3egVRDgvxpxCL1EMNwnEGAfbfezrE8D9v+hhAFIzslTb+cxBKCPTQDsU1SDsD6MBZlNY92KAMqHWL+uQFTt4+m7dxCpNWbZt3sgn6WHqepRPDTOPxHIaSHiLDup6ro6EOI+DunbRhLA8ASkL3L/ndsQwJ5r7Z7u/SHEkVvYsxOpisvuRhyUA/APIf1NncjJOkvvx21I3OhR8o+08XZZm+9D+6Wf8Em7CCH4us6u1eVtziMQrjsd5xBVd1qf/pDWz2NUHoDOgK/lGQSX3qrNPnQmDkWIfDWhx8xrM5xRQ2NvaDpy+hOKMr4RlHH2/29QGvYboewPZVMoO8fZf17GynvWYfK67sD6dNkh+8xay/6NkdLPH/rw+gHuTPfud5TLtCMgPR9Rkr+kKvKbgTz+b0WA13Uhzi0MIDGNA5RW5tNDHdZfIBHNLQSibQD32Bx+0Z59D4kJC6K0/zXN8+gEyH5t9fWkdh+xcuPs/t8JjqXu81O/7kOhf16E9BUFiZh2sH7MRCbVzvVdkfq1B5ELqs5NuP7pz8iHZYLV9YZU/h4UX9EjZAzYWHuRCPJLKD3Dilq9ua2LqeanmkuE1PkiAqTTkI7qkwTi2AI5pM4fZm7cSnQ24nwnYFlqrY4c9+8p5Gx9j7W5unZWjiOQ2kGIIOnDTNQJ45Smre0lCEHc2GL9Wom9M8JaQhgG5X08ESHXHyFLQj8nDUQIeBqWh2ytOmttLKLKBQ9rsTfBzu9PqeZsakK5H8pHUfr1u6GciVJkJAT1vIw2/qzD5HXdgfXlInL59NiB/DISK6w2gHN8KYUC28w0YHoalB1RMrH/hjIZyh5QNoNyeEJKp0N5E5QjoWwJ5Rsoz8sQB6G/xf8uF/f7HyNRzRTr8zQiAGYrhOee+hlQrbTnU5BIaSckNnMA4HHsjmsBUJ4ikhTmdk9HSOEHqe2n8+ekuX43AmauSHev/oPT/U+RqO0kAxw/tLp2Q5zuUGbfHkWjEEjPw+XMJfxonPuaZvd7EXEG83gdCe9fG0MXosr7rF85v9JmBDLwZJSP1b7/fou2HFCfC4y3ci6Cepfdv86erSEiUPh46+lK6sD4R0iE20SRNJoImeekjv32LDvO+nqOrI1hbK3vB6Ao6m4Y0I72WT+DQ2P5WA+wunYnENq+SP92ANprbrmY/fQ8HFGrPdBpa/2w3V+KCKsJ6X028MiGEy3Fmp6vaRXimi5K53uo67oqgirleZiv6VmHy+u6A+vThZSZn7H/90cU9QiUZnsJ8P4CRz9pVNfnbdNeg7L1HghlCZT5tqEnJQQ1EsqvoPRDeQqaHxjegdcdDoej6JamA3g+AuhTEaX4aYI6d7GOh0/6MlXTcbeserMdzqMQUD0HmQ67zmMLwprQOQCnoN2fJwMgD63TV5vjzQhurwtxdwMI8Rxvdbj4ZiXwYQbHuVuAxFpbEUYabmDiCPh4ZCzgCvdeAmF5FPO6I26xOTwIia/uQ6I3B4yXIuOIH6U6j0UcWh9m5IAQ1wJkMn92qv8f0V5y0Ws7Euu12xx/gapy34FlD1XjBhe9utj0ACIkUCFSajQRkn+F9cNFac4pPUBwQkttrpuI+/acRjltylkkJEWkDcnm3IXgzPPcF0KvdK316xKr++1ov9yDCJUuYEtr4x9TvTMJwsJ1USuIYLYH2LcXI6nCHVbmIVq7Ffjau7Ny3XXi6esY6OyC5kQ7y/1/HYJ63mW8fbYvZ+1f+AFtbW2TgLNLKX9o8e4XaJNtPRs+vguSpexg77dCLM3Bdn8QIgOPRALy4xBJ579XQ3lIh7T+60XGAG1IDr8Bou7PAzZCh3C0lV2CxGDXICrzHSiszcUoGsW/IKRyHdJDjEZApQsprdusvRvs2/kIeOyKgEZBXMUj1t6LkQ4CBICvtPZeiUzC/VeIcD+jkHjmbqQU397er0GIFRRTcHMrvxDFPxuDAs5ukeqdTuQR8t/HkDhvERL5vByJ3Ta0dkAAbCJCZtsjhOJzX2r/9yEAvgkCfrfa30Ns/rxsHwL6A3a/N0I+rtO6wvqE1fWRWjtLkPPrtmiufS3G2pzln/exgRDVPMQVfohq/z2SxjzEIY0hDEb8txUS3Y1Lz1xnWBBS70Zr/RASlW2Aot07Qpppzw8ENrU52KBFn+9ECBTr68b2/BxiDfcH/s7+PxfN/b/Z/T32fgRhSOHrugTt+4PR+p9JxIW8G+3ZfVBIrhHo7IAIk2uQ5eH+aO+PQgYl49Ea+Pl6+nc2ND4GI89GFNzi9O5N1pg7M77Fnl+PUnzPrlZ1FqUcUq//hV/r3wsIKv0ygmpra/tHZGjw9+hQjAYuLLD5bHjPLgia+qncEUGr/e3+44hs/Q5CUKcgSO+/l0CZ1xpBQQCdedbMjvZ8vjX5EgQgN7TnnvdoW3TY5iOguLW977P+L7L6tqc1QPHgq1sTgMfx6pYIibgYZAPCbHkLBAQ6EDDfBAHaDan+GkgsNY4Q/0DkFxphz/9/e2ceZldVpf3fyUASAwKGBAjGIKKooK0tIH4OiO1A2zh8Lbatoig9iEKLirSAfg4IrSiT04eCDRgZRaYwgygdUGSMYAISIAkJSUggSZGi5nvv7j/etdj7njr3ViVEqkL2+zz7qVtn3Gefs9faa56cXL8PEVrsf3cGABGZiYjoTEE2JcdU60f5GT0IuAeN4zjiWLpjxhZE9VWrdwQaS6+mDCJwzmiW27at0biNbXMdf54uRGRfSKzVNY6Ym25L2zeIgCIJ4Gn7PR6pbB9HjGYtcaz9+3H0oDEs0Dh47r7e5BlA34wfvzV6V2n/e4mqxxdbH1yyeZE9y1j0DscgFZ3396V2f3+GCcS1XwrPlvIi4rsNdt3Fdu2d0TvptOf0cff36t/IS4mqbx8vv96gd34d9LwHJl2L9J+9DJ5AZRrQgkFdSQjvJ2NYGDP0IZstzkfqhhkhhK1RypcCEeT1RvmL37o14etLDp+BJlkNTa5JaIL2IOJxCpr4E+y4J5Gt42VofixAuvwCTd6ZSKoai7z6Xmn38y7uTGQun7XjjrX7bYk85d6O5ubbEYGeguwV81HCy5m27QuIOGHnY9fbHn13S9CYvgUJn07kJyHVz2tDCGMR470deZaNQXTgTBRQOgFJLtOIrsnpc3Ql172QWOZiEmIa41BqqAnIDXsCUvM6sdoKOSZ4ZeL7iGggwra93a+OJI4CSaGTEBGcYs9dS85zj8knkeR6MHoP26HvqxeYGUIYj4joCxAT2cruORepryC6W0+zsemxaz0WQtgROZhsg97nLfZsAXgdYlpXhhDGoO/hVvTuQYuZr9q4XIze5UTiYoRkbBegDPdTQwi7oHd7gT3fFHsvH7X7Tkbqt6lI1XmN9fm/bNs2NDOnxTYup9rY7oZKtoyxPk628fq6ffddiNHtbOO+jpjJ5AkbX1+4jUVzImVOPh+asMoWSW9CH8kVVQcNDxtEPzZbjLSOcTQ1mm1Qq4CD7ffe9v+5qQ0q1UPvBOF3yf8fh/DtxAb15mRfNzQ+0N4TrZypYB7KkdaLJrTH6KykOju3q0O6iW7RdURkViFCvTfSPNaTbanjgSdZDXa/GTYWXpG3gRjT9+wajyNCdTLRbflOzPsLqdfSqrudDDZIu93q43bOwXbMdPv/CqI7cECalUXJdfqJ2QNOS96rG8DPQwvgtBRJH4qHmYHoj+cj9Oq0NWRgd22Dj9F3kd1nMa3tiX3WxzPQ4joQvR23IgZFe9oej2ObipwhLqM5LqwDSfT3+Pu0axXEXHA3l+4/n2j7WWfnfd7O+792fc+2/jMiIU/fzQDRHtqw/RPsHFcDu5fjYWh+eHb8/ySGDjyejOO+RK+6W2j+9h609+zxfAcgjXkXmgeeiLfDjnGHlKo55Ul8A2Ls19lxNxI9Hhv2uyq8IwDhP6Hm5dhPpNmLrw5hLnKW+p393wPhGggvsd992Qa1YTR5pDswmhrNDOpApE7qNELyY8SgnvHi21AG1QONqa2JWoPo+rqoYv9TxMj0k5EOvkGsxJqe73E3KfF3dUxKgNK+LKParf2HNgYeH5V6rf1dco2nEfEej1btKRE9vIKI3E8s5pbe90/Wz/Ps3L9HUlD5/CfsXb0+Gb/jSu/1CSQRex+vsmveY2Pszh/LaXY2uBMZ5S+iuRyGt0XIvnMOMWjVx3Uh8ja8meag00Asonc2cuZwtaVnh3AX7QX2+4dIteeBuR1IqpxTes7riJlIJiAJqOxaPYBiz95g5zxo45MmIHYnmNMZXKG3C3hHcs+ApKoppTFegwUnE9WVHtz7NXuXZa/TVTQnL66VfqdZNVwd/YfkuO5k/5W2/+WIAXYRM72X55RnDKnTYl5Og66eZA6fC2EvCJOQR+7eyK28z+hA+fx9dV724ltfmjzSHdgk27OMg5rd2jtvtU2WNGDWve98lRrQCr+dF2AV0/sPtIJ2O5GnA9qRWETQvbbcQ6pVRL33o6tENHzfb5D0c7IRDc+E4Jnaa4jwHYAIvxOX9yNC9wjVq+HVSZ8mo0BWz7zhZU4GJW5FzKDLzvVM158iZjN/K7GWUPmeTyEm6rW2liDm9ggi7IuTvn7L7pdm2r6XyHgmWB+us/YI1d/CcuAYmgN1f2DXe4AYDvH9imd1DzqPfbsZSWTu9NLBYAnd39tV1kcPN/B36WaXuvWtbte4ws79IIrBuj+5ln+fVaUzys0ll5WIKXsWi8uJ38MKYv2wuUjaG+7379/XI/a7n7hwS1Nn9SDHjqpwjfps6K2bFLUBLcdBbUAb8Q5sku1ZZJJoQNe7FHPVivC7+skn4y9oVq+sz6SsamfZPerIW8/jU3ZGq9qAiO5UpJ9/KDn3IsQoU+bYgdQlnt+ulTuvqxzrRBVgPzH+xvMBpnE8A0g6OQ2NWRoXthZlOJ9J9ECrYQXynvnARTxdqjgIJaNdXNE3v/cdRqhSifNhJI30ItvatxLC9jiSxB5K7ukZ2c+juUjeJ5Bd8xEkjXps2gCSXFcQ1arpODizuBupZT2XXQfw6uS+k5Pjgx37ehRLlqrAOq0fpxJjg9L7NNBioIcYWJu+s16qv0V3Ye9DquhbaGbAPYgJ3pxsuxHZd/ZMjv0XJDHXkET7j7Z9BWIgz+b7T/v9uL0jD9J1p5zKcz8Jp27ovA85k8QGtRHvwCbb4NAN+Fi7gmJd3IU8nSxrk9+eidn/v4PmOJ9WDMDbcKQrv3c/CoL9DdFL7mnErOpoVfwpIgH35il5akZ47rDJ7czPA2afRqvgfVBszQDyxk3ja6qIh7cBIyQeG/YHJEUEmonfQmJm9TVG2FYy+Fkft7GfTTNjmoUYi7tS72r9dmmzN7lWP3LseBuRoDcVnENZI5xgvxJJainTWWvPPxd9D8dgalPkeDEnObYbMZK5xOBcf+cBSR2zidLlnGS8/PyGPbdLOStonRey/B56kDv2/XbOhxDjc4bo6rtgffPUWbVk3BYn1+6zfR+x6/XZeNbtPdbRQmpWRV/aNX+Wqvi28nELiCp0D0PoTPaneQprwInPdt6POM3aBNuId2CTbvFjHUrdVy9/pMgon5aPcNvALcm2n1CdAbwdQfftTgyrjimnMUp/p6q6y5Da7czkeq77v9P67BJHeq0rkROGB6S6reyq5Jh+orqqBuxm47Kv7T+XyIDWEAMu70QSTbt8b2lbS1x1n4Dikd6KGIaPRR34QnL/WvKeXkcksv2ISLskODfp475NE0sMJyBGvpAYVOrM1ZnEtna8Z1IorP0ieX8NRMDfglSnV1tf7qG1urjcupBUeBsKX6gn565CDHAAy+yOmJCf9yhxYdJAtrN3EGuYnYiknD+W7tmBGMEqu2eNyKi6kf3tQKJ6tqxWrrf4XW7lzCHDmSe+7XdEjYY70lxeOmchsMXGmPe5rV8b8Q5s8g32DHBJkAG0u/SBdtv2S0KFeI+yCNRQfaWjbIKtRi65Xpm3XQJLJzDz7e86JCG4FOPnt5rY5ya/VxJX515VtcqR4hHiKte3fwOpaY4jel81SvduIPtFDXi9Pf+rid6AvWhVvhy43va7d1r6vGXvvzVInXQikk6d2KT1gryl6XLSfjlT7kFMow+pnlwF595eTxO9Gfcn2jScifwbksJOL93jRkzCQrYgJ/ae2WEJ0cllYTJuVeomlz7KCwwfkxOQvfGHSJWWXmMA2dTSbSck3+MyxHQ+Zde7wcbMmaar2sqlLPx5PIHx7si2eVZp7OcRa02l77CB3v+iZDy6kmv/ksHjUGY07tXnsViBaN9sxdzSb2QOyubRh+bB+9BipgHstTHnfW7DbyPegedNg6kBvhxgVoDZ9vfLoY3Xjk36XZL/JyDVlBvXv9pmYj7lE28a9B4FjQugdzaEc6H2K7hsB632h1INnkSz59RaYrLTYETmNUgFmKYSKjOfJ2gurNhHTIHkRKIBPJ4872VIGhprxM2PeZjBaWdSgu6qxCvsHk5A3Z5VI7pSf4XBLsgDRrxWIinoEMRU0pxszgy86mqZ0blH3xPW3/Ixnn/OiwqmBSTT65TrR7n95ziUJWM1WjCUHUca1ocD7DlnE1MYubPGTPv/dKJ04tevGtuFybV/ZmPaC3w2eWfOHFN1pS9sfAwW0ex8EEr7u5Cksh2Sir0Pi2mWgu4iFrZsxZz+DKx9NZxwDHTPgnA9dJ0HA0dBY2q855MtrlFD6b/cCeN4oit80/zcWPM+t+G3Ee9AbhUvRfYPrwfUalI9ti/MuxMW9DC4qmcXhF6oXwVd+4kIt0quGoyILipNfM8k4NtWoFV5F1L5fRrp7dvp+busr79GNgwnFgNIKklVOinx/i2Kp7nd+r0Vkgz/jJiuM0gva/9LxGyeRuq72ZjTAgpgviu59nwkrV5Fs2roSZrVS1cg5vajZCzmJcesj23ExyMghuo1x1Yi9ecXifYzfx+prWkdIuhfpTnp7yXElFW9yO6VFg68l1hp2EuXnG9j+luUmqmG3OGPptkLz5/PHXZWJNt6kcR3edK/VuPRh4J2D7BnfAjZN1MGtogohaXvaahQjNP2gY5r4OmqyrZdEPqg9ntYvHdzDF7V9a6mVPMrt5FvI96B3Fq8GBGd8krXV58Dn0G1pWqlSVluNU3UxhG6VisPu6qVfUqEaygeKDUk9xrBucf6OZXonbUIBfAuItrX2jHHS43YrUZBrXViItE9bDx2Tvo2AxHmHyV9X02Mu3F7zq/sWksRQ3I39waSGNoRLX9+X3nfatebTbS51ZEK8oN2X7d1HYekwsOI6teAmMuv7dncFrWmzf29D08R7ZWu+j2TqK67yMZ4rPVjv+QdrkNu4wXR1bwnOfYQu8el9jxfJKptW7lcl7+TOUjycLXc2ciBoo6+mYeS97TC+nta6RmrvpNW9+4E5hwGvcOdA09D+GxzLFXa5jyXczu39aCDI92Bza0xzKq9yEMsEFfAPpHXHgt/GU4130chTG6eoGkGhbT1GBFJiUSVJ1QXcvteggjzLsCRtu8BBpcH77HrnYrcvQNyH/4qzdJUSqjWEDOwX4fUnpORxOUxYT9HLtQuhZVd06sYrjPWBTTHLtWNoO5HrO30MWR/SZmXZ7Bw5wL/38tg/D/b9x37+w7b7tWOXS3ov9MA2l5kDzzW9u2CSpJclxxTQypDLxNRVVq+v/TsbmOZi6TG/yG6it+EmNa2yPYUkIqvHCPlTKgT2dhORMz9YGIsUqqW81ipJXbux9HiZTWS2m6oeOd+nj9nHUl/VYuHPqDfF2hDzYG0dUP98OZxd5tmDdhmpGlDbhV0cKQ7sLk1hlm1F63A9wLehTydDgfu2Ac6POVKuc1EKf5bTdCnIbyhekUajIDcTrPUtgp5FT7S4hwnlF6u4TgkYTyK9PpO3D6CVG81e7atjDB57JSX1niyBVGqas6Y/mL9Dkit5QS6B3i6NKY1onv8vXa/BlrV74dUZJ6a6f8QvQ6dMLoNLNiYuBfit237oXbu+Yho/3+iE4hLYjVisLI/x9mIKXmm7h2BzxBrQf2ZaBd8HFiePNPWNKtKe5Gqro4kmyXJfcoLjqpWQ4zf7W81xNjcBf41RG/DbxK/k/dYH45DCxKvq9bT4j4NpDJdkLxzd6S5rc15Yc8NYE7euqCxn97HISjV037A7iNNF3JrQQdHugObW6NN1V5KBeGqWidc0yqafSgGVUOF1toQpwEjDjfRXNCtQZRAVpWIRzcxH9oBxBLuPzfCWnaS6C9dN43peYgo7Sws7fu9EcrUljaA7Dh3E1fxtyFm6HFkTyL1lUt3j2KGb0SoPKtDA0mr64g2Ffc6PJjoYt5h/biY5uq1PUjt9hWkwkqf+yEbj8VI5bcGMbuD7Bi39a0kxhHVbCyXYHkFkQu8e24+guxkPp4X2X5Pm7QGMeHVNDP9FfZ+76O57w8SvfyW0jzGKXNLpXnffjWSrAPyAPyc/b6PZmkpHdtyCqaHkLdjO9trAAYuh9pQar1WrZEzOmxSbcQ7sDk1BlftPR95WF1jE/adSKV1khGmlSjj96QQAgGmXQF9f4Oq9r4Jwr028Q6CUECYiNR6J0JYZJPacwbui6r5jmvPpJyIX00k8p79wZnWSgZ7aZXbakTABxCDcMJ2K2IWZ6BV+Bpkl5lr+7uQfWlXYjmEssv6eYioP4qYTHl/nWYpxQntEuCPyfu4w97JL2m2z92J1IrziS7vOyXHuM1pPM3pl8qtZvdwr7BTbfu9xIrFyxBxfyXNjGG5fQeP0Zz4dg1iJikR72awo4LHl7l332+JSWm9b3Ub40UoU3la1fYCJN3cbve9F6mmPR/gantvwfozVDxWWk6+B0l3non9SWIC5Lbf5gvhCc+J998QXomStL4bwuKEEYFhoXIiAAAVF0lEQVSqXO9qc+VzYk6+v+fFsrU9gL7r+4G/teecjpxPnrBx+Xwy9nsjh5t1aA6cMtI05fneRrwDm1ujOSHtOWjF+mZUPmCiEbHZxJo3VwLfCSFwAZw6FcIfkTR0DpKaem3izaRZgqpiULtA+NdqNVor1ZpLS53ErA4DyAZ0GWI8XycyoJYZoW2fE/MTkCNFD2LKJxKlrFvs2DsQEzvWzulD6jzPZu2E1l3KT7LzbiNmswgM9mDsJAarBiOO30VE9ixEtD0f4Z527Sl2rwVEpuDXe9D67C7yA0gaSN3u3YX9EWBV8j0cR4xtegxJ2A8jtZ+rW/usv2Vbz1PJ734Uw9RAJTzGosVNSPrQSczp2ECBvy9CUvHjdl93I38fUZ23PzGA3G1fVczEnzMgAv8123aCjYerK/+SHN9SlVfR+j8N9S4Il0N4GYT77fv+NlqwpQzqHyCsRbbY7SBca/suhL4tNXZ7oe9rV2I9Mk8ntQVSuy4E3mPv6jbgE/Z7S2CfkaYnz/c24h3Y3BqDGdSsZF+BVpovS7a9CVgUQuD98ODXkkkYILyCWFp+OAzqeAizWq9Uu43QlWNG0jQ2a4hpjBbSzAg2pHlwcUCrUneQaCCi7JKaEz7vY5mp9tHsfbiSmMFhNVKvldVKZeLqDLmDZmeRMhFNn3e53WchzQ4Fvq0juX5ZyutJnqEbEc3huLF7zJKPxUKabYfuVp7ey+/hQdweE7WS5mzrS4nS2KNUe70Np7mNsJfolNKFpBIfh7KzxDKaY/AGMahZ9i3vD+HnybdeR5nFFycM6pZk/4chfMd+vxvCkXBHxdx8I7CktO0Y4Gz7PQflYdxupOnI5tJywcKRx9Lk91RUnO7uoig6iqLoQPaRqQCrYJuTUUU3b0tpLns6FLanZbHEBpLgxiMG9VCyzyvjjtXp3Gz/T0LEoKo6b7v/U2xl3XoSqZG2Q0RsFSrA90KknjoXPapX1z0P2XwWJX0cTzTg34vS2HSgofIVMsj5YTaxiKBLVE7ct0yeaSwaFxAB/Q0xW0W39X2R9bNAkobXzHrKtrs0djkicp1Eidn7NAm9e68y+zAi7ti2C4hOGZOR2/0kxERuJH4GwcbC39kfbJyutvuNQ6rLs5A0M83aX6y/26P3249iyMpVkdshfc+FPdsAeqf+LDsTq9zW7a97FTaorqTrGP+03cO9cHwevMhuviw5eIfkdzqwS4Hdq7/JmcB0n3s2/45FYwJKYvsK4C9FUdxZFMUBbfqasTEw0hxyc2sMlqCOT/aNQURvp6pzD4AFxyerwnLbmaElqDMhzGqxOkUr6VbeXqmBfCgpxFfGa0v7h+Oh5xkYBhBz8QwDnoLmc0jyWEbMkuGlwrdFMT8DpesFLFee9elo+30Hsj+sKh3vdhb3oHRvOpeo/FldNebb90GE+YjkmKU0xzql5y9C6rhv2P8LaR73R5CqzJ0krifahL5HlFL8+KdQOqI9ifFfLpmss7+elb3DxrOPWK69bNtq1cohCK6yG05ewFQNWNU8rKKqH/XdLP/du1FNplZzAQgPJf8fDOGrQ0tQbyLJSt9mDo+x76IXmDzSNOX53LIENYoQQmig1e6pRVFMAyiKYqeiKN4D8BG49nQI7lPdhZbFnXb+9ojCtUM/hPuqV4/j0Uq6IMbTgIjwqUQi7VnAq+CSmUsfW9lfJ2hDfW8Ddsxudq+t7Zou/RyGiMhYZMyuIwllJ7vH48gO5Wqyq4kS1o1FUVyIiOiXiqJ4AtkgdkO2opV2fh2Val+LpI/tkY1sKlL3jCW6lt+IbFPBtnuG+tOS8dgGOWGcadsaKAHv/vZc7iTRQNKF55PzAN/tgCOKoliKbEahKIobgC8Tiy567adJyJvvduQN9xJUmXcqehdbImHi+zZuU9B7n2B/X568h3ZwCajH/u5qfydVHJuiCznMjLf/ryd+vu7UsTWS2qqkfLpRhdpDUcDZfNv+FHKrHA4Ogf4z4BVFUbyhEHYtimImWrB0FkXxlaIoJhVFMbYoij2KotgLoCiKg4qimGrz1Eu3N4Z524wNwUhzyM2t0UaCsm0Tgf9CvGYdUuvIkwimXQ19eyLPpB0gHIjKTgdkOJ5h+77fQoL6KdSmtrYZ+Wq8H3nVPdjiuHbSU31HWHwsdM2B+ddD1yXQcRwsmzq4jMFwrtdJ9OTzfctRAb9uYjYIt+f8xsbx18gu8xmqKxPfi6Snq+z41SjYtiB6tznRvB45caxJtqX9vA+l73FX/CXIgaQcP+aM80FkiF9k/W7Y/VZZX24C/mC/l6B4scUVz3ATcuuu2XMWNjbudOF/lyM399RZxGOq1sdJIaBvsmcHWPw9ePBcqN8DD18Ka46G3iT33XDj2Wp2zY5S36okqNpr4N8bSsQaZkHYA8JWEF4M4dPDlKAC9OwkBu+l5ecRExhPR+rUx9Ei5Y/AO23fuURpfT7mzZnbX5FejnQHclvP9iyq+dp5l9gkXF4iAk8gm81NNKtq1iLX2oGEQFcylL1g4ErorcqL1g8DvVC/DjreK7f6QDURcmbkTDTNONFOxdhJzIjQSVQjueddA/hXIzLp9RtEFdzFxDpMHtDr168nx3zRiKqroi5HzMjTDtVRrJRn964hteHniemQ0mv3E21gZyGpqJfmhLn+7J9AKZd6aHbN7iIyu8doz3i8Om76XDUbkypHEveuvP4AuGw29Nr7rKXvuAtJN5dBbc/W9+63vnXZ+KTf2hok9f6+9Ow1FJ4wcWPNgRGfx7kNq414B3Jbz/YsqvmGpKonckBYQczm/AOiW/RylBj2EoZnd+r+PHR0QWO4uQGPj7WDqpjUhcTkpC4hLSWqHt1ucgRySmiVEseJ33BX8ylB/p3d/xCabUrL7d7zkarr+8m+r9u2y5Pr/CNSQXYgZt9BtB3tac/aQ2Rm69NPl+Q8BKCKGVyMJHJP0eTu5r5Acab2AFJxVY1FAH59LDzYxfBy33VD49DBNrL0m/Hf61B14fOTsVmAkteebeffhscCbsQ5kNvobyPegdw2oG2kqp7IRvETLJDUts1E6qA088IlxFIMg9oRsLqrRXaLVi1J3tmK+C5FUogTUHdTPpvoZeZE2FMJdaKcc6s2gNgP1VyCLP/fQF59XoG4YYzgB8T8hl4DyYnzcFM6DfcZ/Lh7kZqyZve+P9nnZd49Ca/HO3Ugz8R2SWt7NyT3XRc0PtP6mncjRu4hDYtRwtmtkm9xO5Qu6gV/rTmQ2+huI96B3DawPQdVPVF2g3MZ7I33DGHcG3o2NC/aELkBvbmKMbVRdCNHhMOIcVs30N47rIqoe4kHJ9atGOYAkpi8hEir2kLtWh3ZNTxZ6/w2xzaScxrJ3+EyqjU2Hj3I7vQhYsVcP3YlURJte931yX13NoQ3D/8dL0PlRqaM1jmQ28i2Ee9Abs+iPYdVPWlOk/MM4b4GOjc0L1oNwiXrJ+m4XSaVPjxlTwNJVp4NoRWTCAytdmrXUrXaJSVmdQ+yM6UpixaikhOe/aKd1DhUn9yuVHUNH4NbiXWdyhkfBpBKrUod2Eqi67oKuof7jssMCgg/rh7b05719/8cz4HcnvvmOcIyNmUUxVSU0PS1yK25A3mW/YIQntg4tygWoNQvnudu/A7Quwh2mtj+1LboBV4CtScGB/uCiOlEYpbvgmiXGkd0V3biPwlJVy8Y4rY1ZCtyZ4HJtm18cozfL/3/AeBV1gd3me9ApUQ+Avy79aubWIPpB7ZtTHLfO1FetxTl+7mbf92e94X2eyytEezYCdaHicnxHig7we7j9/MMHFsxGL3TIDwKk4b7js9B+uFb7f8Cuci9A55cpXGbYn06P4Tw8WFedmg8B3MgYwQw0hwyt9HfgL9FdOZpZHS/CDg+wFGnQ//LIGwL4X0QliUr2N8j9dAL7e/vk30zUVBxF4SjhrbH9CGm+EyZ+6SVpalW11iDpJ2L94BvHA09t8D8P8MD50HtHLjhpSqF0kX76sPH2LX6iZVgXdqZg/IqLqC574vtuIeJCWOrWicKer3ttXD0t2DF9bD4fph3MXT+GG5/uVRiHsQ8lJRXJRl5tvRWatsmCW4ChDcSkxIHlDJoFwhbQngVhEtbSFBvteu8AMJ4jcNHUHmLFcihZJX9/vRIf+O5jc424h3IbXQ3JL14ZpnxyCutHzj+XLhhCoS7UcLaw40oBQirUZbpWSgO63z7/8kSgwq0zGwRkKrMc8O5vWgFUq39zoj/APIInNeG0A4A9T0hXAKhyg2+Gxq9UL8WnnqjGFS5+GLaemy/x1/NpTkbheexm0csmb6K9jkLVwMP7ANrfwsre6FRVcK8255hr2ZX+UGMBblqz7ff65BDwvVEZwhXjfYRvTcHXWcW1M+hOSnxr9BCpA7hQmNAyysYVLDrWDzSLPue3m73Ps6+p/ciaW/bkf7Wcxt9bcQ7kNvobqiy7zKaPf1uBY5/Pyw5KiFGnaiUxyJjOnuVCOw+RsDKDOqq9oTbCXEPzfE7TmRbZdZOzw+HQn19yoO38T5r19IkrOtzXgO750buY4PmzO8+Zt1E54h2fV19DfQFmpMSl9vfoCDxIRjUbPt23m7vclzyPa0iZwbPraLlVEcZQ2E6sCyEEJJtSwGWwdiZycYtkYFhGQqySfdBzOtTRkfFtgQFsuFMRPYTd9teZKeORUyqv+LcADz6FVh0MoyZTHsDzmK70QSkRztU51ehl5gV/EGklutGdqbx1uenkAS12PrhqaIq8TnoPxkZw9r1EWLwVQfwBaktq+AqwDGIASxCkhTIVjfBflemFDJseyBsUU5KPAt4HTFR6zxad6LUH8fqEEIt+b8bfT4ZGU2oMkxnZKRYAexUFEWRMKkZwCNbwLJFyt83DkSBV6Oo1OlIx5ViCUpAByLEFjXai+KI3s3g79HTHK2yv+OBl9ktXkR0mnAnhDKKybDzjsR05MPFZMSk7oa+OyMxd6SX2w2esR/dhhjSHig/3R6lS1ahvjWEk2C8J7J7OUr0989D9HEccDxsd4v6WcY2iIGOQYm9PZ3SA8izcDzwd4hxeI6+FF3AIQth92lwVGF59h4F/g2lG/GkiK+jNSeHZ97xfUM8TkbGIGQJKmMoePG/w4uiGFcUxQcwD7Qe+N4vYNyfkCHjWFRQZ2dkWFiA0gPUkFfF/agmPIioXWj7zpDjwFOIcHpgbkAecLcj4vhmlF385YgxLUYBuvehmJ4qdHweGqehOgnriwlQHNt6EbcUSU5Lrc9bIZr9UVTobwJiWosRU1hXeRUYU8C4sSbJrECeFG8bZh8nAl9tnbx3KXJ9v8suPRExzb9Hji8g3vYfyEYFutYPkRPFo5PhJ11QeFLiLtTRqXbw2UiCagVLYDwG+MUwHykj4xlkBpXRFiGEfuQY8S9otX0QqqfUNzeEXx8Gf/oQsCNKRnehnTfFDjrZfn/P/t/O9n/bjt8axs9RbrvzETP8OKLRJwKHIl53aAhhSghhHMp+cTixJtOOSIob1PVpsMXXYcxFiDrOQ4neXo98tmcA32zz7E8Bl8DYFpNkBgpkno4EiU5i+Y0nkYA4wfr7KrvlIDjH+LP9fwsSEacjr5QZduIbbF8ZY4H9YezW1X3cEfGILj0KHwO2CSFsEULYATGtU0IIZ6FX8gQqcX8EEpR+vBUseAlyHwd4NXAk4sTbW7/fXH1vAL4B4WPQKOChoij+qc2hGRmDkOOgMtYbRVHcDvw0hHA2KkVwM0PHHlWhG9iXEO56lv05GpUVL4hqv+XfhC2OgmkvQFziGMQppgC7I4b1LuBnKJBpMaqzMYDEpn9Aott0aPyo9WIuzfTgNibvx1h0Xqtz6++F+u6wxY6ISx9OrDt/LhJ1tkai5PetjxMRY33YjnkQwmugGIhOEOPR7zeEEOYONX5DYhS844zNE1mCyhgSRVHsWxTFDqbiOxgFQ14HQAh3okV193peths4ciMRrkdRBu9TkFSwfwhhxjfgBqeo05F/9duR/m0MeoiPooR/ZaxA6R/OREWjKlBDtpXUS84ZE0Rm0U/0mCtjzDshvAMFUIGkpH3t90GImY5DA9yHPDLKuBSKPaXOmwi8Dwk8jzN0XafhYXS844zNENlJImM42A2VeJiM7EQHhhBWPLM3hJ9SFCCNXlrGvApeUfVIQvjpxuhcCOECVMOHoigOJDKDbfyYZcir4naUJnse4hx9wIcrrrnUjt8WeIsKAvYiXjUOMaELgCvXo5vHItObZ6G4FnjjJ2HeeHjbwYiBziPan05CkcPL7YbrqPaWWwrsCv1/kDr2emsbFyP8jjM2T2QGlTEkQghnAGcMcdBPKYq7kCbtvYhJpBVWexCdvQb4znO0qu4A5RVahsrRfhCp0a5FVPYLVBP9GYhhdAAvhfkhhE8WRbEryrj9YeBHQZLFsFAoFc8pwOkoye3/AF1TLAXRdCStTdf9uAXZ7W5C6sgxiFlWiWEzgCsVUP3Xxeh8xxnPY2QGlbHxIIL0odGSF20pPDgX+r4IEw5Cqr1OJBlNRC6A5yP/9jJ2RPafQ6F+Ajy0a1GMB6aHEP65KIrDkUf9+uC/gXNDCOuKongjytV3AnKk6HkLTDoFeKcd3Ikm51SkJ/wurd0APwa934Rp5oRwKTJbzQgh/Gk9+zg0Rtk7znh+IzOojI0PEaiTRrAHVxZFUSsg7A3jv4TcAUHFhY5EUtS+wD/ROlD4l8iT7hVyw/4SSq80J4QwjLjUZoQQ+pBGESQ9vQll5HgMOO6tqGqfq/feg2LGXoH0ql9EklIVZgKvhQPvUi0oL+3+NaSa/Otg5N9xxmaA7MWX8fxGUVwKfIANcwhSOfcQPrRxO1XCptDHjIwRQGZQGc9vbAou0ptCHzMyRgDZzTzj+Y1NwUV6U+hjRsYIIEtQGZsHiuJQRruL9KbQx4yM5xCZQWVsPiiKPRntLtKbQh8zMp4jZAaVsflhU3CR3hT6mJHxV0ZmUBkZGRkZoxLZSSIjIyMjY1QiM6iMjIyMjFGJzKAyMjIyMkYlMoPKyMjIyBiVyAwqIyMjI2NUIjOojIyMjIxRicygMjIyMjJGJTKDysjIyMgYlcgMKiMjIyNjVCIzqIyMjIyMUYnMoDIyMjIyRiUyg8rIyMjIGJXIDCojIyMjY1QiM6iMjIyMjFGJzKAyMjIyMkYlMoPKyMjIyBiVyAwqIyMjI2NUIjOojIyMjIxRicygMjIyMjJGJf4XM5vyYOczrWoAAAAASUVORK5CYII=\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "_, _, _ = plot_structure(sm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The reason why we have a fully connected graph here is we haven't applied thresholding to the weaker edges. Thresholding can be applied either by specifying the value for the parameter `w_threshold` in `from_pandas`, or we can remove the edges by calling the structure model function, `remove_edges_below_threshold`." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOyddZgcVdaH398kgQRiSJBABF88EHSRsMiyaHAPtsAGdwm+i0twFhZ2IUGC24fL4u6wuCUQXEMUCMn5/ji3MzWdnunume6umpn7Pk89M90l91RV1z11zzn3HJkZkUgkEolkjbq0BYhEIpFIpBBRQUUikUgkk0QFFYlEIpFMEhVUJBKJRDJJVFCRSCQSySRRQUUikUgkk0QFFYlEIpFMEhVUJBKJRDJJVFCRSCQSySRRQUUikUgkk0QFFYlEIpFMEhVUJBKJRDJJVFCRSCQSySRRQUUikUgkk0QFFYlEIpFMEhVUJBKJRDJJVFCRSCQSySRRQUUikUgkk0QFFYlEIpFM0jFtASKRTCDNA+wGLAf0BMYBbwIjMPsuTdEa0FrkjEQqgMwsbRkikfSQVgaGARsBBnRJrJ0CCLgfOAOzl2ovYKC1yBmJVJCooCLtF2koMBzoTNPm7unAL8DhmF1eC9Ea0FrkjEQqTPRBRVotkpaQ9LqkCZIOamK7vpImSuoQPj/+V+lavNOfjeLPQV3YbnhQFjWju/TDQ3A+zZBT0uWSTqi6kJFIlYg+qEhr5ijgMTMb0NRGZvYZ0DX3uSd0WwW2A2Yps71c5/8yZi8X21jSycCiZrZLme3kDrDyHDBHnZvvmmQE8G/g6YScBoNKkTMSySpxBBVpzfQD3m7GTn3roFO5+xkwzc1sw8rdt5kMUwnKqRFqKWckUhWigoq0SiQ9CvwJuCSY7w6W9Jqk8ZLGhtFLbtv+kkxSR6R5esCcuY7/ZCA5vBmDr/g9fF4HOA5YAx+WjIa6cbBxN+l6SV9J+kLSqZKOCf9PlDRZ0iTgJGCn8N0bQZYxktZPyHaypOsSn4dI+rRO+vEU2Cz3/deh/R8Ssr4K9AL+BwwFnsOHiT19dd2uMHgu6bxw3HUkfS7pKEnfBtm3kLSxpA8k/Sjp2IQcdeGcPpb0g6SbJc1Zzj2KRFpKVFCR6iLNg3Qk0rVId4e/RyL1aslhzWxd4CngADPrCrwB7Ir3z5sA+0raosCuu5UbFnQtcAUwAR+y7QadloeFgUWBFYDNcXPjysDdwKXA8sBpwMNm1tXMli/WjqSlgMuAIT/B2T8An4d18+HK8uY8uXYAlgUuB1YHJuJx5zn+6OHoOebDR1YLACcCV+L6eSCwFnCCpIXCtgcCWwCDgN7AT+G8ilOlex5pf0QFFakO0spItwOfAn/HO8JNw9+/A58h3R7Cp1uMmT1uZv8zs+lm9iZwA9655rNcXZm/+92BpXGH7Y/AA9DhPvjEzCaZ2bfAVfjgZSl88NUDj6b7HShnbtI2wD1m9mQPWPo06JgUdDcgN9Sahp/gkCYOVgcd5oJ5El9NBU4zs6nAjcDcwIVmNsHM3gbewRUr+KDsODP73Mx+xQeb20hq3G9d43seafvEIIlI5SkeFp2bwzMY2BCpxWHRklYFzgSWwYMfZgVuKbBpz3KP3Sfx/6d4L98btpskbRK+rgO+xTvxZXBL3Obh+4/LaKo3MDYn5+zAXImVg3GtMRp4H9eCqxQ5YCe/Bzl+MLNp4f8p4e83ifVTqA8m6QfcIWl6Yv00YF7gi5kaSuGeR9o+cQQVKZngx1m0yEa5jqrksOh+8M8zpAtaKN4o4P+APmbWA7d6FQowSFrAmB2YnPj8dYEdkgfpg2u+H+BGM+sZlu5mtqCZrQn0BV4EHsbNfavkXbNJ+LXJMV/i/6+o14fjJtPQ59QZDz28DjfvJUdPjUVSTPWRXHMYC2yUOMeeZtbZzJpSTgXv+Rga+PUKhuwHP93CzZQ10kaJCipSOdx0k+uoSt8NNBD+hrRSC1rvBvxoZr9IWgXYqZHt3pzuE1oBGAA8CXwG/AycUaSR+YH1YdpG0F9S9xBMsL6kQyTNivu/hLeRcyEl3V6vAztI6iQ/320S624FNpW05gR4+3j4PTl8AXeyjcA1cVJBzRsa+y3x3XSY9oOP7HLMlwzQKMLlwGmS+gFI6iVp8ExbNeOerwP8u15JrQQQ/HSflHqMSPsgKqhIJRlGQ5NSycjNci0Ji94P+IekCXgAwM2NbDcyOdrYANgejyQYiDtMinEtTH0ZPsJ9Nj8BFwMHAN/j/qjBuOLZFfgEeFnSq2H3E4BFwn5/x0d+AAQ/0P7AqB5w1BzAgnltr4E/tCviNrgc6+J+svlwx1KOZz1PX1EK+JYuxPXgQ+GaPg+sWmDXZt9zYih8pBhmFpd2ugBH4/6ECbhbYz2gA3As7juZALyCm83ARwJDgQ9xU9mlhHRZ38C8f4epfcF6gQ0BGwdmYbkLbCmwHmCDwN5JrOsH9rD/P2UN+DPwMjAe94+cF9peB/g8T/4xwPrh/5PxEchNQe5XgeULnjvcbjDNEjKUsUwzuK0m96gROf8EdmWZcuJWwem4n2kiHnVowF/xAeSTYbtbcEvnz/jgcunw/arh+w65Yy4Buy8D0w1sGtgZYAuDzQm2LdgPQZbR3o5NBTsWrA5sVrDZwfaF382j5Q2f1Aw+SPwnnltwIvAMrnsvwBX7e8AKiXPrDdyGB6SMBg5K+9mKS2WW1AWIS0o3HpbA/Qy9w+f++Jv9kfjUmiVwU9XywFxhGwPuwQMN+oYO4S9mxgFw88Iw/WOwCWBbgu0SOqj3wWYDewjsN7CzwBYB+3VmBTW5jyudIaG9rsBq4f9SFNRUfOTSCTgidFadZjp/WNlgUjMV1CSDlWpynwrI+WJQ8uObIWfe9eof7uc1uCuuS/h+T9xcOmtQCK8n9v8Y2CD3eXl441T4zcAuAFsVbCzYL2D7gO1QQEFZeEFJKNjJ5vcqX0F9jw9qOwOPhnu5K/4CdSqeQQR8QPkKPmqeBQ///wTYMO1nLC4tX1IXIC4p3Xifw/MtsH6yE8dHUoMb2ceANROfbwaOMTOWg68uTXSQ74F1DJ3SP8IbdW7dNLDeYI/NrKBsKR81/R2YO6/tUhTU84l1dXjQwVoFrwEMbYaSmmQwtKb3KiHnrmDdwa5uppyNKKiFm/iN9Azb9AifTwWuCv93mxV+HxPa/APYIwkZvkzc/yIKysyVZL6CujIhx4HAu4nPywLjwv+rAp/lyT0MuDrtZywuLV+iD6qdYmYfAYfgHfu3km6U1BuPImsqNDoZ6DaZEJY8Drr2S6zoh0dtfQN8SUN/SV1oZOZwMLgW3gIWB96T9JKkUtxCOXIh2phZLkihd8EtPcT58HAO+bEIDZjunedk0sgSnpBzJEz/GZ+XVYhpwPTys5nPuGaSOkg6M2SPGI8rNKh3a40CtgrBIFstDj/n7uunwJa4RusJLIkPdZIx7E1QKPQ/P/y9qXD43pLG5RbcRD1vaU1HskxUUO0YMxtlHhrdD++Ez8I7rEXKPVZPmPhp4vNn+CS7eXENkVxnoZEFChxnRfjCzHbEJ5ieBdwqaXbywrNDZvL8zAR9Euvr8BiDLxsV2jvxQcCdeMc+JW+LKQa/3gO/XQZ71lw55ShBTuCXd+GNDeB1wb8aO1KR73bCAzzWx6dZ9Q/fy8Wwd/BbuRGw06YeKAL4hb8fd0zmll8ofI8LhMSPm/mrkhkLjLaG4fDdzGzjFhwzkhGigmqnhFIV64a34VynNx1Pin2KpMXkLCdpriYPBqwJT5/n5hwm4q+w2+NKajvgXuC/uJNoOO7g+OPMh5lyGHSU1CuMgHId13TgA6CzpE0kdQKOD4dJMlDSViEi7RDgVzz6rHHMXsZsa9yndiIeTHB3+HuioM9gOGw/GCqpuYlbW04ROYG+y8LKj0J3fJJwIb7BfTSN0Q2/Zj/gLwOnF9hmFHAwsPaucB9BWQ7FcxbmXkS+A+5qpJF5cSdRYAolRho2wovABElHS+oSRoHLKGaraBukbWOMSwUXmMfgSINrDe4Of4806JW/LR5Z/SIe8fYjHvzQG7fMHI87pScALwELhn1m+AnC5xHAqWYexXcSTF0QbG6wncF+TPgZbgdbMvhQ1gZ7K7Eu4YOaMrtHkX2L67m3gS0S7e2O+5W+xR3rY2g8iu81YMVKXFdcz74BbJv6PS4u6wZ4/9+5wLrB+OB2HPWBCR0T67viemUCrmt2LXDP++IvDPeG39uUnF9xONjiYF3xaL5hjQRJPAu2GFhPsP0bj+I7NdHmXsDjic+LAr8nPvfGMz99jUf5PZ/7XTT3+YhLNpZYUbctkJVy4J6HbTDNG5lPB+7ERwnNaLqFtZeKH38Q7sxf0swmF9s+TeT34RUzO60GjaV2z0smK89HpGyiia+14+liHsc7ic40fPgInzuH9Y9XuSLsGTQztc7vHlNRLJFDapjZE3hFi6PSlqUEDgcOlZQ/z7caNPueh/2qe8+z9XxEyiQqqNZMM/LeUc2y5f72mYuMK5npMOUImCRYrSpyVY4jgQMl9U9ZjiYxs9F42Y6za9BYs+459VGR1av4m7XnI1I+adsY2/uCT4h9Hbf7lz4DPsuTTevn7hTL1jAtN2cHWAgPbz+OkJ0iiwuequjWtOUoQc7ZcX9T4XlgGbjnVZanrOfjpOA3rcnzEZeSlziCSp+j8Fnx3czsojL2y24OtBLDosP6QZhdbv7WvyawI3BWqhFzTXMuHi24btqCNIWZTcJHfBeHkPxqN1j2Pa9U06FK8ZSQEX2ipIkf+mTvbD4fkZKJQRIpI+kRvHTDv8vYaR48yqq5DyB4Z9EXs3IK6pWPV1HdDY8a7IlHkL0JjCzUdghpvw8fVe5n9fWLMoOkLYFTgAFm9nux7dMiKPnHgVFm1tjcqGo0XNY9b3lzGgPsZWaPhC/Kfj5Oxid1XVf/VW2ej0jTpD2Ea88LnmNsGv4wTMTnl7yGJ0odC5yc2LY/HoG0R3cY1xPsMjw327J4frb9EyaLD0M4d3ewucC2ywv5/S3kQAvHfhx/wMFDuZ8BLsEThr4HrFfj69INeAwPHZ45l176903AI8CBactSgqzL4/Of5kxbliqe4xiSYeUeOj75ObDVw7OxHPWptQzsk/B8dAVbPzw7ORPfY2C9PQnuEY22EZeaLNHElyJmti7wFHCAmXXF59rsir91bgLsK2mLvN1W/RzuvQmfiXoa3lO+jSfGeyJsdAKeFvwnPN/PgXkHkUcvLdeIaKvi/qC5gZOA2yXN2dzzLBczmwBsjPtR7pCUH3mVKuY91sHAifLRQmYxszfwTN9/T1uWGrLcF9BlE3xC34+4XXZrfAIxeMqMgXhG2hOAkXkHkL+ENPZ8RGpEVFAZwsweN7P/mdl0M3sTH0EMytvslG7Q/c94753LCbQAsBY+/AJP5/0pnuenM+7cKUBj5c+/BS4ws6lmdhOeQHaTRratCmY2Be9TxgP3S+pey/aLYV676Xo8gWrWOQHYXtKyaQtSRe7M5eJbG9a/Dn/D2Rjv5DYAVsJtx5/hs89PwVORrA1sVviYjT0fkRoRFVSGkLSqpMckfSfpZzy6be68zb4hpADqQsOMmF1wOyF4fLEBq+CF7K4q3GRjOdC+CKOEHJ/SWNLVKmJmU/HCse8C/5WUfy3S5mRgsKQV0hakKczsB1zWCzMcfNJStrCQi+9JeORTPCVJz8TyNJ6G5EtgDvwFL0e/wsdsSY7ASAWICipbjMKrmPYxsx542e1CHcqbzBwl1YD5gCvxh/FfeLnZj6h/KCc1zIE2X97uC+R1ZH1pKulqFTEPktgP99c9ETKuZwIzG4ePTi5uBR3/FfjLTnWzNmSDN3vD1CE0TF47CTgGmB83fU9K7PBZ4v/ZmTGp601oNDFxpAZEBZUtugE/mtkvklbBTeWFGElhxTWDW3DfE/jbovCb3Qs3B14LHUfDtZL2ZObs5fMAB0nqJGlbvHrCfc05oUpgztF4YtSnJDWV8LTWXIUPXndMW5CmMI82PAgYLmm2Ytu3ckbuCtPvBh6kPgrpcfyZ6Ieb+04CfsNHVncndl48bL80fNFEYuJIDYgKKlvsB/xD0gQ8Q/XNBbcy+xbPHdYoL+GRDl3x1NYXUp/G+gqYfipMXdh9S0sDz+bt/gKwGO5DPg3YJpiJUsXMzsQzAzwpaem05YEZI7yDgLMldS22fZqY2eN4ItXWkK6p+Zh92xfuuxOmn46/lPUBzqG+8Nco/Ec+Jx49smti9x4w/UR45R04Dy9bNon6971IDYnzoForngDzcRI1kspgMj5ZcqY0M5J2x0POG4mrSB9Ju+CBWZtaNVPllIGka/HKrselLUtTSOoLvAoMNLNPi23faqnS8xGpLXEE1VrJcg60KmNm1wF/A+4LWcazwNHA3ySVXeyxlpjZZ8BFuIJvu7Tj56MtERVUa6aMsuVhfTply6uAmd2F+31ulVTTEPhG5PkS7/TPS1uWEjgHWCnr6ZpaTDt+PtoK0cTXFpBWwnOHbUzj9W7uw+vdtKk3Q0mr4UX2DgpzttKUZVZ8zvT+ZvZgmrIUQ9JWuPtlBctwuqaK0I6fj9ZOVFBtiRrnQMsKkpbDg0ZONrMrU5ZlU3wktZyZ/ZamLE0RwuIfBu40s0vSlqcmtNPnozUTFVQl8SSVhR6AEfEBqC6SFsM73EvMLDX/Suj47wUeMbNMm/tCJORjwFJm9n3a8rR5Yv9QNlFBVYJYUjoTSOqDK6lbgRMspR+3pCXwhLvLmNnXachQKpIuBGY1s1ikr1rE/qHZRAXVUuqrdnam6aCT6fj8v+iErSIheeuDuII42MyKOcerJcc5wFxmtmca7ZeKpDnwVFIbmdlrxbaPlEnsH1pEjOJrCSWUlN6IGZmSZyopLam/JJPUsRbitgfMTSXrAisAV6d4bU8B/hIygmQWM/uJ1pOuqXXRjJLzgsvuk06oumythDiCai4VmAgoz9QwGq951LYjqWpMSOdzO25C2cHMfk1Bht2AfYE/pjWSK4WQa+5F4FwzuyFteVoLoVBib6B30ocn6TVgwMcwZeGG5rzixwTehymLw9oxojCOoFpCdkuuRzCzycBgPBXbPZJmL7JLNbgW73OGpNB2ybSmdE0ZZDSJPIyhpMlsAGpm/r6wX+wfiAqqZCSNkTRM0jt10rjdYPNfoO4nYFM839cc4f9k0q51gFwt92nAEcDcULcQbNkbtq/hKbQ7wqhpB7w68cPB31LL9qfjtSLPyFo9q3zM7Bm83mXsGMvjWhqm8tutlxeIRFD3K/7M98VL4wylYRmCc/Ds6r2pL4kj75c3nlV6RtJeuW0l7S7p6aqdSQaJCqo8dgY2fB/O+wB0Ku7Z3AMvmPQZPp4/oJGdrwTuwYsKvgy/dIa9ayBzuyaYTvfCc4M+LmneIrtUuv0XgQdwP0/WaRXpmjLG80B3SUsGU+kOt3ukHuDlPT4AXsfL3XwB/COsewCfMPcw8CFeGTuBLTBzGZx2R1RQ5XGJmY1dDBY7AepuAObCC+zMhtfKOI76suv53IyXae/j+3U5wc0DkSoTRjKH4T6pp0LC1FoyDNg9hJ9nFjP7Au8zh6ctSysjN4raAHh35VD/0PAiXOfjWdO7AccCN4adbsZfbpfBa1Cd3PCYXWZvWFOxXRIVVHmMDX979sMr+E3Gs5b2A7rj5aPH4ea8fL7ElVOOpeP1rxmhptTfgX/iSqpmysLMvgHOBC5oBZFy5wPLSNowbUFaAR2AbfAw/Z2B3YFrFErFf4f3DwOpr+r7l/A9zNwf5Ff17QDtPro3dpDlkfs9jfsMtxsPx4sqvQCMB54MGxSKjZyfeg0H8G7xBJaRCmNmF+A56B6TNKCGTV8MLASknti2KYLf7lC8PPwsacuTcTrh5uOrgAWB7YAhFkrFz42b/N+mvqrvz8DEsHN+f5Cs6hsO/gsNo4TbnckvKqjy2F/Sgh/Ch6fA9O2BCfiPsCfwI97zNcZ2eJ2Dz33bKf+A/tUWODIzZnYVcDDwoKQ/1qjN33AL7wUhqWyWuQc3PzfmTm33SJoPH0EJN57kRsZdpsP/wDvXvXFt/21Y+QU+ixy8PxgBvIOPtPL6jinzwnvAVpJmk7Qo8NfqnE12iQqqPEYBDy0Bhy8Cdjze40zB35ZWw4fwjbE3sCGwPLAidJ5SH+AXqTFmdgueF+0uSRvUqM0H8BfqQ2rRXnMJKaIOAYbVOqgkq0haUNLOkq6Q9D5u1pudMFrCu4GjgbU/hWty+50FLIr3Dd2B9XGLC/gk/kPwWeWLhr/JJleE/fGq9N/g8/2vr8a5ZZk4UbdEwqS8vczskfDF7fg8m7KV/HSwOrgDs60rKmSkbCStiQdP/M3M7qhBe4vgFuHlQg2pzCLpXGAOM2t3b+6S+gODEksP3IL/RFj+Z2bTQkqrg4FtQ42y3AGa3T/gpv87Y/8QFVTJFFBQzc4kMRlsZxh1J+yZ5ZIM7QVJK+IZyI8xs5E1aO90oI+ZZXoCb5i79R6wRQiXb5OEwJVFaKiQOlOvjJ4A3imUDUTS3LgS/zBvRSw5XwGigiqRmRSUf5nMtVUqk7+CE3vDWrhjdQcz+6iSskbKR9IfgIeAc8zs4iq31RXv+Lczs2er2VZLkbQ7Pr800+mayiEopD9Qr4zWxuOanqB+lPR+i7PhN7N/ICaMnUFUUC2lmdmKw0OyHz794VAzu67aokaaJph1HsF916dVs1yHpJ3wuVmrhlRDmURSHfAccKmZXVNs+ywSzmFpGiqkKTQcIX1Slfsds5m3iKigKkELSkpLWh64CZ+RfoCZTSSSGpLmx0dSDwJHVktJhReUp4ARZpbpYJmQkf1O4A9mNj5teYoRMjosR71CWgsPZpihkMzs0xoKFEvON5OooCpJM0tKh0SmFwFrAtub2es1kDbSCJLmxAvIvQkMrdYIJ/i+7sM7/nHFtk8TSVcD35nZUWnLkk8oqbIC9QppTTzyLamQvkhPwkAsOV82UUFlCEk7AhcCpwIXp1URNgKSuuGjhu+AXasVzCLpX8AUM8t06HmY9/MW7ov6IGVZZgFWot5c90d8nmvOf/Rk1isZR0ojKqiMEcKQbwC+xqP8vi+yS6RKSOqMm1874mHEk6vQRi98btQ6ZvZOpY9fSSQdDqxrZjXNhhHuwyrUj5BWxXOv5kZIT8XnpG0SFVQGCW+Ip+GlInYxs8byz0aqjKROwNV4mqvNquGDkXQgPmdmgyyPmsPv8k3gcDO7t4rtzIbPbc0ppJXwibE5hfR0qAQcaeO0LwUlzUNhG/CILNqAJf0Fz/N1JXBKrLqbDiEK7BL8Lf4vlX5bDz6U14ETajFZuCWE3+TFwDKVqlIcwu7/SL1CGoCnC8oppGdaQ3BG6rSy/q0U2oeC8klzw/DsIo1F0dyPR9G8VHsBGydElV2DV9nc2czGFtklUgVC1N3pwObAnyvtdJe0Hp76aikzm1Js+zSR9H+40jirmfv3wAMZ1sYV0jJ4mbScQnrWzCZVSNy2Tyvu34rR9hVUG5iHEN7gj8LzTu7TIKVKpKZIOgbYB1jfzD6p8LFvBV43s1MredxKExKXPk+J6ZpCVORa1I+QlgBepF4hvZB1pZxZ2kD/1hRtW0G1sZncklbDAyjuwefo/JKySO0S+e/qeNzc91YFj9sfeBlYIesj5ZCuaUEz27XAul7Uj44GAQvjk31zCumlSpkH2zVtrH8rRJtUUJL61sF7v4J1LHLzhgILMFM97hm5sCQdCyxsZntVS95ykNQTL9S5OJ4m6b2URWqXhEwQ5+GBExUzm0j6O7CEme1QqWNWg2S6JuATGuaxWxB4hnqF9IqZTU1J1LZJe8n1Z2atfgHG4CaX+u/hdoNpBlbq8hjYAv7/NIPb0j6vJs5XuJnpO7xqtNKWqT0uwGZ4qZ91KnjM2YBPgUFpn18TMvbBK8g+ipuNfgT+Dzgcj7jrmLaMrWkJ/dcUvJbhN3iqra5NbL/70vB9qf3baPdL2dT67zLdvyWXtlkPyqNZNqL59a7qgI3DzO/MYc4VwDp4p3B9yDwdqSFmdjewPXCzpE0rdMzJwBHARSG6L1XkLCRpd0lXS/oEeBXYCp/I/C5wtJltbmbDzexli9GmzWEzM+sKrIgr+eMb27AfdOsBc9BG+7cGpK0hC7wdHAN8jBerfQfYMrFub/yByK1bEbgWdwDm3kCOetKjrWwq2I1gA/PeKM4D2yz8vxvYcWATwTqDCWz2sLwM/8CTuV6XkGE14Fk8hPMNEm/PwO64uWMCXpF05xpcry7AZeGarZz2/WuPCz5x9GtgxwodT8BjwL4pnIuAxfBS5tfiGRq+xics74cnXa1LbL9iWN8z7fvQWhfyLEDAObifeab+BFiyA0ytC31Uj9CP3QM2AKwb2IJgJyX6uz5hBJXr1571YqtTB7rpNddmf3y7juFzzfuygtcm7ZtT4GZtC/TGtfz2wCRg/vD9F8DK4SFaFOhX6Aa/7gXobCrYJLCuYB8kbthKYDfkKag8E19uuSapoHB31Q940sc6YIPwuRdeXXM87j8gyLx0Da/b1ri56YhkBxKXml3/ZcLvc58KHW+5cD/nqrLcApbE3bE3AF8Cn+PVW/fBI+6aNCHjPtHz074HrXVJ9l+4+fTtoKQK9ienwDNrFHBPvAk2DewNsHnA7mjcxGcnga3rGdxzMsxQUGn3ZcklcyY+M7vFzL40s+lmdhPwIT5Bci/gbDN7yZyPrJGMxB28ujLgBv3B+JNHONh7+GSWEuiZ93kX4D4zuy/I9zAedbVxWD8dWEZSFzP7yszeLq2ZlmNmt+HKe0vgXrmZM1IjzKP5BuFl0o+swPHeBG7GR/EVQ1KdpGUlHSDpFnz0cz+wOvAwHg7ex8x2NrMrzKyUukjHAbtIWqqSsrYz7pQ0DngaDyw5mUb6k1kKBEasAyyLvzUvB+wYDtIUnaBTE6tT68uSZE5BSdpV0uuSxoUbtgwwN/5m8XEpx5jm2n8GO1GvoEYBW1By6Et+hul+wLY52YJ8awLzm08s3B5/E/1K0r2hCF7NCAp7EPAK8Jqk9WvZfnvHvPDkWsCekk4Lk3tbwonANqEkS7OQ1HELUm0AACAASURBVEHSipIOlZRLfns7nq3hbmAVM+tvZruZ2VVm9nEJCqkB5lkKTgEurMA5t1e2MLOeZtbPzPZrqj/5zaPwGvAC8CfclNMDuBwolu5kKhSMrMxCX5YjUwpKUj88rc8BuGmjJ55BWcBYvCxzIRo8UON9kDSDDfCn8nVcUe3UWPsNP07B04QkGQtcG35IuWV2MzsTwMweNLMN8CHxe+FcaoqZ/W5mxwO7AiMknRHyyUVqgJl9js8B+gtwcZhk3dxj/QicRBkdv6SOklaRdKSke/B+6nrcVHcTPrl2MTPby8yuacwK0Qwuw3/3gyt0vHZPY/3J9zB2uo9wZrATbhUaC/yMa5Zcp1joh9MZpn7r7pMc85XSdq3JlILCbZ+G6xMk7YGPoMDTwBwhaWCILFo0KDTw0MyFcwf5D9yWPGgn3IF1JB4Pu0Ejjc+LO5R+9o8CRuZtch2wmaQNw5tpZ0nrSFpQ0rySBofaTr/iARuplcg2s//iDuzlgSfDJNBIDQgjinVxa8vIFkbjXYmbmrcttFLSLJL+KGmYpAfwn/h/8NH+SLzW1JJmNtTMbrAq1UUyn+d0MHCepC7Fto80TWP9iaROt8C3X0Bdsv7LBGBOPJ3Ei7ilKEcvvKNPpj1ZAext6C2pb0g9NaxY29U4z2JkSkGZlxsYjs86/wY3qz4T1t2CZ/gehd+PO/F7AnAGcHwwux0x0vUMlrioO+G1vLfFvYCF+ANuu10YmB06KM9Gaz67fzBwLK5Ex+J6ry4sh+FO5h9xU9u+zbwUFcHMvgU2BW4BXpRUsJOLVB4z+xkfRc0N3BpKRjTnONOAg4BzJc0eXorWlnSCpEfw3/ql+PvVv/BJ5cua2QHBn/tNZc6oJFn/i+fUO7xWbbZh8vuT9XDd88mXsNx8MG4+/McF8E/cHtwNd1pulzjQbLiTcA38Tec5mL4h3GNwI24legWPGmys7dT6sjaZSQJoPzOtS0RedvoGfHLloVaF2kaRmZGXqLgOf5nawswmlrn/bHgAw4VAV7xPymzpiTBSfwUYYBlP19QakLQQPjLdFa++fL6ZvdJe+rdMjaAqiqefOZwCDsUi5HJVZf7mlYP5+QzEzagvSlqmyC6RCmBeiXdHPJT4YUlzNLW9pK7BhHy6pGfwUPNT8M5oDmB1M1vZzI4ws7uzpJwAzGwMXprk7JRFadVIWk3SzXiU8G/A8ma2i5m9ArSb/q3tjqBytPFsv+USnO27Aufis9WvKDdqK1I+4bqfC6wPbGihJLnqS0/k8tgtjWdqyI2QngtRVUg6Hk8ku3Xtz6B0wqjvXWCImT2ZtjytBUkd8CDjw/C5oBcAV5nZhCZ2GgoMN+isNti/tX0FBeDmrWH4fCWjcL2U+/B6Ka3izaKlSFoCt0F/DOydtTfxtkhQUqfhReXuw4NYSi49EfxY7+CTgR+pidDNRNJ2uOtjoMXUR00SEu/uCRyC+96HA3eWfN2klV6DkcvAEr9DXZeGgXutun9rHwoqh+ee2o2ZK06OpJVWnGwJocM7Cw/82MnMnk1ZpDZHmDCdLD2xEJ5xYl48ddDtVkbpCUlb4EpugGU4Q3hQxo8BN5nZZWnLk0UkLYhPqdkLN+EON7PnmnGcOuCjdeCIVeH60+GWujbSv7UvBRUpiKTN8XQ1FwFnhcixSDOQV0AelFh607D0xKtmNlXS7njOyI3N7PUyji/gQeAeM7uowuJXFEnL4dkplgxzuiKApBVwM94meL7DC60FxS8lrQNcjI/ATjKztSshZxaICioCzHibux74HdjFzL5KWaRWgaQ+NFRIcwNPUa+QXm9M4UvaGo8Q3srMnimjzaXCsZeyjL8ZS7oEwMwOSFuWNAmjnI1xxbQ4/jJ4hZnlZ6tpzrGvwfMQCM9PelBLj5kVooKKzCA4aY/D5zzsaWb3pyxSpgijl/40VEjdgCepV0j/M7OSJzVK2hB/i97FzB4qY7/zgdnNbJ+STyAF5OXe3wU2CPkF2xVh0vKuwKF4BN1w4JYQ3VmJ43fHM84vHo79mJldVYljZ4GooCIzIWltfO7OLcCwSj1MrY2gkBajXhmtDcxCvTJ6AninpVGQktbA8+PtZ570t5R9euIpaDaZEXqcUSTti+d2+1N7iRiVNC+wP5516Hm8+vITlT5/SXvhv4EtJb0J7JH130M5RAUVKYikufCUOQvgdY4+SlmkqhMU0pI0VEjTcEWUGyV9UI1ONvgl7gWONbMRJe7zVzz6a80sd/xhZP4KcLqZ3Zy2PNVE0tK4GW8rPPfh+Wb2fhXbexY4E3gI+AmYw8x+qVZ7tSYqqEijhA57fzxh6SFmdn3KIlWU4BdYhoYKaRINR0ija9X5h9D/h/BorqIBEEH+F4ALsn5vwqj8Wjxgok1lMQnPyfq4YloBTz11mZkVSyje0nb/gEf/9cHTwo00s2Wr2WatiQoqUhRJA/A5U88DB5SbricrhDf55alXSGvhucZmKCQz+yw9CWdk9H8EL5Z5ajHlKGl13BS7ZJMTOjOApBuB983spLRlqQSSZsWzhByGT5I9DxhVqxGMpDOBDmZ2pKQ9cRPqkFq0XStal4LyOSWF5jGNaK1x/q2FkNn4Yjzn5PblhEanhbzMyIrUj47WBL6iXiE9Wa3s3i1B0nx4KPkjwBElKKlrgC/MbFhT26VNiHh8HVjJzEanLU9zCebvobh14S08OOGhWppZQ4b8z/BKvO9Iugj41MyGN/OA2exbLQMlj4susLLB7QZTDCYnSx2Hz1PC+pVTl7WNL/gb47fAgRQpBZ6CbLPgCvRYvIMfjz9kl+CJ7OdNW8YyzmUOPKv/v/G35Ka27Y3XfVo0bblLOK/jgdvSlqOZsi+Gm+9+Aq4Glk1Rlk2A5xOfnwTWK/tYGe9bsz+Cirn0MoekRXCT35d4OPoPKcnRGViVepPdKsCH1I+QnkpLtkoQUuDciZfUGGJNRFNKOgoPlti8VvI1h0S6pr3Ny3NkmuBfWgs3462BlzS51FKeJyjpVuBhM/tX8EX+BCxi5fi9WkHfmm0FVX8By0kpn8vWG5VUFQllJE4DdsDn8DxRgzZzpSdyCmkg3tklS0+0eOJjlggd+o346HAbayTAIPhD3gIOsozPX5O0JZ6hfQXLaLqmYB7eBldMPYDz8SCE1AM8JM2N59BcF4+0XRSYamZNZsrPO0iz+tZO0OF3nyDe7MwX5ZBdBdVO6p20diRtBFyFv1meahVMDCqpG/7WmvMhDQDeoF4hPWMZDwyoBMHfcBU+SXgz82KIhbbbBHfUL9vUaCttwqjkIeBuy1i6ppBdfi+8BtNovBO/x8qYfF1tJB0MrIxXux2PZy7Zw8w2K/EAraZvzbKCuh1PYloHPhmlQ+l7TwfupEBZAkkdK9mJRmbkn7sWf8vf2ZpZqC5MPk2WnlgKnz+Tm4c0o/REeyOYcS7CR5AbNmbKkXQvnk3g3FrKVy5ZS9cUCi0eBOwOPACcZxl8wQ3K/XV8ZDcMH133xfvyE0o8SIO+tUwa7VurQg2ceWOAI3Bn9c/45LXO+A/h6bxtDVjUYJ4h8PtQsI3AZgN7GOxesCXBuoL1Bjsn4dC7G2x5sB5gq4O9Br8Y9ErIcHSQ4Ve8TPtteW1fhCdtTMXp2doX/Mc+DC8XsHmJ+8yF1785H6+BNAH4L169ehDQOe3zytKC51o7DTdrLtDINovjARPzpS1vCedzPvCvlGVYFbgZ9/OdA/RJ+7oUkXfF0J89ir+3/4Lnz7wKeA0fUY0FTk7s0z/0rXvUwRc9wS4DexFs2dBn7p/oSz8EWxusO9hcYNsl1gH2Afyyv88fnJhYJrs6mdHmnniKq5/wgKV+zTrfGlzQMXi9m9542et38RDNphTUkUPg9+5gT4NNA5sCNh/Yk+FC/Qj2Svj/VbBeYM+D/Q42AqwvTB8HRydkeB2f0NYFmB+fkNkzrO+IR6YNTPsH2NoX/A1/dFD4nfPWzYPb9S8OLwvjw4/3WNyUN0va8reGBTgK+AR3ihdafzZwddpylnAePYGvgRVr3G4HPNPD06FvOATonvb1KFH2i3PKBzfT7YWHm++ET9atw0PFvwG2CNvlFNTlP8Kw++GXWcEGg30D9nnoPx8P/ekOYKcm+t2n8hTU+x7Zd0SeXNcDN4T/BwMf4VlZOuKRm88263xrcEHH4E703OezgcuLKKhrdwMb0jDk0fqAXQ72c973Q8GOz/tucbAb4cGEDHvmtXU/HkkEsCmeUy31H2BbWELHczPutD8UuAx/6x+Hp/M5Cn9z7ZS2rK11AfYBPgeWKbCuOx5huWracpZwHnvh5UiqPmUB6IpPj/gYn3S+LdAx7WtQhvyd8dHxQuHz47ivbDxQl7ftBXiapaSCWsDgWgOb0/tHy/WXW4GdH/4fArY32Ni8PjWnoD70/69JtHU0borvEj7fD/w1sb4OH2H1K/ecm2ODbA5fJ/6fjP9QmqIn+HAnyW14Wch+uP0nV9nrU9yT2TOxjPVG507snu8XGQnsEv7fBfehRFqApL6ShuClzVfAH4wz8VuyCzCXmW1iZmeb2QuW0Qiu1oCZXYGbqh+RtEreuvHAMcDFwXeVZa4GZsVHAFVB0gKSzsBfVNfBQ/ZXM7NbrHX5ozcH3rSGk5z74IFDK0t6TNJ3kn7GrVRz5+3/DaFv7YJXzMzRBbfTgY8gDJ+zsTRuOyxAT5gRJHUwPlrLVYLuB1woaZykcXi2FuF5PcsizR/vJBJRJGH2fI5x0LBuMXjYyl24LW4LYLvwfR+8RsS4xDIZOBjeTuyeHw1yJ7CcpGXwEVSmc5llDTkLS9pD0ghJo4GX8eH9G8DW+Jv8QNz0cDgwe2oCt0HM7AZ8BHKPpD/lrb4O91HsWnPBysC8VtZBwFlh3lfFkDQgZNn4H/7bW9XMtrbWWzl6D2bWF7nsHKOA/8N9aD1wK1V+Fwqhb22K+YAr8SH4v/CyzwUyRY8LuSNHAttZw8CoscDfzKxnYunSnOuepoJ6A1g6/Ig6Aycn1r053R+uGfyGa5CfgU54z5cTfm/8bryAa6FJwN3w6xfQaBZh83xZt+I39kVLOQdb1gkKaXFJe0u6Drd7PwNsiF/6TfBMDduY2cVm9qaZTTezt/CXsQnAq/IQ10iFMLN78He1myRtlvh+Ot7xnx5CpzNL6LgexX2RLUJSnaRNJP0XuAd/SV3EzA4ys49bevy0kBcUXRUvy5Ikp6C6AT+a2S9hRN3YiPRNYEoj6wBP7Ph5+H8OXMslFcV0+GWs9613AceZ2dN5h7gcGBYyuyOph6Rtm2qzMVJTUGb2AfAPPN/Yh7jDMsfIQvtci9uMuuNXIDfkWQnX+AfgF3RRYAR0uq/4qGgk/nYfzXt5BIW0lKR9Q5LPL/EIu0F4ePB6QG8z28HMLjOzRusimdlkMxuKm53ulXREKzA9tRrM7HH8BeFKSTsmvn8J9weUFn6cLscA+0hatDk7S+oiaW9cIZ2Kmw4XNrOzzOynCsqZFrsCt9rME4VzCmo/4B+SJuBRsI2VNRlJ4ZHVDF7CNWFX3KZ4IbBwYr1AW7h/eQngfEkTcwuAmd0BnAXcKGk8vu1GJZ9psq1G+pT0qUGsvqS+eNG3+YLdvt0SFMayNCw9MYGGpSfGNKaEyminPz5q/RnYzcy+bcnxIvUEc/UD+ITpy8N38+IdxFpm9l6a8hVD0tHAGlZGuiZ5ktP98CrQL+Hu6Mdb+jvNEmHu0we47+z5xPez4mHcc1o5GdRb0TyoLL/FnoHH+DeHX8L+jRI65MOAG9ujcpLUQdJASYdJuguPDroFV1J34KG/C5vZHmY2wsxGV+KhN7MxuAJ8FXhN0notPWbECebUQcBRobPHzL4BTgcuCB1dlrkAWFLSX4ptGEb3V+KmpvmBQWa2qZk91paUU2ANfK7TC3nfLwV8UpZycqrat1aUaodGtmiBoQaT8kMdiyyTDIY2dVzcYToRNwdkemJepRbcdbcqHuJ9L+4sfQcPAd8BN9fVWqb1cHP36cSQ80pe1wXCb/t03JzTCZ9/uFnaspUg+ya4VWOmOXHhXNbHg3m/xk1ZvdKWuQbX5CrgqALf7wFc16zjVqlvrfSSXRNfjlaQcTeLhOH/StSb7HITaHOly5+0DJjXgolmBO4+3NF8hBVpISGh6AP4W/eBeMf+T3zeVKZLgku6D/ivhdpGITFxrjBgRzzf4PVZP49KECIbx+Ipob7KW3chMNaam9aqFfSt2VdQANJKeBqdjfFAvS6JtVPwN6v7gDPIYP6sWhAiIVejXiGtjNutk6UnfkxPwsYJ5tZD8Ql/+5vZLSmL1CaQ1B24G4+43BM34b5gZrUz0TSDEL78DJ6XcSs8/ultvDN90FpFp1UZJO0ObG0FEsFKegI4xcweaUEDme5bW4eCyiH1onDVx5FkIOFkLZFXuE2WnlgRf4iTpScKZr3OKvKH5UY8WvBQy0Bpg9aOpC74dIqpeKTc08DylsFKwjlCJN/N+DzRG/DErW+mK1U6SHoSuMDMbs/7XniAxGJWib4vo31r61JQ7Zi80hOD8B9SsvTEs9YGSk+Et/7LgOXx0vJvF9klUoRgIrsG6IVPpl7AzHZpeq/aEjrcNXEz3pq42XcXYLCZvZiiaKkRFPWzwIKWVz5F0kK4VWTBVISrER3TFiBSmCKlJ07AS0+0uRGGmY2XtAv+Nve4pOOBK9qTWafSmNlvknbGFf+fgAUkrWFmz6QsWq7WVa4w4Bx4hvNdzGySpLfwdE2rW4bqMdWQ3XFfW6HaXgPw+U9tmjiCygiS5sLnHq2NK6TFcAd3boT0YntwCicJvogb8Uwre1sbq5Zba8Io5Ww888TPeEXbaU3vVTVZegB/xfO45dJp3pOUJ/gmnwUuN7MRaciZFpI64Ndl40LmTUl/BzqY2fE1F66GZHkeVJtG0ryStpV0iaT/4RF2f8NTDR6AJ1Zd38xOMbMn25tyAjCz93E/21f4nKnVUxapVRNGoUcBV+AvQEfWWgZJ/SQNx8uFrIyXsV/bzO7KV5Zh1HQgnq6pe61lTZn1ga+b8L3FEVSkckjqTb25bhA+ufBp6kdIr1rryqxcUyQNxjvWC4Gz0nrzbytIOhNXUKvXwscT8sMdBmyApyG6yErMfynpP3ieuZor1LQI6cWeMrNLG1n/KbCemRXI49p2iAqqSoQ0SkmFNCeJOUjAG7GTLY+QMPN6PCJtSP68kEh5SHoUT+S7lpm9VoXjd8DTuR2G54y7EPiPlZm5pTWla6oEkubELSoLFZoaEtaPwQuutmnfXFRQFSDY9hem3n80CM9Wkcxj93Zb/zHVgtDpHY/Xu9nTzO5PWaRWS5jM+xFeOWCwzZyVurnHnR3PcnAIXkp9OHB7SywEkg4D/gxs1NYDZiTtjyvjHRpZ/yd8/tOatZWs9kQF1QyCQlqchiOkOhoqpPfa+oOUJpLWxmse3Qwc20ikU6QIkg7AgxUWwEelD7bgWL1xn9HeuJVgOD79ocXPQQiVfwNP+XN3S4+XZSS9jJexKHgvJB2KlxA5oLaS1Z6ooEogKKSlaJjpeyoNFdJHUSHVlhD5eBXQG0+T1Kbt8dUghHm/hmeYPwTP5HFrmcdYHjfjbY6/NFxgVai9JOnPeLqmpc3s10ofPwtIWg7Pldm/MReApJG4f+rfNRUuBbKroDxHW6GZzSOqPbO5kdIT40kopJgzLhuEl4f9gZOAg81sVMoitTqCyehqPPz8Tvzt/eoi+9QBf8EV05LAxfh8taqm0wqZ958zszOr2U5aSDofmNRU+LikN4C/WrVTD6XYB88QIXMKyiuuDsMLXDWWG+p+PDfUS5VpUh3xsM2cD2kt4DvqFdKT1rCkcSRjSBqAz5l6DjjQzCamLFKrQtIteGn0G4GH8fRCFxbYrjMwBM+d+BtuxrupViZWSYvg8wMzna6pOQQz5ufAHxuzBoQk0OPwGlBNVsZtgSA174MbFSVTCqpG2XUldQIGUj9C+iPwBQ0VUowQa2WEzM8X43OndjCzNj9PpFJI6ofX6FohfPUIHjH5DzOzkHV+37C8gj+nj6Vh1pZ0GtAva+maWoqkrYBDzGztJrYZgJfYWKZKQmQrw7lVt47JGGD9At+vBbzf4PtG6pM8BrZAC+uTALPiaYOOAx7CzXVvABcBW9MOasq0pwXYCR8BH0h4CYtLSdftJHw0BDAvPhF0BD7/7Kfwd8kMyJkrQbFG2rJU+LzuBnYvss3uNLcGVMPj9MdHRx1nfJ/og08C2zkDNaJSycVnZk/h9ewdH1IOB2Yr81CzAcORXiZhjw0ZnFelYemJ9/HR0SX423UmS09EWo6ZjZL0Am6uWl/Snmb2Q9pytQLOAd6RtA7QAfgGV/Zv4Yrp6xRlm4GZTQwVgy+StIq1gfmEkubHX6J3LLLpAPzlutICVLQPrhRZSXU0DB9SNofOv8HxktaXdEpIT/8dcCZ+8c7FswGvBBxtZv8XlVPbxzyKbA3gQzxNUqNmk8gMfgf+D/cvXIzXj5ofn8t0UfCRZIUbcH/InmkLUiGGAHdYcd9ptVIcldwHF5jM1jnsX3FqoaAGSHpT0s+SbpLUWdI6kj4HQJrnZdh4BajrBmwLbI/PxEwyHJgHf1qS4UW/Qt1RMLjOQzMPxc13/c1sNfxBuwIYKunrvF0jbRwz+83MjsBzHN4k6aQw0TeSQNKcko7BsxcshVsb/mlm/w4jz03xygd3SSr3DbsqmNupDgROkTRH2vK0hBCJugc+ZaLYdk2OoCQdI+ljSRMkvSNpy/B9B0nnSvpe0ifAJomd5vkENh4U+uANgO8TxxyDR0X8B+gLrBu+fx533veEuuVgy4Gejiwnx+6SPglyjA7Z9JG0qKQngj74XtJNTV6cKttUxwAv4vNU5gTexTMArAN8bmaMh2P6wPQLwH4Duw2sE9hxCR9UB7ATwvp7wbqA/RjWHwK2Cfz+gZeg6Ibbcc8I7a+DK/yzcD9Ul7TtzHFJZ8HfbR7BzbwLpi1PFhZgUdzk/RMwEo+MA59i8S2esDi3bcewzVNAj7RlT8h1OT7vKnVZWnAOq+HVr5v0l+J+oy+KbLNt6G/r8Hf9SeG3PxR4D085NSfwGDkfFBy5Kkw7FOwXsCfAuiZ8UKN9OxsCNhFsMtjnYHOG/nga2APwS2dvqxeeRWc8sESQaX587hr4yPe4IF9nYM0mz6fKF34MXtsl9/ns8IOaoaBuggd7g01PON7WyFNQncGmJtb3Ansu7DMb2Ef+/TWhjdWB0eH/dfBQ2M5p/wjjkv6C+1WG4b6VzdOWJ6VrkCsMeDtuCj8d6F1gu4vwUVTyu7rw/atkJLAImDucx9Jpy9KCc7gCz4ZSbLstgHvLPPbrwGDgURLBDHjaKAM6/g9u6xCUT66P3bGAgvo4sf5MsF3yAiZW8Ejo3YKCGocHoHXJk+eacL4lvSTWwsSXdKxOxiNwkivnXgB/anL0yTvAXDSsrDgbMBH/VU7G48W7wo6SxgEP4Fo8x3fWDktVRGbGzKaZ2RnAlrhP5aIwr6fNI6mjpO1xy8wI4L+4KfxYM/uywC4nAVuHsGZgRvmLg3Fz+pMheW+qmNn3wD+AC4MJrFURTKbb4B13MYr6nyTtKul1SeNCf7gMrsR745GPOT7N/TMW5pkD1yo5+hU4drJf/hR3UPZMLO945Of8ZjYJH70NBb6SdK+kP4Rdj8K7+xclvS2pSR9i6kES88L3X+AqOkepM2LnxmeQvQ3c4GbT8/B0K3MnNsvQRK9IFjCzZ/GHfX7geXlhxDaJpO4h0erHeJ2xM3DTy6WhIymImf0EnIgrciW+NzM7AXdJPCUvS542lwHz4S8erY2tgBfM7PMStm1SQYW5bFdSX0+uJx6BKbymWlLH9M39swB89xNun8tRqA5K/iBiCD5Myi2/wCgLGT7M7EEz2wB/xt4LcmFmX5vZ3mbWG/cN/7Op31DqCmpjeKwO7BLcWXQX7rQqhTo8K+XBMO1nH8J2xUtG/yTpCdzxOGtWHLuR7GBenXc7PLfb08Gp2+rewBtDUl9J5+KBD6sA25rZWmZ2p5Uelv1v3K+7ff4KMzsXNw8+IWnZSsndHMyzpB8EDA9TTFoTe1J68FaxEdTs+Av5dwCS9sBHUOBJlQ+StGAIKjkmt9Ny8NxAmH4S7g95GnfkN8UuYZsH8VT4U2DKxTApHH9eSYPlWe1/xQ1e04NM2yZG3j8FeRut8pC6guoGV90Gv/0HHyZeh4cMzVri/mcBi4Lt7iHFQ/FB1Yn4w9MB6A58K+kZSadL2lBSt0qfR6T1EUYDVwB/Ao4ArlMrr9wqaWVJN+AJYAWsaGY7WDOKEgZFdiBwduhs8tdfiefie1jSqi0UvUWY2aN4hosj0pSjHCQthOe5u6uEbefAvR2NJuE1s3fwgOfncD/rssAzYfWVuD55A/ch3p7YdeQomPoCHj3xd2DXIvL0CUKfjvtT+kKXozzopi4shwFfAj/ic1H3DbuuDLwgaSI+peFgM/uk0fMOjqt0kW7HHXl14DNsh+LDnxKYDtyJ2daNH16z4cETuYm7A3HLYC610TPhjTrSTgm/kfPwUts7WLUTcVaQEDq/Gd4p9KO+MODPFTr+KODjYNortH5j3K+1Q1AUqSCpP66kVrASq/WmiaSTcVPcgSVsuw5wmpmtUSVhGvTBZVK0D242aUewmBl7wj5fwuSpYCNC1N6XpaXZsJBqY6UyI1s644rqRNxZPAF/qzgfj5SZM+1rEpd0FjxM91vgcKAubXmKyDo7sB8+GfklYAeSqWsq186C+GTdhZvYZlC4bqlGRwInE9I1ZXnBFcEYfIRbyvaHAJdWTSZY2QqkmqtWH1zydUr7RoWLv08XGD872LJg95R3YVqcBwqYBTcRDsOjAMfjaeUvxiNs5kn7GsWldgs+3+RZfKJ35u497ng+Dfc13IGHjVc15yBwLJ7poKltgzeDMwAAIABJREFUVsKjdndO8drMFjr+QWnfpyJyrof7k0q6b/gIdZ9qyvQNHDYJpjdDOVUtF1/qN6rBUp+scFqRizKtmhcGj2pfBTgSuAcPUnkHjxbaAQ+lTP96xaVqC9AJN7F/AayXtjxBpuVDR/UTPsF20Rq23Rn3f2xQZLul8EDcfVO8TtvivpaKjyYrKON1uP+l1O1fB1auojwdgQfPgUcnw/RpxRVVVfvg3JINH1QSaSV8JLMxNFqL5D68FklN/ATBxr889T6stXCTR7I8R+Zt3pHykbQePkdlBHCymU2tcfsCNsRNjkvjo/p/WQr5JOWpbM7AM040eh0kLYzXlLrSUigsGK7Zo8AtZvbPWrdfDEk98KlEi5rP4yq2/SzAz1SxBpSkc/BSK4+sDxs/7KPz1Pvg7CmoHFIvCldzHEmNqjk2Lprq8PDNnMJaG59GkCwBP9oye3Ej5SCvhTQC/x3uZDWophwmEO+C55f8HQ/guMFqVBiwEZmEm8DvN7MLimzbG1dSdwPDav0shND3/+JZ2DOVyV7S3/CR6DYlbj8AuN7Mlq6SPDvjk503A54EVjOzj7LQB2dXQbUiwoO7JPUVeQfh0wOSCuvDqLBaL+Gl5FDgaGA/M7u1Su30wgMf9sUDd4YDj2bltyNpSbwTW9rMvi2y7Vy4QnsZ2N88E0XNkHQxHuiyfy3bLYak54FTzOzeErffDfizme1cBVkG4vdoXTxE/yszO6bpvWpHVFBVICisRalXVoNwG++T1Cusd7PS6URKR1435wY88eyhlTK5hFQwh+KTh28Fzjef15I5JA0HupvZ3iVs2x2f7/IFXoyvZiZSSbkE1X82s8rXUGoGkpbCfzt9zScYl7LP+cCXZnZOhWWZF8+LkJuzdCvwBzObUMl2WkJUUDUgKKz+NFRYXWmosN6q9RtmpHmETvcy3C+5vZm93czjCJ8kfBg+gfEy4DIz+6ZSslaD4EN5D9jMSvBBhOwOt+DzZbazGubGlJcw3xFYJwsvhMHXM62cUYqkx4DTzezhCsoxC24CfQwPzX8Rzwp/XaXaqARRQaWEpD40VFhz4VlGcgrrdWsDlULbKkG57IZXoT0ODwgo6WEKncP2uGLqjPuXrquWA7wahCSfe+HlEoq+WEnqhAebzAsMrtVbeghwegUvwdN07aHqy9IJT3O3jpm9X+I+wrMxLFHMpFqmLJfhCWS3xHMi/BVYIwtKPElUUBlBXvI5F3AxCFgAT1OSU1iv1jqCLFKcYJq7Ea/ns481kZEkpKvZB08f9D6umO5vjSPn4JN7AbjIzK4tcZ8OeO7DAcBGtYpElLQWcD0eMNFogtwayLEZHjDyxzL26Qc8a2YLVFCOffCJv6vhE4bfBTY1s1cq1UaliAoqo4TIsbWoH2EthOfYyimsl9KM6IrUEyLuzsajoHYys+fy1i+Cl6nYBZ9Xd56ZVaNsd02RtBpwG2X4LcKI4CxgI9w39FUVRUy2Owr4yMxOrEV7jchwB3CfeQ7DUvcZjL/4bFJ049KOl6sFtqaZfSDpPKBbKf7ENIgKqpUQHL5JhbU4bjfOKawXamnbj8xM6EyuAC7AO+HVcTPeIDxZ5yVm9kV6ElYeSSOAr8v0qQif67gnsH6NwvYXxCfvrmRmo6vdXoH258FH2X3NbHwZ+50EzGpmx1ZAhj74qHdPM3sgBGw8QQkRmWkRFVQrJTiq16ReYS2NhyXnFNZzaZoz2ishYel9uIn2J+BcYISZTUxRrKoRTNP/A1Y3sw/L3Hd/PGx/QzN7txry5bV3HDDQzLaqdlsF2j4UT2JbLFF4/n534PPfbm5h+12Ap4Cbzezs8JLwIF6h98KWHLuatC4F5W8hhSaOjUh78m7ahBIif6TejzUAvzbJjO2ZCR9ta4TIvr/iprzPwzII2MPMHkhTtmoj6Ug8992mzdh3CG4e3cTMXq24cA3b6oxXMfibmT1Szbby2hX+LB5oZo+Xue9oXIF/0ML2r8HLD+1sZhZG+6cDA8xsamb71qbyIGVm8Uy7txtMMZiclxNqcvj+dqtirqrWtuBJM9fFy7s8jhcNexGPOtsU6Jm2jG1hwSuTnounvroJWDWxbhAetXUuMEvaslbxGsyCB31s3Mz9t8Azoa9VA1kH40qqUw2vz0rAJ5SZHR9XFBOBDi1s/zDcujJb+Nw5yLNB1vvWmjdY9pKRBLKtfQk/yrWBE/CJgvklRuZKW8bWtIROZxQeAnwe0L+R7ebCJ6q+BCySttxVvB4b4T6WZilivA7Xt8BfqiyngIcoI1FrBdq8FDixGfsNwiP4WtL2n/Fy730T3x2HK53M9621uDlv43H/5e9ffwGbunj5S1RSpd2XWXCTYK7EyM/EEiPFrlldeAN/IoyMDgd6lLCf8NDy74Ad0z6PKl6fe4AjW7D/6ngl2G2rLOeS4V5U/TceXgx/APo1Y9+DgX+2oO1FwvVcO/HdgsAP78JxraFvrVlDJVzMMXhEj39X4wJauBlsr7SvQ4rXP1li5G7cwf8ucDk+E7932jKmeG1mw/PjfUB9YcCyTUR4tuj3gauA2dM+rypcp8WA72lBORo8O8cXeKRZNWU9D59cXe1rsgPwcDP3vZpm1oACugFv4Xkjk9+P2g7+bTDpBrBVwGYD6xX+vxRsOth5YAuBdQObH+wQsKlVLk5Y8Dxq1VAJFzRfQd1ewtDTLFzQaTMPSW8rs/12raAKXI8OoUM9BC+K9wNeufXfwJCkyaCtLnhhwFPD2/aduIm0RYUB8RRXV+OpggakfY5VuGZnAiNbeIzFQn9waBXl7IGbvgZW+Xo8iM+Na86+rwGrNGO/uvDMXpn8veLTVD77De46B6bPA3YL2PjQh74KthPYL/D/7Z13mFXV9f4/C5Am6iAilgD2BiiKSlFQY9SE2GLvEY3B/rXHGg0WEiMm+jMJVuxdgxogJhq72EVirFgQxYaIiAxFWL8/3n2dM8O9M7fOvXdmv8+zn7lzyj77nH3OXnuv8i6fBv51GE+/At8BfEyeY2tBz6/kFwiCB/E93Y28Sb5Fqr8twzG3IJ6uWmBeD/idQ+1k8MHgK4FvCv5YQghtB342+BCUIv7dsO3csK0LeAfxTK2SaMsglCl1DoqJ2D5svxixjy9ARsmrmqsDqqWEl74fcDziVfsC+ACloRgBrFPo4F0pBXkyjUOryL8AG5TgGgcHwXd8S3lu4b5WCCugQQXW0wutNi8o1fNBXpfPlrD+XmFi1ymPc9uH8bBzHueej1hoOiS2tQWmDISjvobazuD3ZqmRmgW+I/gxddtqHbo3y/tU8gvUF1ALUBKstijx2XMNj3N3HE6fAbUrg08Iq6N/ga8M/kVCQPUEfz0sPReFbeuAvw3+HcxfWxlAfx/qXzO8LMPDYLtT+L972P84cQWVS7+mUowcjdi9Z6JMqrcCRwEbVtPAG+5nF2RAn4lSnJfUcQQx3r+EVmctxkkFrbBfJEevtTT1rIpWEX8utK4M9bcJ7TykRM/hXET+m8+5mwFv5HHenuE7XK3B9qOBJ5bC6RNgQds6lV3GcltQ8QG+CviUun3zHU5rjnepDc2Lp919oosE9ZbQCemw6W3QcTj1pcmWKAIyhcNRdGo7lJ8bNJXfAOgMnfbXDLV/2HUIohmZ6O5LXczAL4VLROQIF95097HufiCaAPwYCfphKFndTDO7y8yONbM+IR6jomBmHQPx6X+RC/5twNrufomXONGdu09DjirTgFfNbFgpr9eMuA1YjD7RvOFiN9gBMb1fb2btCm9avfqXAicCfwhxhEVD4CocgVbi+aA/SvOeyzX7ILXeXu7+WWL7yijc5ESDTWdDh1XQuJnCEOTT3gmlWAA4CJiLDK9HI5bfgE5Iy1ByNLeA+izxez7QMcNLVzMd6ZFqEuVppDROoWeaE1dLVgIdkM4foDewr5nNSRXExLB6XncSUQ9BYL3r7te5+6Hu3gt5ZU1Ec4sHgS/M7D4zO9HMNgsfcVlgZt3N7DykptwX2do2c/eb3H1hc7XD3Re5+2nASOAuMzs/kKpWLRID/8WB8aSQuuYgV+k1gDvNrEMRmpisfzIKuyiYSqgBhiIV3Yt5np+TgApC6AHgVHdveM3fIbvRa0BNN+TJkkxGlbJ7dEO2liTWRwuBY+tvrsm2bYWgbANEGnji95yeSE8wJ1G+A5KEX01Nx7+ScOoQZu4zgFvcvSZRlnf336e5fkQR4O4fhgH/CHdfF9gCEVX2RfbIWWb2gJmdYmYDmmNgNrONzOxqNDHsjdTKP3P3RzzoQsoBd58EDECrz0cDd1zVwpUn6h9AweSsLsqu3dEn/6CZLV9onQ1wJnCUma1fxDpHAOMKeKeyFlBhkn8H8KC739xgXz+U2uW8sGnOYDRzfyCHxnyP7CUJZGTtLyYqSUB9jgztAFMPhgUPIReYlPfC44g/JhsshsXvydC4KYpZ2QrYz8yOMLO2QbWzfWIgSF4/ogRw9xnufpu7/9rdN0QTs9uRLeZm4Cszm2BmZ5jZwJA/p2CYsIOZPYQ0GJ8hBu5feZ7JBksBd5+JVgv/Al4ys93L3KRCcQ5wmClNfEEIq9r9kX3wYTMr2gzexah+KXI9LxhBXbgnssfmc74h80e2WYBHo7H8jDT1XAFckFBXT62B2vPRiuhe5LG2FEnDFHnndcgLCuCNcIEd66quRTGTpUepjVzUd5K4NbF9LbRqaRf+3wMJkjk94AKH2ufAh4F3DUa64eDTE04S1zYw6jXYVttLwZFPo1Xqr5C6aQHqjwXI7rAjmpkNRrPqr1GOm5I/m1iWeVe6A3sDV6KPcy6ao5yD1LEdcqxvOWR7fAW5df+aPDyqyvQshoRv50qgY7nbU8B9nBT6sCgOM2ggvgI5TxQt0BYtKt5BeaoKretIYHwB5/dCKd6zOfZgtLhZxskGBdxPTY2x7o7DqsELz28F3wq8Uxhftwa/Gnwh+OHgq6IYqd7gp4HXtkQvvrxLDnFQaUpGX/0gjNZGBtxxiJPqS5TX5kQ0cym6x1AsuRdgZaTaGYMcWr4F/oPcaLfPNHADXdFs8uNw/M+rsU/DfdwbBuMNy92ePO9hOTQJ36OIdRqyq7wF9CxivcORa3tBvIlhUpz3/YZ3fmIWxw0IY1e/NPs6A9NJx+JTorG1JO9Pc10o59KMTBLI3+JglMvnbeR+/gAiWRxAgWSNsRSnoODK4SjX0nMoZu1J4EK0Su8TZtezkZfo5uVucxHu2ZADxZdhUlU1rvuJe9gpzPKLuhIM3+eHwPpFrHMCBbhQIyfizymAjBbZ7S5p4pgeQQDtnWH/BcBdac9vZpaegvqjuS6UVykTFx/y7Nsfpaf+HzIITgiz8oGFvHyxFK8gJ5idUbDwLKQy/hjxCf4UZQotezuLdK99w7t4azXeF3KOObsE9R6JAoM3LVJ9G1AAXRMy11xWhGe1XyP726PcTqMy7O8d7iEz20uV8Jw224XyLhXAuEtm28jZ5GEbiaUofdIO6dgnh9n5CaGfWmyKEaS2uRpRTjXbLLZIbV8baSZ+VIK69wurloLYKxL1/QElmcz1vLZBWPYp8Prv0wh7CfA3pOFJq7ZGETpNsqcvgaOXKui29bKZF6XAlg73eeM5S+5rrqUnso3sgbx+XiZL20gsRXn2KyCW5w8QncteZFDBIibpoSii/9+hn1LMBL+gStkbUNzWF0jFVTW2NaSKvb1Edf8sPJMdi1BXiq5pYB5teKHAa9eE9zTTOz0S2fRWzLD/x+HbaNQZCDmpvbM1zK2ksbVhqbaMut1Jn/XxJsqY9TEEI25DXfr1vsT060WFmfVEq6QjgUeBy939uRzraI9siql+GoI8R1P99KS7f17MdpcKIbX8Hcjr9HAX60JFI8QvvYmyuj5VgvqHIaeSo9w9lzCfdHUdhngSB7kCj7M55x7gUXcfW8B1hyF6tiFp9m2LnLm2dfd30+xvhyZg57v7/Rnqb4PuazRakb/l7htX6tja7BKxNRTqbCMXI4+eeShYezQtzDbSDM9yAKLOmY2SK65VxLrbofi406jCFCPIQ+4SNNsveOXQTG3eHw2iJXE8Cu/LpxTIr4fc2ScDI7I8vhsa1AtSIyNP4mVyQCFHrpk0ktARCZ5HaMSRBpkmFiJ7rQP/Kvc70ViprhVUlcLMOiMm9dTMfUu0TH8SzdyfclG6RPDDLG9XpMJaB9n+riv1MwpMFptS109DkdB6gtBX7j69lG3IB2b2E+Am5CxygbsvLm+LMiMEjz6OVH1Xl+gam6CBeLS7/7WAerZCFF0bufs3TRx7AjDY3Q/K93qhnhsQifY1iW2dkFPE3e5+aYbzVkFjyg7eSPB5CP6+Ea2SDPXDwYW0uZSIAqoMMLOOKDlgaiAciAhDk6qmkhKVViKCIP8lcDJyRBkD3FuuATcIyk2o66ftUBT9E4nyvlfAR2RmqyI2jhVR1t6KE6QpmFl/JEA2dvfZJbrG2sjueD1SmeXVR2Z2HTDHxZfY2HGvomzCj+RznUQ9ryBnhBfC/4ZCJtog1Wja+zCzscBCd/+/LK4xEQUmD0OkBKcW0uZSIgqoCkCwjWyJXpiqto3kAzNbDTgOGYAnI8H0VCUM/EmEwWJD6gssp77Aeqdc7Q4C9RQUDnGMu99XjnZkAzP7K7DE3U8o4TVWR0JqAnBmPv0SBP//gKHu/laGY/ojr7q1PUt7VYZ62iM14SruPj9sOxXFaG6b2pbmvM2BSUjgf93ENYYjJ6F+wCrAYq9g+2UUUBWIYOzcnLpBcFvEH/fDQOjibatqBCLLk5FH3R3An939nfK2KnsEgbUu9QVWe+pUt08AbxYyaOXZrq3R8/w3ykpb25zXzwZm1g3Z+3Z09/+W+DoTkd3rOFeqn1zrOBnZjn+aTsiZ2RXAN+5eEDGumW0K3Onum4T/d0aq24Hu/lGGcwyp/25y92ubqL89onc72d0nNnZspSAKqCpAGtvIMOpsIymBVbEqnSTCB7UzmulvClwFjG0pKs3gXZcUWCuiASTVV1ObQ2CZ2YrI2aMfcEBjdolywcyOQ/GFO5Zy1RnIWx9EzhO/zFVlHEiLX0OrsAcb7OuAgsMHuvv7BbbzMCQEDzKz9VAYxb7u/mQj5xyInHy2bkr4mtlpiPpo10La2ZyIAqoKEVQ5fagvsCrSNpJC+JAPQoLJUQzZHd6MuZfKgcCWn+qj7VCW2Kep66cp7v595hoKurYheqRLEeHutRX2TrRD4Rij3P3eEl+rE3AXcgzYL9dVpZnthAJk+7r7gsT2fdDKbIcitPFypCn5G6Ly+ktjTh5m1gWtQg9w92eaqHs14HXkyLGMi3qlonUJKOmT0/n630g5ff0LRBiINqL+zH0pFWAbCd5FRyMb02vIvlTW3EvlRBgoUsJqO+BHKAQh1U8vF9spxMw2QoPz28CvK8lj1My2R2qsjTPZWIp4reXCtVYHdnf3b3M8/+8oEHd0YtsExHl3c+Yzs67/P2gykeJeHNnYd2JmFwO93f2QsCHj+GZiU/nC3c/IUF1FonUIKLmLnoUivR2lLE6hFs2qJgGjWTYbZdUhg22kAwmnC+CNUqqazGxDlGrhAMQt9id3f71U16tWmAIkh1LXT+ug2XOqr14sxiozeI5ehpjdD/Qcg5xLCTO7C72Pv2uGa7VFauUBKLVG1qplM1sHUWf1d/ePzWwNtCrp6QUG4odv9iuUimkb5C6+qJHj1wWeBzZzZRvOOL4thbYTYek68NNN3J8opJ3NjZYvoMyORrP2jjSeoDGVI+pUCogEr1Q0h20kfGTDgFNR3NdYpKZosR6IxYaZdaW+wNoIpQ1P9dNzhTg9mNmeiM/vT8Clze3AkQ5m1gup+gY0hy01vKejUazdzrk4HJnZRchb72AzOxNY192PKkKbelGXM3Ard/+siePHA8+5VklNjm9LwdtoMl5d41vDyN0WVUrM2IsYItYp+33mUZBqKZliZDYyJJ+KXN7b5VDXcsi+9HKoayTQudz32BIK6VOMPAVchNJYLJ9HnT3RKvrfwGrlvsfQpt8C9zTzNc9ERMNr53DO8sAMtMp5GxhSpLYcByxCwqmpY3cBpn0Hx5dyfKuEUvYGlKwUOecJin7/Vdnvq1QvwrIpRr5B7rm/QauhZVKMID33GeGDfQzNSKuGvLQaC6LR2ikIqKeCwJoM/B6peNKSiKappx1iff+URuhzmvG+OiGS0x8383WPDe/vJjmcc0AQTm9RhPxciHz6K+DBLI5tD7x1Bpzi8N0dKBNuZ/Du4fdfwJeCXwreB7wL+Frh/0zjW6WWsjegic7Ieha/TCly1siWLqDSPPvuiCn8CqR6mAv8C3mD7YtyLs1G+Ym2KHd7W2sJA/sOKEHdY0FgvYjsTbsBXZs4f7swQP+RAjPJFuFe9kJxOvl/9/ld9xDkPTcgy+MtCPaCM8uGicLDKIXK/lkcfwowaSnc/0dYuir4PeBzg1B6Bfwg8AXgfwB/GXwx+FvgvcDvyDC+VWopewPSdMCHYdY+FZEa9gvCYU6Y2e+eOPbGMOOfFD7MZ4DVVoKra8A3DB2WEjyjwdcJM4qNwe9P7BsHvg34qeA14L1haR+5b4JIX5cgG9U84Kqw3YH1wu9OSBc8Ha0+nqYJyvtqKmGWd3qYOS5G6ohnw8C4Q0u612ouyBmmYYqRKWGisRdiKWh4ziqILPcFZFMpV9sNkZ0eX4Zr74HSdQzL4tjlw4Tty6YmAFnU9cfQT+8BGzZxbA9g1ggY/DXUdga/N4eJ9wngx9f9X+vQvdzva5PPp9wNSNMJH4YPqifKyzINJQZsj3KdfJvqSOoyqQ5ARsL/AB+cBHcuhvnngG+f6KC7wT8BXwJ+Z1gWz0wIqHbg14B/D/7/YFEXCZqUI8njNFhBNRBQfwnHrIkSlw2hBSQyDPeyTxBG7yO25S6UwDYSS0n6rz0wGNlbJoXJ0+vhfd0P6BGOs9C3XyIvv3K1t08QFMsI0ma49o7h2sObOO4w4B8oXunKAq53cBBMa4Xvp1GGd8Qr+EeH0yfAgrZhdZSNcFoK3h/8b3Xb5nsBqe2brU/K3YA0nfAhcET4PRQtvdsk9t+BGJtTAuraxL4TgDcdbnHwqeArNdJpm4GPTwiodRP7vuMHOvrVQt0ZBRTynqkFNiv38ytiP6wQBqz3g3Dau7EPiPQpRnK2jcRS8n5NpRg5FTnFfI1sKVcjR5ed0Sr5hnJNMtBqb2yZrj0IZefNqG4LY8HeaOX5BQrezfU6A8JkoG8Y5yY3cfzWKN3Gig633ALeo8F4NjiMdx3Bn2iw77fgmwbVX2L7zeV+H5sqjbldlxMzwt81gBle3xV2OlqlpJB0Ya4N/9eAdG7zEjtvBvqHnTVoGjkrsX+1xO/OdT+7ZNHeVdAK7r0sjq1omNmPzOwPyGA9FDEoD3H3+7wRKhV3n+fu/3L3c9x9W8SYcA5Si54BzDSzF83sMjPbPbhTRzQz3P17d3/R3ce4++7o3T0ApWrYB7gdCbFtgXfN7KdlaOYFwJ6BBLVZ4YoP2wm43Mx+1XB/iIXqAzzk7rOAUcAVwXU9K5hZDxQbONIVG9gfaY0yHd8GpZw5293nAjXd0NiVpCB5FtlBuqGYmRSuQmPfBKT/TaAm2zaXC5UqoDz8nQn0DB2UQi+UoK0xLBMpPx04CnXWV+GAvokL5dCmdJiFBuJ1s6+usmBmW5jZrcj21wG5u+7r7pPzqc/d57v7f9z9AhcNTDdk4J2DVrofmdkUM7vCzPYKjBMRzQx3X+LuU9z9CnffC00s9kCrmI+AiWY228xuNrMjzWy9XAbjPNv0NXAecGWpr5Xh+lOR88g5gU08icOB27wuiHYsemZ7ZVN3IGy9F5G7prLeNiqggEORCjbFVjFnMPpIm0obfANSYTyK4koaoGIYRTKhUgVUCs8D84EzzGy5QIuyG3BnE+dNRaupH/Ad6uHu4f9xaAXVCBoGQ36OovyXQVjh3YBmXWuYWVszGxz45yoWZtbGzHY1s8fQu/4aius6yd0/KOa13H2huz/l7he5+05IYB2DvKGOAt4zs9fN7C9mtl+YZUY0M9x9qbu/7u5/cfdBKL3ITESf8zMULPyxmd1uZiPNbKMSCZEbkDPCASWou0m4+zQUdH6UmY0yoS0SUOMSx32PVOFjAt9fU7gSeb9ekNiWUUAF0t/RwAkJTdLUGqg9H/nI34sM80upi/QFpaE+G3lgpBm4atE4Wdkot44xja71Q+Anif/7oI/iG6SG+EVi343ARYn/fwU87rCqQ+274G0TOtezwbuCdwM/GXwY+LUNvPgS+tla6jtBDAbeQTr7K8O2hl58f0aru29QIGRFerYhDeZIZHt4GdkelolzauY2pUu/nrSNrFnu59ZaC3K0GINWVEOR3fVINKOfjuzEd6Ng074UKRYOBcPOALqU8d67I5aLK5F97pUMx90N/LaJukaGMWzFxLbl0CQ8bWA74uYbV297GN8c/FbwrcA7ga+C4qCuBl+IYp/agS+fKCOjF1+FlCLHQbWEgsxso5Bh90Gkxig40LBEbW2LcmKdBPwdaWbfQzPrXwJrlbuNra0gr83PgPNJOMwAvZFn2/XI63YWsrGcFPqwUe+0Jq55K3Bxme+7Bjn+vA+cmOGY3uEd7Z1h/7ZIC7N+g+39UM6wdOdsGJ7lsmwfrWR8a7lcfCKIfZx6/g5ZYz6wHe4vFbVNZYKZ9UWJAfdC6tE/u/vb5W1VbrD06dcXUJ+x/T1vsS90ZSAQpN6KzAOHuPvHaY5Zk/r91IP6KUZe9SxTjIS6XkP5lsrmhBTuezrwT2AfT0Pga2bnA33cfb8G23sic8UR7v7PBvsORW7tB6apbyLwqLuPabhvttmgrvCotfTxrdwSsqSlxFx8lVyQyW1n9EF9igI3mz22pMT3txFSm9yOVKufhN8jw76KXB1We0Gr23PQamq3LI5fDcU/Ln6qAAAb00lEQVRc/QWZfr9BMVlnItV5o+rlcNz4Mt/zMcjccy9iVFnGBR+p+T9ETOTJbS8Dp2eodwzwmzTbf45U3O0T27oj7cEjwJKL4PmWPr6VvQElL3VCqqnl8JJq67y0HSrnnhGIMua/4XfVBwxncd9G47aRfkSewGI/8yFhQL4il3cMubYnabS+Rbb8c5GNq0OD4zsg1eEuZbzXF1Da93ZIzfwMUJPmuL2R80G78E7eGiZNaSdLyMFulwbbOiDqo58mto1B7C3zkO17CbBxSx/fyt6AZimwpcN9LsPg/AYdNz9sv8+rhEAxbUfKK+4c5HH1MFo9teoVBIrQ/2UYUN5D+vy/UwTbSCw/POOuYVXxKk1Q9TRRx26IP/DFMAg/jshsfxxWIbs1XFE04z32RWnd24b/26B0JVOAVRsca4jR5jgUDP0KaRwgUDD0R4jObRQwKLHvNzQgjkUBxN8F4eQof5b2t+DxreXaoNJByeF+ybIZJ2+iSjPqmtkGaMA9EA2+l3tMDJgWIf16Mptt3raRiDoEN/ORwIXIC/NmL2BgCa7V21DXT/2QMFgNqbdOc/d5mWsoLsxsDLDQ3c9ObDOUIuQgYCd3/yixrx+i/QLZe4a7ez03cjN7FAlf0GpoJvIUH4U89wa5XN1Tx6+L3tVUHO5p7n5Vg4a2vPGtVQmoFoJEYsBTkA7/apQYsNEkZxH1kSb9ek/qp19/yYucfr0lIwzMd6FVwzGeY0r1RupdHqkTf4FCSRYj9XWqn552MSwUHSFN/MfAtu7+bpr9J6EJ4s7u/k7YthswHq10FiNuw/ENzvs1cl3vgITYxUjgtUXefju7+xvh2L7Ilnxh2HcLymGVJMJpkYgCqooQPpZ9kOpgBaRmuNnd55e1YS0EadKvr0v99OsveBHSr7dkmFlnFA+4A8oG8HKR678M2bBuoq6ftkLqv1Q/PeVioyjG9fYETnX3oY0ccwQiSf4ZUq1fmthdCxzn7uManNMTua07cDwSVJehmDNHLEbdgI1RSMjJ7n5HOHe51jJxigKqCmBmKyG2hRR56xhggldAuu6WjCbSrz+JCD7zTr/ekmFm+yFmsd+jsIaivKtB/fcWsKe7vxC2dUBCKtVPg9F3khJYT3qeKi4zewB4wN1vaOK4fYFr0cQRFGi+InKWOM3dL09zzjzk1TgYrZ4uQI4QXyLVYVu0Ij3C3f+RT/urHVFAFRNmq5JeB3xjPjpgM1sL+L9Q5yRkXyrqjDQie4SJwjbUqQVTtpHUQPhsc9pGKh1mtjbKPvAVcHi+QiJNvYcDR6N068sIvqBpGECdwNoGqel+iJnLRh0eVMBvAr2aUlea2S6Ij7UtshGNQ6ugkcD97r53w/HhI2jfFZ5cAa4xefr9BK0+zw6/rwf2c/fHm2pri0W5vTRaRFF6+fub8KK530XAWu9c5KF0JAmPO+Sxczf6sC8Fepb9HmNZppA+/fpzKEfWcGKKERCVz2gkIIqSzh150T0PHJbl8e2ALamfYuRt4BqUk+lHGc47jYY0Q5mv8R4STEuQim5+uPe9Q3r2RseHafD8b6TGA62ePgO2Lnf/lbvEFVShMDsaqdw60jj57lLEfHAq7mN1qhlwHzL+DkOsyKegNCN/Bm7wIhmaI0qPQBY6iGawjVQbzGwnZDcaB5zvBXpLmtlA5LW6kefoIBFIX/tR10/DUPBwkpVkOlK/HePuTzZRXx/kFn8m4g49CHmI/solpLIeH26C8YerTT/16I0bBVRBqBNOudCNzCcIKTM7By3nOyHh9Vqob3yhH3BE+RFsI1tTNxAOQjPtpG2kxXtipRAY6m9GK8+D3H16gfWNA7509zMKrCdFo5X06HTEpn4K6qtpnmawDGrMx4Dz3P2WxPa+38IOXWSDy3p8mA/+Ifx2E/eLCrilFoMooBIwsw2RUXJd4Bx3v7KRgwvi+jsG/jBWOW/ahW2LEI/XtNAWR8SS0zJVElFdCLmAUraRYdSxdT9JDraRakYQBqcCp6PVyX0F1LUaWuUM8eDiXQwEzcbtaNUj3jqtfpIrrLeQveldFJ6wqyd59iIXaFEQBVQCZnY9MNfdT87i4PtRYrd8cmotfQgW7S6X0jbImNoWONfdLwltiQKqhcPM2qFcQKlZ+1DENJ805i9DxtoSYGZbIweKfwGneJ7ekCGh4I/d/edFbFsnxOu4mbvPCAJrHeoT4HZCTlC9kYt4LXDmD5PaMD7cCW3+hKTo8sDayEviGCS9RqGgsa6IMypgKTAe972LdU/ViiigEjCzR4A73f26Jg5cFemoO+Zzne+BdlLp9TI5QnRC7+/XHuIbooBqfcjGNuLuH5atgUVG8Ioci6iEDnD3/+VRR3vkKXuqu08oUrsOQg4YGdPdm9mJiFpsFeomqQ486nK8mD4GOl6KGHJ3QXrNKSjY6Qakz38bSbZLqCegIIwPVCkDRNFQbi+NSimIP2sJejHmIffuV4G5SA1zQerYJ/U++Q3gPwKvAf8b+Avg/cBXAj8u4akzDnwI+EngK4OfE7x3jpM68U3kVfQwiVwyJJIhxtI6Cxr4+iJet7sRi8B0ZMc5EpHjVjXfIuKuOwLF/hyVz/0gEtd3KRIpMqJT2r+R/QNCe0eg+eYcpKL/HHjM4fSvYX5n8HuzYBj/N3jvZbfPd8VPlb2Pyvp+lLsBlVTQqvtX4ff2BAZsFLfwOQoOZIqSsflI8Frwh8E7gO8B/jn4x+DdwR9PCKi24FeCLwafDz4evIeE38bIDnUuiqNJtSUKqFjqFRpPMXJ0eJeqUmCFtr8WBPEyLOFZnP8gadJW5FFPb0Qq3DHD/h6I5HUvpLE7B9gRWOGH4+CWSeGbX5y/gHIXS0zZ+6acJR/7SauAuz/u7v9196XuPhXpy7cDaKsIcc5DOr6dkX7uQOQnviYyJryaqG8N4AQkiTohvcaJ8L67v+ny2LsE6G9mvZvj/iKqDy685e5Xu/tBwI/QO/ko4qr7J/CZmd1jZsebWb/glFDxcPc3gYHIBveqmQ3KsYpTgNNDYsFC8Euk5l/QcEdQJ96HYqPud/cP3P1id3/U64eD1MxCur92iY1DUPR+J+QVkwVq8ruFloOqeHnLATMbaGaPmdmXZvYNmqGuArBEKx96JI7vlOb/JKVAzwb1TwdGQR8zm2Nmc4DZaIa8ZnHvJKKlIgisae5+vbsf5u69kVv7QyidyN+BL8zs72Z2kpltHuxcFQl3X+DuxyNh84CZnZmtgHXZaq9Bbt15IVxrBIrVSocr0erqd01UNadbODAZK/Is0gWm6MizwJzsDmu5iAIqM25HaoOe7p4y5hrAXLmY5gRr8P+asPR4uNvdaxKlk7s/W3DLI1ot3H26u9/s7ke6+3rAZsA9SIV2BzDLzB4ys9PMbKvgSVhRcPe/oyDnnwMPB3fybHAJ8GMzG5LnpbdDyRNfabjDzEYip5XDvGlewamDoLYD8ECeDUG+E1PzP71lIAqozFgBmO3uC4JL7EGpHddrmV8Qjobvr4AtQhQ6ZrZSIJyMiCga3P0Td7/d3Ue6+0ZIUN2C7CfjgK/MbFJYrQwJaqyyw5VfaQe08HglcN01dc48lOzvyjxXiiMQe0s912YzG4o8wvfw7FgrbuoKdj5wLMrm+C1aNU1BWQehjlpmMTI4L0CeFqnLIuaNVo0ooDLjWGCUmX2LmIbvTu24Sa7heNYr9WWwdG/4x/ea8d1pZnNRqMTPCmxzRESjcPfP3P1udz/O3fuioPRrgdWBvyKB9YiZnWdmwwIbRrna+r27n48mh9eZ2aVZCNDbUZbaEdlcw8xqzOzHZlYD7A7c1mB/T+Rte5inyQeVoeFfAJPOgKWXIzLNHqGMRESNQ5AdqhMibfwo/N5ZNSwFJtLaXcyJcVD5I0aKR7RAhBQj21IXi7Ux8BJ1sVjPeRnyj5nZKmjF1wPFTL3fyLFbABORmvBY4F7PkAXAzPZAXrkLkEfkrl6XeLATymJ7p7v/MccGx/GhCIgrqHzh/iKibMn1Y01x8bX6ly+i8uDuX7v7Q+5+mrtvhRxQ/4DYEi4GvjSzp83sYjPb2cy6NFO7ZlG3wnnezA5s5PApyE78LqJUyphsEIWPfIsEyTrA22b2DzNbAa0s30axtbk2OI4PRUBcQRWKAtjMIyKqDUEgDaZuhbU5Uk+nkjg+7e7flLgNWwB3ohQnJ7r7d4l9bdCKb2PqmF4udPffZqirFxJmnZBwmIpWXt+glBdbFbRijONDQYgrqEKhl2k7YDx6wRpyitWG7ePRsj2+fBFVC3ef5+7/dvdzXWnQuwNnoff8NOATM3vZzC43sz3MbOUStOEVxObQDnjJzDaDH5jFhyPh5aFA/QiQhvgMCY/vgT8h1+62wMooaDdfj8BUY+P4UADiCqqYMOtO+oy6N0WDZ0RrQCnTr2e43iFIsFyEKKF6oQDmlZFzQ3/geXfPGPhrZguAMe5+TohJXAmtaBYDd7h7Vg4XWTQ2jg85IgqoloQip5yPiCgUxUq/3sQ11kfu6F3RSmisu58U1H2XoVjGfTN9H+/ALRu4f2Zm6yG71WKUvv3CsFqLKBOigGoJkMfQWchN3ZE+PYVaFFMxCRgdjLcREWVBKVKMmNluKDZxubBpIbC2u38aDsjq+7gPbtwHfg0cG+KwIsqMKKCqHdEIG1HFyJBiZC71kwN+2DB4tkEdRyIB1AtlJOgI/Mfdd4zfR3UjCqgyIKfMvY1X1GTK+Y9QLutvkOWXRMr5vK4ZEVFCJNKvJ5MDtkOC52wapF83sx2B69Hq6B3kKj4UMNeKqtHvIw3i91FBiAKqDMgpc2/mStIGAq4FXAf8pPGzYyBgRFUgZLM9E+WMeoFl069vi5gm5gG/cPdHw4kxULYFILqZlwe9gbTZQ3PgEDuLPDP6hvPOyvPciIhmQ1gpfQp87u4Ho8QAQ4F/Iw/B/ZENaQVgkpmdH06N30cLQBRQzQwz+w8iwbzKzOaZ2e1m9jczm2hm3wE7mFkHM7vMzD4ys8/NbGygXQGgr9nBm8GeNdBmCHWUx4cild5uKL30pSiNtFFH+789cC60GQx7ttH1HzKzbmZ2m5nNNbMXzWytRHs3MrN/m9lsM3vbzPYr5fOJaL0IhLXvmdm3ZvaGmf2i/m67CnneTUAZa06kfrr15YAL1jI7Can82tyAIna7orTr05MVohQF6yOXvuP4IXCqDTC8p9nJZvZmoj1bhIasYWb3hVQ8H5jSv6caubWZvRS+pc/N7PJiPqNWh2JkPYwlt0L9zL03IhPRNujD6IjiOh5EsRwroPw+o8Pxm3eGec/Cgu/BbwzZOBeELJy9Q4bOVFbOD0LAYiqz53bg64K/A7UfKovvG0h3/xOk678ZJWQD5WGcgYg32yHWgFnAJuV+hrG0vALsi6iV2qCV0XeIxPZwNMc6GQmh/cM3sxlisViEKI3WBVYfD5c5zB8f3vU3wvt/IfjgxLcB+M/BvwafDr4K+KSw705Y2EXX2ArJsvWQ5qMN8DIikG6PbF7vA7uEe5gMHBp+dwEGlfu5VnOJK6jKwAPu/owrz8xC5Op6srvPdmXqvAQ4IBz76/3g/cHQoS0K6ugAPJfDxUYA60PH3rABcj9/z90fcWX2vQcJIoBdkQfVOBez9KvInTemBYkoOtz9Hnef6cpifReKSdo67P4C+LO7Lw773kbxTAORzeh9YKa7f7qHmCM6jUV6uo3R7OpsRNKXXEWdiVZPvZBaY0rYfgO0Hwlvu/uLLkxz9+lIYHV391HuvshFWnstdd/nYmA9M1vFxbqRy6cZ0QBRQFUGZiR+d0eG3ZcT2Xb/GbYD9L4DNqlBH1ZNOHlmDhdL8L7UoDiQzxO7a9HMDzRjHJhqR2jLwUC2CeQiIrKGmR1mZlMS71pfQhZr4BMPy5KA6cAaLh6+/VHG60/NbMILYpJgOvB/1H0nK6Nl0yeJSpIvcmfqsmDPAPrUUSUl0RtYo8E3cTZ1n9WRaOL3VlCX75r7k4hIoeKyabZSJD+EWUhI9HH3T9IcO+Nw+O9YBTsug4aZe5tAUymlZ6DAyZ1yqzYiIjeYWW+0EtkRmOzuS8xsCnWv9JpmZgkh1QupwXH3h1Hm3U7ARSPgkP8hb4pz0IwqV/QE/pf+c5oBfODu66c7z5Uz6sDgLr8XcK+ZdfMEoW1E9ogrqApDUPNdC/zJRM2Cma2ZyCh67R2w9mRY6EhJPwHlCwBN4zImyqmPbFJK/wPYwMwONbPlQtnKzDbO6aYiIprG8mii9iWAmY1AK6gUVgVODO/gvkhzN9HMegRS2uWRenxerRZCtUcDo6lzl/0G6a+zwRGw6Bq9+wNMWC8I0ReAb83sN2bWyczamllfk1s7ZnaImXUP33FqAphvYtNWjyigKhO/AaYBz5my7T4CbAjg7i/1hpEnQPuuyHJ7Y+LEsxBrZg1NJrFpMqV0sH/tjPTrMxHzcyo3UERE0eDub6Cg2slI5dwPeCZxyPPI4W4Wyku1j7t/hcawU9D7ORvYrq9Ms/YL9CEdAKyIpN2kLNuzPyxdUZ/S7Wj+Nx5Y2d2XINtsf+CD0J7rEMEswE+B/5nZPOAKlFyxIYN5RJaIgbrVCrP7gT3Ib5KxFBiP+97FbVQaRALb1odK6PNq+T4iGkUUUNWKSo+UjwS2rQ+V1OeV/n1EZIWo4qtWVHJKaXEEPo5msB2pP1AR/u8Y9j8ejo+oZlRan1fy9xGRNaKAqmaI0DL1ETZliF1KcxBh1iewber9ahOOGxOFVGXBzD40syYoHX84OKs+vxHYtpE+N7PtzSynVBuNohK/j4icEAVUtaOSUkpLrZIrezTUDVhbLlulrWVmHvIIRVQaCuhzg79dbrZnCVpVh0r6PiJyRvzoWwKkjti7AlJKF4OgMxqmqwuF9DnbK8B2fNFakw6V831E5Ipycy3FUl0F2AJ4Fbne3oPyWl3ksOpYWLQueFfw3cA/SfCePQO+JfiK4e8ziX0J/sBaF2PGBcCt4XofIYP7vFAGl+TeYFWH0x1ucXgo/D3dRWtT9ufe3O1EPMOnoQH8m9DPHcO+XYEpBt8MgiWvJfpyNPg64F3ANwa/P7FvHPg24ffQwIXXGdzUr/sjLuOPkVruC8RiPqLszzyWspWyNyCW6imIHDPFILMcipRfBFz0ZxjbDfzlQFx7fBiEHPwr8BrwmwNp5+3h/1nLCqj5rkExKaDWCgKqXUnuC7ZyuD8Ix/meGFDD/7Vh/1Zlff7N3M4goF5A5K0rA2+i1c7mQXgMXABnXA8Lk2TFd4eJyRIRrnpn8JlpBFSKrPVttfu0cM3tESnsqPB+DUd2oa7lfvdjKU+JNqiIXDAIqYWvdJF23o8GMR6GnxyBllcdUAT/ZDTKTUARloeGkw8ENkIU7Q3QCalfmgeV5nmWCeVr55Uu8tbZqLv6IyLjq939+Q7Q7whonyQrbkhHvj7hBcmANmp3ss8XA6PC+zURra42LNL9RFQZooCKyAVrsCxp5wyA2bBi78TGLkA3RMw5EzFsJtGb+qSdCdQUqa2NIwvPsw/5IZdW+bwNc/SKvAA6HwRXFamdnyV+z0fd2hs41czmdIEDGpIV34ykWIqg9XVEtdAEkn3+lYtVv+F1I1ohooCKyAWfEkg7E9t6AqwMc5NpDL4DvgLWRFItuQ9kWFoz/F6eesEqc6hPMl1QJHlwl64NSefmmNmz25qNXlJkb8MC23iWmU1qsO3dVcyeJdHO9YE7s6ivDbQtRTsDZgAXu3vNPLhzDuq7A1EfHwVchfo+RUeeRQc2RVoc0UoRBVRELpgMLAGON7N2ZrYHIV/PTvDoOJRPZyHKPzAQGZCGo4yItyMDw10oS2IqD0F/NPAugtpbxae2T+KaX6IYlXUKaPdu7r4Cmv3/fhqceOSyarJsUYp04E8CQ8ysLYCZrQ4stwS2+D54yH2KyBmHlbedICLjo81s4BKYOg9qU2TF36EVZyovzDi0gsqEHsA0vS5NkRZHtFJEARWRNdx9EXKMOBLNeg9BjOcLT4bzfgeL90YpUN+jbrbfLRw0Jvy+NPyfSvRzYTh+Zeh0FPRBsix1zfmIHPSZsAIaVED7v3F4bjy0vRnsdWQf2xyRifZE3hmZMBsYAW3WgL3aqC0Fu0eHtAzLI6eAVAqVoV1gcj9Y7rXwjT6F0sWugTxUeoY2Dwj70qANMHxrs13N7Nnw7GaY2eGFtNflsn0UcNVycO760OnGsG8T5H43GAmf/6I00ZlwATACOrSB88xsv0LaFdFCUW4vjViquyCWabkCy4tsSQMPs2zLEof7StC+D4Gf/LBNLtnze4L/Ffwx8KnB6+w18FXB/x7a9EHwNFsc/h8Ovp+8EufPgzOA7YrQvo3QdRYgE05b4Krj4e7fwOLLw7WPAx8Rft8SPCAXg18G3gO8Nuw7H/zguvbXtle9ByIB2A3oX9RnXIF9HkvLKTFQNyInmNl2KN32LJQLblOU8RfkvLcL+RF0LgjnlxqbAp3WQCui7RvsOBB4AmhIb/ApYjn9CugKnf4LuwOfmdlhBbZndXTvHdHz/Dmw8FB4YTa0uxo4Ga2STgknHJI4+VSUE+JtYLMGFd8BHQfAjGfd7wibvgqlmKiGPo+oUkQBFZErNgTuRmqp91Fenk8BcH8Rs1PJ3QGhOQk6a0AehCuj5d+ZyFayCBlE9k1z0oxwfNfw/3zZs7LjqmscK6FVE0gt1xVYtDW0nYtoD2aH9qXsT5cB1yPPOQPmkt5Tbgawnm6rdKiOPo+oUkQBFZET3P0a4JpGDhiLnPzGoFVBY3bOpWgW3ZwEnXNeRAJqW7RSOh6tjjoCJ5F+sO+JBMUcJOEGwmPuXujqiZCdeDh6Du2R/FkHWLAisjldG/6ujVZSlwKPImNdSqKl85TrCTykOkuLyu/ziCpFdJKIKD4qlKDTzFa8GBbvD34IStn6LVoZdUQBpbdnOHd1lOToWGA21H4Hr5tZDk51GTEDCaU+aEF3OJJDU4HabYHLqVs9fYtmld2po1yYm6Hig2DBy7Cqme0XvC67mVn/DIcXhgrt84jqRlxBRZQGlUXQ+ZCZfQ8sHQXvXAbfHyunAf6K7DjHo9F1PzIH5dyC7EEbQ6cv5Un/H+QinjfcfR7BHdzMnkBOcE8jTrpRQ1FcUUpA7YJyim+AdKwnEwLR0qA3sCns8xKch9KSfwOci6IBio/K6vOIFoCYUTei9aFa0oFXSzsjIkqEKKAiWh+qJR14tbQzIqJEiDaoiNaHakkHXi3tjIgoEeIKKqL1oo6ItbI9z6qlnRERRUYUUBGtGyJUPQu5ejv1OfpqUajRRGB0WVck1dLOiIgiIgqoiAigajzPqqWdERFFQBRQEREREREViegkERERERFRkYgCKiIiIiKiIhEFVERERERERSIKqIiIiIiIikQUUBERERERFYkooCIiIiIiKhJRQEVEREREVCSigIqIiIiIqEhEARURERERUZGIAioiIiIioiIRBVREREREREUiCqiIiIiIiIpEFFARERERERWJ/w/iGz3iP6+CYwAAAABJRU5ErkJggg==\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "sm.remove_edges_below_threshold(0.8)\n", + "_, _, _ = plot_structure(sm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this structure, we can see that there are some relationships that appear intuitively correct:\n", + "\n", + "* <strong>Pstatus</strong> affects <strong>famrel</strong> - if parents live apart, the quality of family relationship may be poor as a result. \n", + "* <strong>internet</strong> affects <strong>absences</strong> - The presence of internet at home may cause student to skip class.\n", + "* <strong>studytime</strong> affects <strong>G1</strong> - longer studytime should have a positive impact on a student's result. \n", + "\n", + "However, there are some relationships that are certainly incorrect:\n", + "\n", + "* <strong>higher</strong> affects <strong>Mother's education</strong> - this relationship does not make sense as students who wants to pursue higher education does not affect mother's education. It could be the other way round.\n", + "\n", + "To avoid these erroneous relationships, we can re-run structure learning with some added constraints:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOydd5hUVdKH3xpAiYoBA0owrRkT5gCmNYuuOYfPwJrzijlnzLumXQUxB1bXHFYxu0bUNWdRVlEUJSoyv++POs00Tfd0mO6+3TPnfZ77TM9Np+693VX31KlTZZKIRCKRSKTWaEhagEgkEolEshENVCQSiURqkmigIpFIJFKTRAMViUQikZokGqhIJBKJ1CTRQEUikUikJokGKhKJRCI1STRQkUgkEqlJooGKRCKRSE0SDVQkEolEapJooCKRSCRSk0QDFYlEIpGaJBqoSCQSidQk0UBFIpFIpCaJBioSiUQiNUk0UJFIJBKpSaKBikQikUhNEg1UJBKJRGqSaKAikUgkUpO0T1qASKQmMFsA2AfoB3QHJgBvA8OQvk9StFmoFzkjkTJgkpKWIRJJDrPVgSHAFoCATmlbpwIGPAKcj/Rq9QUM1IuckUgZiQYq0nYxGwwMBTrSvLu7EZgGHIt0bTVEm4V6kTMSKTNxDCpSt5jZ0mY22swmmtkRzezX28wmmVm78P+o/zMbgSv9zuT/HTSE/YYGY1E15jIb/zhcRglymtm1ZnZqxYWMRCpEHIOK1DMnAE9LWrm5nSR9BXRN/d8duq0BOwNzFNleSvm/hvRavp3N7AxgSUl7FtlO6gSrzwPzNLj7rlmGAX8Hnk+TUzCgEDkjkVol9qAi9Uwf4N0SDurdAB2KPU7ADHezDSn22BIZYgUYpxxUU85IpCJEAxWpS8zsKWBD4OrgvjvSzN40s1/MbEzovaT27WtmMrP2mC0wN8ybUvxnAOndmy/wDb+H/wcCJwPr4t2Sz6FhAmzZzexWM/ufmX1jZueY2Ynh8yQzm2Jmk4HTgd3DureCLF+Y2SZpsp1hZrek/b+XmX3ZYPbj2bBNav23of3xabK+AfQA3gEGAy/h3cTuvrlhbxg0n9ml4bwDzexrMzvBzMYF2bczsy3N7CMz+9HMTkqToyFc06dmNt7M7jKzeYt5RpFIS4kGKlJZzBbA7HjMRmD2QPh7PGY9WnJaSRsBzwGHSeoKvAXsjevnrYA/m9l2WQ7dp9iwoBHA9cBEvMu2D3RYCRYHlgRWAbbF3Y2rAw8AfwVWAs4FnpDUVdJK+doxs+WAa4C9foKLxgNfh20L4cbyrgy5dgVWBK4F1gYm4XHnKdbxcPQUC+E9q0WA04AbcPu8GrA+cKqZLRb2PRzYDhgA9AR+CteVnwo980jbIxqoSGUwWx2zkcCXwJm4Itw6/D0T+AqzkSF8usVIGiXpHUmNkt4GbseVayb9Gor83u8LLI8P2P4IPArtHobPJE2WNA64Ee+8LId3vubGo+l+B4qZm7Qj8KCkZ+eG5c+F9umC7gOkuloz8Avcq5mTNUC7+WCBtFXTgXMlTQfuAOYHrpA0UdK7wHu4YQXvlJ0s6WtJv+KdzR3NLPe4dZWfeaT1E4MkIuUnf1h0ag7PIGAzzFocFm1mawIXACvgwQ9zAndn2bV7sefulfb5S1zL94SdJ5ttFVY3AONwJb4C7onbNqz/tIimegJjUnJ2AeZL2zgItxqfAx/iVnCNPCfs4M8gxXhJM8LnqeHvd2nbp9IUTNIH+KeZNaZtnwEsCHwzW0MJPPNI6yf2oCIFE8ZxlsyzU0pRFRwW3Qf+dr7Z5S0U7zbgX0AvSXPjXq9sAQbpHjC6AFPS/v82ywHpJ+mFW77xcIek7mGZS9KiktYDegOvAE/g7r41Mu7ZZPzepFgo7fP/aLKHE6Yw65hTRzz08BbcvZfee8oVSTHde3KlMAbYIu0au0vqKKk545T1mX/BLON6WUP2wzjd4iXKGmmlRAMVKR/uukkpqsIPA1sNDsasfwta7wb8KGmama0B7J5jv7cbfUIrACsDzwJfAT8D5+dpZGFgE5ixBfQ1s7lCMMEmZnaUmc2Jj38Z3kZqCCl92Gs0sKuZdTC/3h3Ttt0DbG1m602Ed0+B39O7L+CDbMNwS5xuoBYMjf2Wtq4RZoz3nl2KhdIDNPJwLXCumfUBMLMeZjZotr1KeOYDgb83Gan+AGGc7rNCzxFpG0QDFSknQ5jVpVQw5m65loRFHwKcZWYT8QCAu3LsNzy9t7EpsAseSbAaPmCSjxEw/TX4BB+z+Qm4CjgM+AEfjxqEG569gc+A18zsjXD4qcAS4bgz8Z4fAGEc6FDgtrnhhHmARTPaXhf/0a6K++BSbISPky2EDyyleNHz9OUly9jSFbgdfDzc05eBNbMcWvIzJ4bCR/IhKS5tdAH+go8nTMSHNTYG2gEn4WMnE4HXcbcZeE9gMPAx7ir7KyFd1new4JkwvTeoB2gv0ASQwnI/aDnQ3KABoPfStvUBPeGfp64LfwReA37Bx0cuDW0PBL7OkP8LYJPw+Qy8B3JnkPsNYKWs1w4jBTOUJkMRywzBvVV5Rjnk3BB0Q5Fy4l7BRnycaRIedSjg//AO5LNhv7txT+fPeOdy+bB+zbC+XeqcS8O+K0CjQDNA54MWB80L2gk0Psjyubej6aCTQA2gOUFdQH+G3+XR8sInNYN3Ev+G5xacBLyA297LccP+AbBK2rX1BO7FA1I+B45I+rcVl/IsiQsQl4QePCyNjzP0DP/3xd/sj8en1iyNu6pWAuYL+wh4EA806B0UwuaSOAzuWhwaPwVNBG0P2jMoqA9BnUGPg34DXQhaAvTr7AZqSi83OnuF9roCa4XPhRio6XjPpQNwXFBWHWa7flhdMLlEAzVZ0L8qzymLnK8EI/9LCXJm3K++4XnejA/FdQrr98fdpXMGgzA67fhPgU1T/68Eb50Dvwl0OWhN0BjQNNBBoF2zGCiFF5Q0AztF/qwyDdQPeKe2I/BUeJZ74y9Q5+AZRMA7lK/jveY58PD/z4DNkv6NxaXlS+ICxCWhB+9zeMYBm6QrcbwnNSjHMQLWS/v/LuBESfSD//01TUF+AGoflNJZ4Y06tW0GqCfo6dkNlJbzXtOZwPwZbRdioF5O29aABx2sn/UewOASjNRkweCqPqs0OfcGzQW6qUQ5cxioxZv5jnQP+8wd/j8HuDF87jYn/P5FaHMZ0JNpMoxNe/55DJTkRjLTQN2QJsfhwPtp/68ITAif1wS+ypB7CHBT0r+xuLR8iWNQbRRJnwBH4Yp9nJndYWY98Siy5kKj0wPdphDCkidA1z5pG/rgUVvfAWOZdbykITQyezgYjID/An8APjCzV82skGGhFKkQbSSlghR6Zt3TQ5yPDdeQGYswC42uPKeQRJbwNDmHQ+PP+LysbMwAGovPZj7znplZOzO7IGSP+AU3aNA0rHUb8KcQDPKnP8DPqef6JbA9btG6A8viXZ30GPZmyBb6nxn+3lw4fE8zm5BacBf1goU1HallooFqw0i6TR4a3QdXwhfiCmuJYs/VHSZ9mfb/V/gkuwVxC5G+TaGRRbKcZ1X4RtJu+ATTC4F7zKwLGeHZITN5ZmaCXmnbG/AYg7E5hXYlPgC4D1fsUzP2mCr49UH47RrYv+rGKUUBcgLT3oe3NoXRBtflOlOedbvjAR6b4NOs+ob15mLoPfxRbgHsvrUHigB+4x/BByZTyzSyP+MsIfETZl9VMGOAzzVrOHw3SVu24JyRGiEaqDZKKFWxUXgbTim9Rjwp9tlmtpQ5/cxsvmZPBqwHz1/q7hwm4a+wu+BGamfgIeDf+CDRUHyAY53ZTzP1GGhvZj1CDyiluBqBj4COZraVmXUATgmnSWc1M/tTiEg7CvgVjz7LjfQa0g74mNppeDDBA+HvaQa9BsExh8BgMys1cWvLySMn0HtFWP0pmAufJJyN7/Axmlx0w+/ZePxl4Lws+9wGHAlssDc8TDCWg/GchakXke+B+3M0siA+SBSYSoGRhjl4BZhoZn8xs06hF7iCxWwVrYOkfYxxKeMCCwiOF4wQPBD+Hi/okbkvHln9Ch7x9iMe/NAT98ycgg9KTwReBRYNx8wcJwj/DwPOkTyK73SYvihoftAeoB/TxhlGgpYNYygbgP6bti1tDGpqF48iG4fbuXeB7dLa2xcfVxqHD6x/Qe4ovjeBVctxX3E7+xawU+LPOL+sm+L6v2OWbYPwzu0EmgIT2qdt74rblYm4rdk7yzPvjb8wPBS+b1NT44pDQX8AdcWj+YbkCJJ4EbQUqDvo0NxRfOektXkAMCrt/yWB39P+74lnfvoWj/J7OfW9KPX3EZfaWGJF3dZArZQD9zxsgyitZ94I3If3EkpouoW1l/KffwA+mL+spCn59k8S8+fwuqRzq9BYYs+8YGrl9xEpmujiq3c8XcwoXEl0ZNYfH+H/jmH7qApXhD2fElPr/O4xFfkSOSSGpGfwihYnJC1LARwLHG1mmfN8K0HJzzwcV9lnXlu/j0iRRANVz5SQ945Kli33t89UZFzBNMLU42CywVoVkat8HA8cbmZ9E5ajWSR9jpftuKgKjZX0zGmKiqxcxd9a+31EiidpH2NbX/AJsaNxv3/hM+BrebJp09ydfNkaZqTm7ACL4eHtJxOyU9TigqcquidpOQqQsws+3pR9HlgNPPMKy1PU7+P0MG5ald9HXApeYg8qeU7AZ8V3k3RlEcfVbg60AsOiw/YBSNfK3/rXA3YDLkw0Yq55LsGjBTdKWpDmkDQZ7/FdFULyK91g0c+8XE2HKsVTQ0b0SWY26WOf7F2bv49IwcQgiYQxsyfx0g1/L+KgBfAoq1J/gODKojdSMQX1iserqO6DRw12xyPI3gaGZ2s7hLQ/gkfhHaKm+kU1g5ltD5wNrCzp93z7J0Uw8qOA2yTlmhtViYaLeuYtb86+AA6Q9GRYUfTv4wx8UtctTauq8/uINE/SXbi2vOA5xmbgP4ZJ+PySN/FEqWOAM9L27YtHIO03F0zoDroGz822Ip6f7dA0l8XHIZx7LtB8oJ0zQn5/CznQwrlH4T9w8FDuF4Cr8YShHwAbV/m+dAOexkOHZ8+ll/xzM+BJ4PCkZSlA1pXw+U/zJi1LBa/xC9LDyj10fMpLoLXDb6MfTam1BPos/D66gjYJv52Ui+9pUE9PgntczjbiUpUluvgSRNJGwHPAYZK64nNt9sbfOrcC/mxm22UctubX8NCd+EzUc3FN+S6eGO+ZsNOpeFrwn/B8P4dnnMQ8eqlfDtHWxMeD5gdOB0aa2bylXmexSJoIbImPo/zTzDIjrxJFrrGOBE4z7y3ULJLewjN9n5m0LFWk3zfQaSt8Qt+PuF92B3wCMXjKjNXwjLSnAsMzTmD+EpLr9xGpEtFA1RCSRkl6R1KjpLfxHsSAjN3O7gZz/RHX3qmcQIsA6+PdL/B03l/ieX464oM7WchV/nwccLmk6ZLuxBPIbpVj34ogaSquU34BHjGzuarZfj7ktZtuxROo1jqnAruY2YpJC1JB7kvl4tsANrkFf8PZEldymwL98dQXX+Gzz8/GU5FsAGyT/Zy5fh+RKhENVA1hZmua2dNm9r2Z/YxHt82fsdt3hBRAnZg1I2Yn3E8IHl8sYA28kN2N2ZvMlQPtm9BLSPEluZKuVhBJ0/HCse8D/zazzHuRNGcAg8xslaQFaQ5J43FZr6jh4JOWsp1CLr5n4ckv8ZQk3dOW5/E0JGOBefAXvBR9sp+zJTkCI2UgGqja4ja8imkvSXPjZbezKZS3mT1KahYWAm7Af4zX4eVmP6HpRzl51hxoC2UcvkiGIutNc0lXK4g8SOIQfLzumZBxvSaQNAHvnVxVB4r/evxlp7JZG2qDt3vC9L2YNXntZOBEYGHc9T057YCv0j53YeakrrchZ2LiSBWIBqq26Ab8KGmama2Bu8qzMZzshmsmd+NjT+Bvi4Y/7B64O3AEtP8cRpjZ/syevXwB4Agz62BmO+HVEx4u5YLKgZy/4IlRnzOz5hKeVpsb8c7rbkkL0hzyaMMjgKFm1jnf/nXO8L2h8QHgMZqikEbhv4k+uLvvdOA3vGf1QNrBfwj7Lw/fNJOYOFIFooGqLQ4BzjKziXiG6ruy7iWNw0Oxc/IqHunQFU9tfQVNaayvh8ZzYPriPra0PPBixuH/AZbCx5DPBXYMbqJEkXQBnhngWTNbPml5YGYP7wjgIjPrmm//JJE0Ck+kWg/pmkpHGtcbHr4PGs/DX8p6ARfTVPjrNvxLPi8ePbJ32uFzQ+Np8Pp7cCletmwyTe97kSoS50HVK54AcxRpNZKKYAo+WXK2NDNmti8ecp4jriJ5zGxPPDBra1UyVU4RmNkIvLLryUnL0hxm1ht4A1hN0pf59q9bKvT7iFSX2IOqV2o5B1qFkXQLcDDwcMgyXgv8BTjYzIou9lhNJH0FXIkb+NZLG/59tCaigapniihbHrYnU7a8Aki6Hx/3ucfMqhoCn0OesbjSvzRpWQrgYqB/radrajFt+PfRWoguvtaAWX88d9iW5K538zBe76ZVvRma2Vp4kb0jJd2RsCxz4nOmD5X0WJKy5MPM/oQPv6yiGk7XVBba8O+j3okGqjVR5RxotYKZ9cODRs6UdH3CsmyN96T6SfotSVmaI4TFPwHcJ+nqpOWpCm3091HPRANVTjxJZbYfwLD4A6gsZrYUrnCvlpTY+EpQ/A8BT0qqaXdfiIR8GlhO0g9Jy9PqifqhaKKBKgexpHRNYGa9cCN1D3CqEvpym9nSeMLdFSR9m4QMhWJmVwBzSopF+ipF1A8lEw1US2mq2tmR5oNOGvH5f3EQtoKE5K2P4QbiSEn5BscrJcfFwHyS9k+i/UIxs3nwVFJbSHoz3/6RIon6oUXEKL6WUEBJ6S2YmSl5tpLSZtbXzGRm7ashbltA7irZCFgFuCnBe3s2sHnICFKzSPqJ+knXVF+UUHLe4JqHzU6tuGx1QuxBlUoZJgKaZ2r4HK951LojqapMSOczEneh7Crp1wRk2Af4M7BOUj25Qgi55l4BLpF0e9Ly1AuhUGJPoGf6GJ6ZvQms/ClMXXxWd17+cwIfwtQ/wAYxojD2oFpC7ZZcjyBpCjAIT8X2YEJpiEbgOmevBNoumHpK11SDfE5aHsZQ0qQzgJWYvy8cF/UD0UAVjJl9YWZDzOy9BrMJ+8C206DhJ2BrPN/XPOFzetKugUCqlvsM4DhgfmhYDLbvCbtU8RLaHKHXtCtenfjxMN5SzfYb8VqR59daPatMJL2A17uMirE4RjBrKr99eniBSAwafsV/873x0jiDmbUMwcV4dvWeNJXEMdfLW85p9oKZHZDa18z2NbPnK3YlNUg0UMWxB7DZh3DpR2Dn4COb++EFk77C+/OH5Tj4BuBBvKjgazCtIxxYBZnbNMF1egDuwhplZgvmOaTc7b8CPIqP89Q6dZGuqcZ4GZjLzJYNrtJdR3qkHuDlPT4CRuPlbr4BzgrbHsUnzD0BfIxXxk5Di8xeBqfNEQ1UcVwtacxSsNSp0HA7MB9eYKczXivjZJrKrmdyF16mvZcf1+lUdw9EKkzoyRyNj0k9FxKmVpMhwL4h/LxmkfQNrjOHJi1LnZHqRW0KvL96qH8ovAjXZXjW9G7ASUAq3cld+MvtCngNqjNmPWenLrPWVGyTRANVHGPC3+598Ap+U/CspX2AufDy0RNwd14mY3HjlGL5eP+rRqgpdSbwN9xIVc1YSPoOuAC4vA4i5S4DVjCzzZIWpI4Ygddu2xe42UKp+O9x/bAaTVV9Nw/rYXZ9kFnVtx20+ejeqCCLI/V9mvAV7jceihdV+g/wC/Bs2CFbbOTCNFk4gPfzJ7CMlBlJl+M56J42s5Wr2PRVwGJA4oltmyOM2x2Nl4efI2l56oFQtuRzPNffSIVS8fPjLv93aarq+zMwKRyXqQ/Sq/oCdPB5UelRwm3O5RcNVHEcamaLfgwfnw2NuwAT8S9hd+BHXPPlYme8zsHXvu/Us6BvpQWOzI6kG4EjgcfMbJ0qtfkb7uG9PCSVrWUexBVuruHUyOz8H7CRpMmN8A64cj0Qt/bjwk7f4LPIwfXBMOA9vKeVoTumLggfAH8ys85mtmRoo00RDVRx3AY8vjQcuwToFFzjTMXfltbCu/C5OBDYDFgJWBU6Tm0K8ItUGUl343nR7jezTavU5qP4C/VR1WivVEKKqKOAIdUOKqlXJH2aKp75JdycWn8hsCSuG+YCNsE9LuCT+I/CZ5UvGf6mYavCoXhV+u/w+f63VvIaapE4UbdAwqS8AyQ9GVaMxOfZFG3kG0EN8E+kHcoqZKRozGx9PCx4sKSRVWhvCdwj3C/UkKpZzOwSYB5Jbe7NvcW0QD/grv/7on6IPaiWcD7uIy6aacD2MDX6+JNH0nN4x/evIfNDpdv7FA/uurDSbZWBs4Ataj1dU41Ssn4Ix51fRlnqlmigSqUFJaV/huPvg67Ai8G3HEkQSW8AGwJnm9nhVWjyPGDDao1/lYqkX/DI6CvNLOqKYogl58tC/NIViKS+M917TStLKim9sDQU2B64CXjJzPYsv8SRYpD0AT5L4EgzO6WS4eCSJgEn4Iq/XaXaKRM34+ma4ne0WGLJ+RYTx6DKQQtKSpvZSsCd+Iz0w4LyiiSEmS0MPI4HWx1fqZpSwQA+BwyTVNPBMsHFdx+wTOhVRYohlpwvmWigykmJJaXNrAsegb4esIuk0VWQNpIDM5sXLyD3Nh48kW3edTnaWRVXTMtImlCJNsqFmd0EfC/phKRlqVtiyfmiiQaqhjCz3YArgHOAq5KqCBsBM+uG9xp+APYK85gq0c51wFRJNR16bmYLAf/FS4d8lLQ8kbZBNFA1RghDvh34Ftg/vc5MpLqYWUfc/doB2DGU8Ch3Gz3wuVEDJb1X7vOXEzM7Fp+MWtPZMCKthxgkUWOEMOT18Pl8b5rZgIRFarNImgbsiCcJeaQSJTNCBeCz8YCJWs/TdxWwhJlFAxWpCm2rB2W2ANl9wMNq0QdsZpvjZWJuAM6OVXeTIYRYXw2sAWxe7l5tKEs/GjhV0j/Lee5yE76TVwErJFGlONIMdabfCqFtGCgvzz4Ezy6SK4rmETyK5tXqC5ibEFV2M15lcw9JY/IcEqkAoXdzHp4dYNNQmqKc598YT321nKSp+fZPEjP7F/CCpHqYbNz6qWP9lo/Wb6DMBuNJxzvSvEuzEZ/BXXPzEMIb/Al43smDJN2fsEhtFjM7ETgI2ETSZ2U+9z3AaEnnlPO85SZMLn+ZOkjX1OppBfqtOVq3gWp6eJ3z7ZpGzU6WM7O18ACKB/E5OqWmUom0ADP7M16bcnNJ/y3jefsCrwGr1HpP2czOAxaVtHfenSOVoZXpt2y0SgNlZr0b4INfQe3zPLzBwCLMVo97CjAA6TUzOwlYXNIBlZK3GMysO57L7Q/AriEDQqTKmNkeuHLYRmV0m5jZmcDSknYt1zkrgZl1xctB7CzpxaTlaXO4W28UxRmnFDP1W1llqgSS6n4BvsBdLk3rvXDYDIEKXZ4GLeKfZwjuTfq6mrlew91M3+NVoy1pmdriAmyDl/oZWMZzdga+BAYkfX0FyLoH3uNrl7Qs9bwE/TUVr2X4HV4mqmsz+++7PPxQqH773MelNL1pXU3rt/SldYaZezTLFpQeRt8AbBlmftcccq4HBuK5vm6tRAh0pHkkPQDsAtxlZluX6ZxTgOPwsPNaL/l9G/Ar/pIUaRnbSOoKrAr0B07JtWMf6DY3zEMr1W/p1JyBMrMTzexTM5toZu+Z2fZp2w40s/fTtq1qZiOA3sADZjbJzE54Do4y6Pg7Psuyf0YblwHbhs/74t+EybhFG4unGe8KHV+Hw83sDDO7JU2GtczsRTObYGZvmdnAtG37mtlnQb7PgxuoYkh6F1gdryT9pnm3P1JFJD2N96T+ETKBlIN78LlXB5bpfBVB/jp/OHBOcD1HWog8OvQRYIVs+sTMlv0aLn0ZGrriseQADwGr4EURewFnpJ1zg/C3O67bXgJOhfb94dHUPmbW18yUeimqti7LSdJduCzd152Anrjx3AW3HQuH9d/gCtnwIpR90rrIM118o2EkoUs7GdQV9FFal7c/6PbweR/QybO7+FLLzfizviW0swgwHk/62ABsGv7vAXQBfsHHDwgyL1/F+7YD7m46DmhI+jm2tQVYIXw/Dy7T+fqF5zlf0tdWgKzXA5clLUe9Lun6C7cv7wIX59InZ8ML62YZnngbNAP0FmgB0D9zu/h0Omgj+CxNhr74fu2T1mXpS831oCTdLWmspEZJdwIf4xMkDwAukvSqnE8kfZntHO38RQJwh/4gPPSNcLIPaOpB5SHzrXBP4GFJDwf5nsB98FuG7Y34m08nSf+T93CqgqR7ceO9PfCQuZszUiXk0XwDgBPNrMUJVSW9DdyFFw2sdU4G9jSz5ZIWpI65z8wmAM8Dz+Avxln1yRxZAiMGAivib839gN3CSZqjg6fwykViuiydmjNQZra3mY0OLrQJ+Jvp/PibxaeFnGOGW/+Z7E6TgboN2I6CQ18yM0z3AXZKyRbkWw9YWNJkvMc3GPifmT1kZssU1kx5CAZ7APA67vLbpJrtt3UkfQKsD+xnZueWIXXRacCOoSRLzaKmdE1X1EG6plplO0ndJfWRdEhz+uS3LEUQ/4NX3OwBzA1ci2c5bo7pMD3b+lrQZSlqykCZWR88rc9huGujO55B2YAxwBI5Dp0lVv4X7yTNZFM83G00bqh2z9X+rP9OxdOEpDMGGBG+SKmli6QLACQ9JmlTvEv8QbiWqiLpd0mnAHsDw8zsfDNr7k0pUkYkfY27/TcHrmpJJVpJPwKnUx+K/xr8ez8oaUFaC7n0yQ8wpjGjAOLuuFdoDD4gPZgmpZjti9MRpo/z4ZMUCxXSdrWpKQOF+z6F2xPMbD+8BwWeBuY4M1vNnCWDQQMPzVw8dZJ/wL3pJ+2AD2Adj488b5qj8QXxAaWf/V8DhmfscguwjZltZmbtzKyjmQ00s0XNbEEzG2Re2+lXPGQ0XxXNiiHp33hE0ErAs2ESaKQKhB7FRri3ZXgLo/FuwF3NO5VDtkohaQc03PoAACAASURBVDpwJHCpmXXKt3+keXLpEzPrcDeM+wYa0uu/TATmxdNJvIJ7ilL0wBV9etqTVUDvQk8z621mc+OpkpptuxLXmY+aMlDycgND8UCT73C36gth293Aufi9n4jX6pk3HHo+cEpwux033O0MSrupuwNP4r/yXNpiGdx3uzjQBdpZho9WPrt/EHASbkTH4HavISzH4IGAP+Kutj+XeCvKgqRxwNbA3cArZlbTSq41IelnvBc1P3CPeemOUs4zAzgCuCQojJolvBS9iU99iLSMTH2yMW57PhsL/RaCCQvhXy6Av+H+4G74oOXOaSfqjA8Srou/6bwEjZvBg4I7cC/R63h2mlxtJ6bLWmUmCaDtzLQuEPOy07cDTwFHqwK1jSKzY2Zz4D3vefFxhkklnud24GNJp5VTvnITeuqvAyurxtM11QNmthjeM90br758maTX24p+q6keVFnx9DPHkmVAMQ+pXFU1//CKQX49q+Fu1FfMbIU8h0TKgLwS7254KPET5uXkS+EE4JCgsGoWSV/gpUkuSliUuibMt7wLjxL+DVhJ0p6SXgfajH5rvQYKwBMiph5iPh9qI3WWSLFYJP0C7IXPsXjazA6ug8H3uie46Q7EXdejzMunF3uOMcDlwCVlFq8SXAisY2Yb5N0zMpMwrr2Dmb2AezteAPpKOiFrbzRNv6mV6rfW6+JLx91bQ/D5SrnqpTyM10upizeLlmJmS+M+6E+BAyX9lLBIrZ7wMnAqPp9u01zz+Jo5viPwHl5y5ckKiFg2zGxnfOhjNcVCm81innh3f+AofOx9KHBfwffNrP+bMHwFWPp3aOg0a+BeXeu3tmGgUnjuqWwVJ4dTpxUnW0JQeBfigR+7K2alrgpmdiT+5ruppA+LPHY7PFho5RA5V5MEY/w0cKeka5KWpxYxs0XxKTUH4ONJQyW9VMJ5GoBPBsJxa8Kt58HdDa1Ev7UtAxXJiplti6eruRK4MLikIhXEzPbFK/RuJenNIo4z4DHgQUlXVki8smBm/YAngGXDnK4IYGar4FFyWwEjgCvUguKXIR/oVXgP7HRJrca1Gg1UBJj5Nncr8Duwp6T/JSxSq8fMdsAnuG4v6YUijlsOz2SznGr8zdjMrgaQdFjSsiRJ6OVsiRumpXCDcr2kzGw1pZz7ZjwPgeH5SY9o6TlrhWigIjMxs3b4uMGfgf0lPZKwSK0eM9sMD0PfU9JjRRx3GdBF0kEVE64MhKjF93F3ZmZmllZPmLS8N3A0HqQwFLirXO5Z8zI7X+EFTIcCT0u6sRznrgVadxRfpCgkzZB0Fp6H6zozGxrm8UQqRDBK2wEjQo+qUM4EtjWz1SojWXkIrr0z8PpWbSZiNGRjOAsvPrkVnn1oNUm3lnnscGfcKI3Ds8a8VcZzJ040UJHZkPQsXl5mCeAFM1syYZFaNcG9txlwdRibKuSYCXhvtx4U//XUQbqmcmBmy5vZP/D8dQsA60vaVtIoVcZdtT9wUwh4Wgov1dFqiAYqkhVJ4/HSHcOBlyypgmVthBAosSFwVojyK4SbgDnInf+4JkhL13SxmZWS+aCmCblBNzWzR4B/45Oyl5I0uNgozSLbXQbPzPYIsBzwiaRplWovCeIYVCQvZrYyPmfqZeCwUtP1RPITEiA/iUd3nZ3vrdvM1sZzLS4raWIVRCwZM7sD+FDS6UnLUg7MbE48S8gx+Mv+pcBt1TISZnYB0E7S8Wa2P7ChpL2q0Xa1qC8D5UX4ss1jGlavcf71QkhUehWec3IXSaMTFqnVEjJNPIYbquMKMFI3A99IGtLcfkljZr3waLP+kj5PWp5SMbP58DGlQ/FyQEOBxyvkwsslQ3s8OGITSe+Z2ZXAl5KGlnjC2tStqoGSx3kXWF0wUjBVntYjvdzxlLB+pGD1xGVt5Qv+xjgOOJzwghOXitznefDUSH/H35Kb27cnXp9uyaTlLuC6TgHuTVqOEmVfCvgr8BPuXl0xQVm2Al5O+/9ZYOOiz1XjurX2e1Bmg/E3lI40P2bWCEyjznJN1SNmtgTu8huLh6OPT1ikVklIgXMfXvJgT3ni2Vz7ngCsJ2nbaslXCmnpmg6Ul+eoaUIAyvq4G29d4Drgr0p4nqCZ3QM8Iem6MMfqJ2AJSfkK6aafpOZ1a20HSTTdwM7kl7Uh7Dc0HBepEJI+xX+sHwGjzWxAwiK1SuRjfVvjgRD35wkwuAJY1sy2qIpwJSIfnzkWrxJcs5WezayDme2G12D6O+5y7SPplBowTvPjdVdfM7PRwC9AY4nGqSjd2sFsmpktnmf/slG7Pag2Uu+k3gkK8Ub8zfIcxcSgZSco8huBPsA28mKI2fbbCh+oX7G53lbShF7J48ADqrF0TaG67AF41OHn+P18UFJi1bEzCVGeq+PVbn8BngP2k7RNgSeoG91ayz2oIXjXE4Aik8N1JK2EcTrWsvLbkQzk2SZWBdYDngoD4ZEyIp/YuQ8+aP2UedLjbPs9BHyCK9eaRf5WfCRwaq5rqTZm1tfMLsUro68K7CBpoKR/1ZhxMsLcJ/yF5V1gZTz4pFBm0a1FklO3VoQqDOZ9ARyH/7h+Bu4MF7kv8HzGvgKWFCywF/w+GLQFqDPoCdBDoGVBXUE9QRenDeg9AFoJNDdobdCbME3QI02GvwQZfsXLtN+b0faVeNLGRAY9633BX3aG4OUCtk1anta44LnWzsVTBy2SY58/4AETCyUtbwHXcxlwXcIyrAncBYzH66T1Svq+5JF31aDPnsLf26fh+TNvBN7Ee1RjgDPSjukbdOt+DfBNd9A1oFdAKwadeWiaLv0YtAFoLtB8oJ3TtgH6CKYdCisAk9KWKYR3j9Dm/uF7+hNN7tHir7cKN/QL3I/bEy97/T4eotmcgTp+L/h9LtDzoBmgqaCFQM+GG/Uj6PXw+Q1QD9DLoN9Bw0C9oXEC/CVNhtFAL7wW1MLAZKB72N4ej0xbLekvYL0vwNq4a+RKoGPS8rTGBa+u+zk5ovbwarY3JS1nAdfRHfgWWLXK7bYD/gQ8H+7jUUC3pO9HgbJflTI+uJvuADzcfHdgxfCi2C+8KG4X9ksZqGt/hCGPwLQ5QYNA34G+DvpzVNCnu4LOSdO7z2UYqA89su+4DLluBW4PnwfhPfllg249BXixlOutlovvSklj5Xm5HsC7pM3RrwHaDcJH4hvwLlcHPPznFzwGd9Ww8/XAwfirUDvcF9IR7FHYKEOGMZKmygc5n6Up9crmwA9KlVOOlIy8ns0qwELAy6EwYqSMSLoIuAB4xsxWyLLLOcBmZrZmdSUrDnm6plOAq6qRrsnMuprZ4Xhwzwl4YMlSki5XjU9yhpkRkLvh2V1SdMEN/R2S3pHUKE/KezuQGbx09jyw3OYwZ5dwogWARfAwxVTNlw54AsGxuN5dL+MkDb66X5pcfwGWwXtN4B2Q8yW9Lx+TPg9YOUxCL4pqGahv0z5PAbrm2b87eHcnnXvxspB98Dufquz1JR6O0j1tGeONzp92eGbJ5OF4ZVPC3xH5LiJSGEHx7AL8DXjezParg3xxdYWk63DX+ZNmtkbGtl+AE3HFX8vjzOBjKXNSwXRNZraImZ2Pe1IGAntJWkvS3aqvoJ5tgbc16yTnXniC2NXN7Gkz+97MfsaNxPwZx39H0K2dgAXTNnTC/XTg3W8BawDL477DLHSHmUFSR+K9talhWx88SnOCmU3Ap0kYbguLIskv72TSokjC7PkUE2DWusXgYSv347647fA0vuBP6ORwUGqZAhw5a+LEzHDF+4B+4Q10a7yLGikTcq7H88sdC9wSSgNEyoSk23EXz0NmtmHG5lvwMYq9qy5YEagpT9+FYd5X2TCzlUOWjXfwnsaaknZQ/VaO3o/Z7UUqO8dtwL/wMbS5gWuZXYVC0K3NsRBwA96Dug44BPfXZZ4neEeGAztLSu8AjAEOltQ9belUyn1P0kC9BSwfvkQd8ZT8Kd5uzAjc+w23ID/jXdC5aBL+QPxp/Ae3QpOBB+DXbyBnokb5fIx78Af7iqSvynFRkVmR9F/8ZWwi8IZ5iGukTEh6EHdV32lm26Stb8QV/3khdLpmCYrrKeCklp7LzBrMbCsz+zfwIP6SuoSkI+Tz9+qSUFB0TWBkxqaUgeoG/ChpWuhR5+qRvg1MzbEN8MSOX4fP8+BWLt1QNMK0Ma5b7wdOlvR8ximuBYaY2fJB9rnNrKRM9okZKEkfAWfh+cY+xgcsUwzPdswIfLRvLvwOpLo8/XGLfxh+Q5cEhkGHh/P3iobjA4vRvVdBJE2RNBh3Oz1kZsfVgeupbpA0Ck99c4OZ7Z62/lU80/WpCYlWDCcCB5Va2sXMOpnZgbhBOgd3HS4u6UJJP5VRzqTYG7hH0pSM9SkDdQieCX8icBoemZiN4WTvWc3kVdwSdsV9ilfgKdNTGNh2noNwaeAyM5uUWgAk/RO4ELjDzH7B9y1pAnktT9QdiUeDlKLIGoH7kJotAGdmvfG6LQsFv32kwphZX7zX+jOwj7zQWqQMBHf1o/iE6WvDugVxBbG+pA+SlC8fYbB9XRWRrsk8yekheBXoV/Hh6FGqWcVWPGH89iN87OzltPVz4mHc86qYDOpV0K3lopbfYs/HY/xLYVo4PifhDf4YPPolGqcqIekLPMblDeBNM9s4WYlaD8GdOgA4ISh7JH2HR1FdXgeBKpfj6Zo2z7ejmS1nZjfgrqaFgAGStpb0dGsyToF18blO/8lYvxzwWVHGyamobi0ntWug3D1xLB7vUAxT8KSGOVNxhNIRv+D5rFpFbZp6QtJ0SSfjbovhZnZeLedlqyfCOMv6wD5mdn4wSlfjkVVbJypcHiT9is9JutzM5sjcbs4mZvYwPmY1BviDvDBgTfcOW8j++Ly2TMNbbAYJp4K6tdzUrosvRR1k3I2UTnDRDMOHD3cLPaxICwkJRR/FJ8kfBmyCh/2vUMIbd1UJBujfCrWNgrFKFQZsj+fHu7XWr6MchMjGMcByykhSa2ZXAGMkXVLiyWtet9ZuDyqF35ABeFj4NGaPQJka1t+HJzGMxqmOCGNQW+MRla+UGu0TmRV5ZuuN8KksNwNP42NRRycpV4EcjUeBLWNmJ+Hzl/bA05WtIOkfbcE4BXbEM+5ky6BeWg8qRR3o1trvQaXjiSWzVX0cTqyoW/eYWX+8ztS/gaOzRCxFiiSU6LgHmI4r+OeBlSR9k6hgzRAi+e7CjevtwKUhO0Kbw8yeBS6XNDJjveEBEkupHLqvRnVrfRmoSKsnTOa9BlgJLy3/bp5DInkILrKbgR7A60BPSXs2f1R1CQp3PdyNtx7u9t0TGCTplQRFS4xgqF8EFlVG+RQzWwx4TtKiiQhXJWrfxRdpU4SIyj2BS4BRZnZwHUSf1TRBue2Bl5IYCGxoZusmKlTAzNqb2a54hNqNwBNAX0nHUz/pmirFvvhYW7baXi1z79UJbfXBR2qYkCZpGP4mPRi4y8y6JytVfRNSCh2EJ0n+HbjGzNolJU/ILnAM8Ck+j+lcYBlJf5M0Oew2Ak8OU9PpmipBeDb74hOOsxENVCSSJJI+xMt3/A+fM7V2wiLVNSFM+Xg88cpSeEbvqmJmfcxsKN6bWx3YUdIGku4PRjRd3kbgcDxdU1vL47gJ8G0zY2/RQEUiSSNpmqQj8Pkx95nZSUm++dc7oXd6Dp7B5pzMTOiVwszWMLM78AnaAlaRtFtIx5STOkvXVE72I3fvCdqIgYpBEpG6ISTMvBWPSNsrR+htpEDM7Ck87dr6kt6owPnbAdvgk0J74UbxH8VmbqmndE3lwMzmxQspLiavoZdt+xd4wdWaKUdfCWIPKlI3SPoan9vzHJ4ZvaQElJGZ7IyPRz1hZpl16UrGzLqY2aF4nssheBXYJSVdVkpasZCu6XzqI11TOdgNeCSbcQqshNeFatXGCaKBitQZkmZIOhMviHidmV2SLS1OJD9hMu/JeMnwkYXkwGsOM+tpXhjwS/xFYl9gLUl3qeWFAesiXVOZiO69QDRQkbpE0rN4afmlgBdKLdMQ4Vo8fdBleF7EHYs9gZmtZGbDcTdcV5oKA75QrsStIdT6SLy8w5zlOGctYmb98GK3TzazW5sxUO2TFiAnnqMt28zmYTFrRARA0ngz2w44FHjJzI6UdFvSctUTkn43syPwN/ZtgX+a2VySclT6dsLcpM3xibXL4m68oypZe0nS42b2Lp4K6YJKtZMw+wHDMyMaM1gZv9+VpQZ0cO0FSXjF1SF4gSsBndK2TsWLbT0CnE+eCKBI28HMVsbTJL0EHC5pUsIi1RVmdjdeGv0OfLLsZZIuz7JfR2Av3Ej8hicbvTPHZNJKyLkEPqm3ptM1lUJwVX8NrCMpS5X1mTWgJuA1oJqtjNsCQWpGB9eWi8+z647Ci2l1ZNYbQ/i/Y9g+KuwfiSBpNF5cGeC1YLAihXMc7kKbhpfrOMTMTk8FJZjZAmZ2Oh49th2eIX0VSSOqZZxgZjmR6/CKra2NrYEPchmnwLLApxU0TjWlgytqoMzsCzPbJMv69c3sw4yVqdTvndPlGgVkSTbVEPYbGo1UJIWkSZL2A87CI9MObyNRXy1G0pfAlcDFkr7CjdT2wDAzux4vDLgIsKGkrSQ9lWBhwPOBAbWSrqmM7Iene2qOsow/mVlfM5OZtU9bOVMHnwENeZI1VkUHJ9KDkvScpKVnrvAuZco4FUPqBvXPu2ekzRDGodbCU+TcZ2bzJSxSvXAxsKaZDQRWAL7DQ577A8tKOkjS+wnKB/iLCJ6Z/crWMmnbzBbGU3vdk2fXlYG3KiBATergWnHxDcG7jaXQMRyfl1neFiKtmuAKWhf4GE+TtEHCItUDvwP/wscXrgLuBhYGxuNJW2speu52fDxk/6QFKRN7Af8sYOy0UhF8BevgLPMFCtbBxVINA7Wymb1tZj+b2Z1m1tHMBprZ1wCYLfAabLkKNHQDdsInuJyScZKhwAL4ryV9gsCv0HAMbNvO7Gsz+87MrjWzTn5qb8fM/mJm39L83IJIK0PSb5KOAw4G7gxjKq3ijbucmNm8ZnYinr1gWdyd9zdJf5c0Hh8baQfcH+pLJU5wLx4OnG1m8yQtT0sIbui87r2wX7M9KDM70cw+NbOJZvaemW0f1rcLcwZ/MLPPgK3SDlrgM9hyQNDBmwI/pJ3zCzwq4h9Ab3yCG8DLwDpAd2joB9uvZjYoTY59zeyzIMfnZrZHWL+kmT0T7MEPZnZnszdHUsWWcG2vAD2BeYH38ezUA4GvJfELnNgLGi8H/Qa6F9QBdDJIoKdB7UCnhu0PgTqBfgzbjwJtBb9/5Lm6ugEPAOeH9gfiBv9CYE6gUyWvNy61u+DvNk8Cz+D1dRKXKekFWBLvKf2I119aKaxfERgHzJe2b3tgOJ7FY+6kZU+T61q8oF/isrTgGtYCPiJEVTezX1/gmzz77BT0bQP+rj85fPcH45k9egVd/DQeoddecPyaMONo0DTQM6CuoD2Cjv3c99NeoEmgKaCvQfMGfTwD9ChM6+ht9QC6AL8ASweZFgaWD59vxyeHN+A9r/WavZ4K3/gvgD3T/r8ofKFmGqg74bGeoMZwMwRaN8NAdQRNT9veA/RSOKYz6BNff3NoY23g8/B5IB4K2zHpL2Fckl/wXsAQfGxl26TlSegepAoDjgS+B87DCxhm7ncl3otKX9cQDNobQI+kryXINH+4juWTlqUF13A9cFIB+20HPFTkuUfjEXdPAYPT1v8xZaDegXvbBeOT0rG7ZTFQn6ZtvwC0Z9r/Aq0C3+DzprrgofA7ZHYK8MKZ1xf6klgNF9+3aZ+n4DPN0zfOvwj+q0nRK+ME8zHrjOLOwCT8WzkFWM1PupuZTQAexa14iu8lTWvRFURaBfI0Sefj0WlXmtmVYV5PqycUBtwF98wMA/6NFwY8SdLYLIecDuyQHq4vz/12BPAQ8GxI3pso8nRNZwFX1GPEZnCZ7ogr7nzkHX8ys73NbLSZTQj6cAXciPcExqTt+mXqwxhYYB7cqqTok+Xc6Xr5S3yAsnva8p5nwFhYXs9rF7zX9j8ze8jMlgmHnoCr+1fM7F0za3YMMfEgiQXhh29wE51iTK6dM5gfD8p/F7jd3aaX4rPh50/brcZmIkeSRtKL+I99YeBlM1s6zyF1i5nNZU2FAQ/FQ7SXlvRXNRUGnA15RojTcENuaesl6VR8SOK5GkkxdQ2wEP7iUW/8CfiPPBFyPpo1UGbWB6/1dRjunu2Op58yvKZauo3pnfqwCHz/E+6fS/FVtvOnfe6FR3VMSFumwW2SLgCQ9JikTfHf2AdBLiR9K+lAST3xseG/NfcdStxAbQlPN4CuxgeL7scHrQqhATgQOBJm/Oxd2K54TrGfzOwZfOBxzloZ2I3UDpIm4Nm8/wY8HwZ16+4NPBdm1tvMLsEDH9YAdpIXBrxPzafRSefv+LjuLpkbJF2CuwefMbMVyyV3KcgT0R4BDE0FSNUR+1N48Fa+HlQX/IX8ewAz2w/vQQHcBRxhZouGoJITUwf1g5dWg8bT8fGQ5/GB/ObYM+zzGDADmApTr4LJ4fwLmtkgM+sC/Io7vBqDTDul9bx/CvLmzMqeuIHqBjfeC7/9A+8m3oKHDBUaz3ohsCRoXw8pHox3qk7DfzztgLmAcWb2gpmdZ2abmVm3cl9HpP4IvYHrgQ3xTAq3WJ1XbjWz1c3sduBN/KV3VUm7Sir0vW8mwZAdDlwUlE3m9hvwXHxPmtlaLRS9RUh6Cngdf451gZkthue5u7+AfefBRzs+zbWPpPfwgOeX8HHWFYEXwuYbcHvyFj6GODLt0OG3wfT/4NETZ+ITCJujVxD6PHw8pTd0OsGDbhrCcgwwFg/AGQD8ORy6OvAfM5uET2k4UtJnOa87DFwli9lIfCCvAbyC2mC8+1MAjcB9SDvkPr11xoMnBoRlNdwz+Cwe1fV8eKOOtFHCd+RSvNT2rpJeS1ikgrGmwoDH4MMHVwB/Vwm1l3Kc/zY8vU7WqrZmthU+rrVLMBSJYGZ9cSO1ijwbRk1jZmfgrrjDC9h3IHCupMpkz8jQwUWSVweXTNIRLJLYHw4aC1Omg4aFqL2xGREizSyTBf2LjGzpiBuq0/DB4on4W8VleKTMfJW4zrjU/oKH6Y7Dq8A2JC1PHlm7AIfgk5FfwV1x7SvQzqL4ZN3Fm9lnQLhvgxK+J2fgyWsTfz555GzAo5xXLXD/o4C/VkwmWD3oUpWwFK2DC75PST+ocPMP6gS/dAGtCHqwuBszuAztz4G7CIfgUYC/4Gnlr8IjbBZI+h7FpXoLPt/kRTyjQs09e3zg+Vx8rOGfeNh4s3NoytDmSXimg+b26Y9H7e6R4L3pHBT/gKSfUx45N8bHkwp6bngP9aBKyvQdHDMZGkswTi3WwTmvO+kHNcsCg8MFz2jupsyAxl9heqVuDB7VvgZwPPAgHqTyHh4ttCseSpn8/YpLxRagA+5i/wbYOGl5gkz9gqL6Ea8wu2QV2+6Ij39smme/5fFA3EMSvE874WMtZe9NllHGW/Dxl0L3Hw2sXkF52gOPXQxPTYHGGfkN1YxKGydJNTIGlY4nHRwCbAnZa5FMhn9vDOv+xyt3flR5kawdsBJNY1jr4y6PZwjjWKoDn3ekeMxsY3yOyjDgDEnTq9y+0VQYcDncMF0n6cdqyhFkGYSHqa/U3H0ws8XxmlI3KIQdV5Nwz54C7pb0t2q3nw8zmxufSrSkfB5Xvv3nAH6mgjWgzOxivEL1k5vAlk947zynDgYexutBVXSstvYMVAqzHmSv5jgc6XszOxbYSNJWzZylQqJZAx6+mTJYG+DTCJ5JWz5Xzd7cSDGYVxYdhn8Pd5f0RRXa7IhH8x6Nz8AYCtyhKtZeyiKT4S7wR5SlmGHGvj1xI/UAMKTav4UQ+v5vPAv7+Gq2nQ8zOxjvie5Y4P4rA7dKWr5C8uyBT3beBn/hXkvSJ/l0cCVkmU22etWh4a3ibeBYSQ8lLIvhSTY3oMlozWBWg/VxNFj1S3gpORov83CIpHxlEUptpwce+PBnPCLtUiDJ2kuzYGbL4kpseUnj8uw7H27QXgMOlWeiqBpmdhUe6HJoNdvNh5m9DJxdqN4ys32AP0raowKyrIY/o43wEP3/STqx+aOqR90aKAAz2xwPZFhB0q9Jy5MiGKwlaTJWA/AxjXSD9X6tKJ1I4ZjXzbkdTzx7dLlcLiEVzNH45OF78JLr75Xj3OXGzIYCc0k6sIB958Lnu3wD7FtNF6mZpRJU/1FS+WsolYCZLYd/d3rLJxgXcsxlwFhJF5dZlgXx6M/UnKV7gGUkTSxnOy2hrg0UgJn9C5/HdFHSsuQiGKy+zGqwuuJvoqm5WO9U+w0zUhpB6V6Dj0vuIundEs9j+CThY/AJjNfgCVqb7ZkkTRhD+QDYRgWMQYTsDnfj82V2VhVzY5pXe90NGFgLL4RhrGdGMb0UM3saOE/SE2WUYw7cBfo0Hpr/Cp4V/pZytVEOWoOBWhJPgNlP2ZNe1iRm1otZDdb8eCmDVA9rtApPSROpMsG47INXoT0ZDwgo6McUlMMuuGHqiLvxbqnUAHglCEk+D8DLJeR9sTKzDniwyYL4XKmqvKWHAKfX8RI8zdceqrwsHfA0dwMlfVjgMYZHbS5dzhcXM7sGTyC7PZ4T4f+AdWvBiKdT9wYKwMzOBxaRlC9DR81iXvI5FXAxAFgET1OSMlhvVDuCLJKf4Jq7A6/nc5CayUgS0tUchKcP+hAPfHi0HnvOYUzuP8CVkkYUeEw7PPfhysAW1YpENLP1gVvxgImcCXKrIMc2eMDIOkUc0wd4UdIiZZTjIHzi71r4hOH3ga0lvV6uNspFazFQXXGXw06SXkpannIQIsfWp6mHtRieYytlsF5NMqIr0kSIuLsIj4LaPfM7aGZLAEfiUXkPApdKqkTZ7qoS8u/dSxHjFqFHcCGwBT42O2sr+AAAIABJREFU9L8Kipje7m3AJ5JOq0Z7OWT4J/CwPIdhoccMwl98yhKtbGapWmDrSfrIzC4FuhUynpgErcJAwcxQyaOBNerxjTQfYcA33WD9Afcbp+ZivVxN335kdoIyuR64HFfCa+NuvAF4ss6rJX2TnITlx8yGAd8WOaZi+FzH/YFNqhS2vyg+ebe/pM8r3V6W9hfAe9m9VUSORDM7HZhT0kllkKEX3uvdX9KjIWDjGQqIyEyK1mSgDM8Uf6OkfyQtT6UJA9Xr0WSwlsfzCaZ6WC8l6c5oq4SEpQ/jLtofcTfeMEmTEhSrYgTX9DvA2pI+LvLYQ/GyD3+U9H4l5Mto72RgNUl/qnRbWdo+Gk9iW9QwROh13S7prha23wkf475L0kVBXz6GV+i9oiXnriT1ZaD8LSTbxLFhYfLuqrhyWKa5sYDWiHkJkXVoGsdaGb83KYP1Qi2Fj7Y2QmTf/+GuvDF4WPUAYD9JjyYpW6Uxs+Px3Hdbl3DsXrh7dCtJb5RduFnb6ohXMThY0pOVbCujXcN/i4dLGlXksZ8Dm7UkY05o/2a8/NAekhR6++cBK0uank+3ltp2i1EN5KXKu3im3ZGCqYIpGTmhpoT1I+Whutfjc0iSlzvBBU+auRFe3mUUXjTsFTzqbGuge9IytoYFr0x6CZ766k7cxZzaNgCP2roEmCNpWSt4D+bAgz62LPH47fBM6OtXQdZBuJHqUMX70x/4jCKz4+OGYhLQroXtH4N7VzqH/zsGeTYtRrcm8t1KotGilgITyKaSF37ts6G/B5ZLXPYaWsKXcgPgVHyi4ES8qN3leKhpLDFS3P3sD9yGu/EuBfrm2G8+fKLqq8ASSctdwfuxBT7GUpIhxutwfY9H91VSTgMep4hErWVo86/AaSUcNwCP4GtJ23/Ey733Tlt3Mm50itKtqnBi2KzyV+HhvIvH/Rd/fNMNbO7mZS6T/+Fvsk9Q4RIE9byEt951mLXEyDt4MtKdgAWTlrHWFjwkdxDuMv0Krxk1dwHHGR5a/j2wW9LXUcH78yBwfAuOXxuvBLtTheVcNjyLipdSCS+G44E+JRx7JD5xu9S2lwj3c4O0dYsC49+Hk0vRrdU2UlVrqICb+QUe0ePrWlBAqxEmr+ulAbYrov1RwAFJ34cE73+2EiPvA9fiM/F7Ji1jgvemM54f76PQE9qVElxEeLboD4EbgS5JX1cF7tNSwA+0oBwNnp1jLPB/FZb1UnxydaXvya7AEyUeexMl1oACugH/JaPsCXDbzvB3weTbQWuAOoN6hM9/df2pS0GLgbqBFgYdBZpe4eKEWa+jWg0VcEMzDdTIArqeCgZJMzK6pF956PVnQMcC22/TBirL/WgXFOpReFG88Xjl1r8De5HmMmitC14Y8Jzwtn0fHubfol45nuLqJnze3spJX2MF7tkFwPAWnmOpoA+OrqCcc+Our9UqfD8ew+fGlXLsm6SNaRZxXAM+1+mG9O9r+P5+9RvcfzE0LgC6G/RL0KFvgHYHTQN9Avop6NPxoA1BQ5vcffdW7ftU8QaC4cHzPd2FR5NMxF1//cM+I/A8XVOBSQvCmYKpL4HWBs0N6gd6Os0IDQCdBFoHLxH/cVh3SljXFTSHD7yemybLWnil1An4nIiBYf25ePbxafig5NXVegD1soQv/YrAYXhetXHh2Q7H57Ms0VLlXSsLHsl0E/ATPn6wVAXa2CMYvsNay30L19UNj2Bcq4Xn6Y33Ns+s1P3Boy5frOD5e4cXu04lHDtH0IedSzj2dDwLzZxp69oBo9eEA3+CqZ1B9xTokfoBtDHoz03rpgp6VOX7VPEGZjVQ0/AiWO3wwmcvZ+4nCcHxY2DqvKCHQu/ocdC8oHFpBqoX6L+h6/lbWLc46EPQZJjS14uCTQF64fNSxof2G4BNw/89QvujiD2oYp5rqsTIYDy791jgazylzEHA0vWkeMP1bIYPoI/FS5xXNHAEz3j/Gt47azVBKngP+1WKjFrLcp4FaArkadG5cpy/Ici5Z4XuwynANSUeuxLwXgnHbYdPc1goY/1g4JlGOP4hmNauyWWXc7k1uPgAzQ8a3bRtiuC4anyXGqguz0t6WJ4EdUR4CNnodyt03JJZrUl/fJJTin3x2ant8VoW4FkP/wB0hk67ehd+LD7PYk88zcjDkhrlmYFfC01EikTO+5KulbQb/gKwIZ4deX08SGWsmd1pZoeY2fJhPkZNYWYdQ+LTd/AQ/FuBxSSdpwoXupP0CR6o8gnwppltUMn2qsitwHT8J1oy8uwGG+LTR/5hZu1bLtos528EjgAuDPMIy0bIVbgf3hMvhZXxMu/FtLk87tb7k6Rv09bPi/dEjzDo9yPMOT+uN/+/vfMOs6uq2vhvpYc6ISQ0Q+g1oRpSIBSpRpr0LgExSBMIUkViBFEkIPlQQw1NOhhBiKggoPQeUXoJkZ6EBEImddb3x7tv5szNvXP73Htn9vs8+5k7p+yzz9nn7LX3Ku9KYRjyae+JbCMAhyHPqTeRdFul+fCeSMtQcbS1gPok8Xsu0CPLS9cwFemRGhLlX0jipNAvw4mrJiuB7kgVtS1S7x1oZrNSBTExrFbszUQ0Iwist9z9Wnc/0t3XRF5ZD6K5xX3AZ2Z2j5mdYmabh4+4KjCzPmZ2PvAecACytW3u7jd6G+YWc/cF7n4GMAq4w8wuCKSqdYvEwH9RYDwppa5ZyFV6deB2M+tehiYm638KhV2UTCWUhuFIRfdckecXJKCCEJqEErimX/NnyG70CtDQG3myJJNRpewevZGtJYn10ULghJabG/JtWymo2gCRAZ74Pasf0hPMSpSvES9KCrmm4zNkkO6KYqMGo5QGDYmyrLv/MsP1I8oAd38/DPjHuPu6wFbIeDsA2SOnm9l9ZjbazL5Z7hlyJpjZRmZ2FZoY9kdq5RHu/ncPupBqwN0nA1ujWLWHA3dc3cKVJ+rPQMnkrC7Krr3RJ3+fmS1bap1pOBs4zszWL2OdI4GJJbxTeQuo8N3cBtzv7jel7RuIUrucHzbNGopm7n8qoDGLkFt0Am3C1FNLAupTYJ3we8rhMO9+5AKT8l54FBk58sFCWPiODI2bIT60rsDhZnaMmXUOqp0dEwNB8voRFYC7T3P3P7j7D9x9QzQx+wNysLgRCawHzewsMxsS8ueUDBN2MrP7UQzTJ4gO6/teZLLBSsCVz2w3ZAd73sz2rnKTSsV5wFGmNPElIaxqD0Yq+4fMrGwzeBej+iXI9bxkBHXhvkBRyf+CKnxz5MiVDy5GY/mZGeq5AhiTUFdPaYDGC9CK6G7ksdaEpGGKvPNapHoC+G+4wM7NVTciGqTKo9JGLlo6SdyS2L4WWrV0Cf/vg4IfZ60CYxwanwbfHrxXMNKNAJ+acJK4Js2ol7atcU0FR/4LrVLHIDk3H/XHPGR32BnNzIaiWfUXKMdNxZ9NLEu9K32A/YHx6OP8Eg3W5yF1bPcC6+uKbI8vIrfuH1CER1WVnsWw8O2MJ89QiVosSHX6EGVymEED8RXIeaJsgbZoUfEmZWCyQN6Bk0o4f02U4j2fYw9Hi5ulnGyQ6npKaox1dxz6Bi88vwV8EHjPML5uA34V+Hzwo8H7ohip/uBngDe2Ry++oksBcVAZSkZffeQyfCXKrXQ0MmC+i9x970F6882pgMdQLIUXYKUwcRmHHFq+Ah5BbrQ7ZhM2QC80m/xfOP479din4T7uDoPxhtVuT5H30BVNwvcpY52G7CqvA/3KWO8I5NpeEm9imBQXfb9InflgHsdtHcaugRn2LYO8mHdc6twKjK0Ve3/a6kIFlxKYJDxLtDOyAX6G0sMnt/cLM5Grwws6A6loTw8vQUlkjbGUp6DgyhEo19LTKGbtnyiYdlekMrwC8ePdjNIbVL3dJd6zIQeKz8Okqm5c9xP3sGuY5Zd1JRi+z/cpY5wa8AAluFAjJ+JPKYGMFtntfpHjmFWCANo/y/4xwB0Zz6/A2Fqxd6etLlRUKZKLz1vhiwJ+iMxZWT905Nl3MEpP/R9kEHwgzMoHl/LyxVK+gpxgdgNuQI5JjoJEr0TkpStUu41lvNcB4V28BWVArXqbCmz/vcC5Faj3+6HPNytTfRtQAl0TMtdcWoZndVAr+7uFidnYLPv7h3vIzvZSgbG1Iu9NW16sqFJmxl0UJPwKBRBSktk28hByTS3YNhJL6QWFcRwAPBVm5yeHfkpPMfIcSnexF3WeYgSpba5ClFNtNostU9vXRpqJb1Sg7oPCqqUk9opEfb9CSSYLPa9zEJablnj9d4ENWtn/e6Thyai2RhE6OdnTF8PxTQq67bhs5mUp8E2He7z1nCX35Lv0RDT2UymCRiScn7KNXAa8wNK2kbo1atd6QVQ6P0LxS08A+5FFBUvLFCN/Y+kUIytX+36KfAYHIlX16dkGqVoswM+BWytU97fDM9mlDHWl6JoGF9GGZ0u8dkN4T7O906OQTS+jdiBM0N4jhzMQclJ7cxv4spxja7lLvWXU7UPmrI83UmDWRzO7HXjD3S8ovVm2IgoG3iGUAcT062WFmfVDq6RjgYeBy9z96QLr6IZsiql+GoY8Rx8n9JW7f1rOdlcKIbX8bcjr9GgX60JNI8QvvYayuv6zAvVvj5xKjnP3QsJ8MtV1FOJJHOIKPM7nnLuAh919QgnX3R74pbsPy7BvO+TMtZ27v5Vhfxc0AbvA3e/NUn8ndF8XoxX56+6+cTnH1rKiGlKxFgpyjJhOlkRzJdadso1chDx65qBg7YuBPahDG0IV+2lrFCs1E7i8nP2F1ISDUCD3/Wiwfx2p0Q4D1qj2/edof1eUtvtDYOdqtyfPNh+MBtGKOB6F9+Vj4MgS6+mE1Mcj8zy+NxrUS1IjI0/ipXJAhfHqI2CPVs49CbFitGZffwiF2ngof632O9Hq86h2A6p68yJzvLsNrpMt/Xq7sI1U4Hl1Qq62j6IVzhnkkRiwDNfNlGLkbeA64CiKSDrXRs9rlyCkLqLGHXiQV+JjwKgKXmMTRJh6Qon1DArCLp+klCdTBvUlyhX2g7RtPVGYxZmtnLcyUnG2av8K39VMFAvqwB+q/U60VupLxVdmmFlPpM/9vrs/3IbX7YGSA6ZUTYPRQJhSCT7uFSYqrUWY2TJIzXAackQZhyYQC6vUnk7IdT3VT9ujKPrHEuVdr4GPyMz6olQ2K6CsvVOr3KSsMLMt0Ex+Y3efWaFrrI3sjtchlVlRfWRm1wKzXHyJrR33Esom/PdirpOo50XkjPBs+N9QyEQnpBrNeB9mNgGY7+4/yuMaD6LA5O0RKcHoUtpcSXRoAQVgZt9FxtstqzgQdkOEqtvT0jaSFFh1YRspBma2KnAiMgA/hQTTP2th4E8iDBYb0iywdkCz0MdotmO9Ua12B4F6OgqH+KG731ONduQDM/sdsNjdT67gNVZDQuoB4Oxi+iUI/v8Aw9399SzHbIG86tb2PO1VWerphtSEK7v73LBtNIrR3C61LcN5WwKTkcD/Isc1RiBV+UDk9brQa9h+GQWUBp2/IqLF8dVuDywxdm5J8yC4HeKPWzJzd/G21TUCkeVpyKPuNuA37v5mdVuVP8K7sy4tBVZ3Ek4XKKdP0YNWke3aBj3Pv6GstI1tef18YGa9kcPEzu7+7wpf50Fk9zrRleqn0DpOQ7bjPTIJOTO7Apjt7iUR45rZZsDt7r5J+H83xFE52N0/yHKOoZioG939mhz1d0P0bqe5+4OtHVsr6PACCsDMNkGDySZeTY+VLAjpFzajparpC1oKrJpV6SQRPqjd0Ex/MxRUO6G9qDSDd11SYK2ABpBUX01pC4FlZisAE9BM+RCvIVLcFMzsRBRfuHMlV52BvPU+ZE/6XqGakkBa/Apahd2Xtq87otQa7O7vltjOo5AQPMzM1kNhFAe6++OtnHMostFuk0v4mtkZiPpoz1La2ZaIAirAzH6DYgdGVbstuVBPtpEUwod8GBJMjmLIbvM2zL1UDQS2/KTA6oM8O1P99LK7L8peQ0nXNkSPdAki3L2mxt6JLigcY6y7313ha/UE7kC2nAMLXVWa2a4oQHaAu89LbD8Arcx2KkMbL0Oakt8jKq/fuvvvWjl+ObQKPcTdn8hR92po9TTUM7io1yo6loCSPjmTr/8NpgygrwMj3P3F6jWycISBaCNaDoRNtBRYb1ZjcDKzlVFCzhPRLHQcUNXcS9VEGCiG09xP/dBMOaUWfL7ctlAz2wgNzm8gD7E2yeWTD8xsR6TG2jibjaWM1+oarrUasLe7f1Xg+X9EgbgXJ7Y9gDjvbsp+Zt71P4ImEynuxVGtfSdmdhHyLD0ibGhtfPs18Im7n52luppExxBQZoOAc1CktyO3zRQakevr5FHw36vFBDG8ngfQVmwjS5wuqLBtxMw2RO7ahyBuscvd/dVKXa9eYQqQTAqsddHsOdVXz5ZjlRk8Ry9FzO6HeoFBzpWEmd2B3seftcG1OiO18tYotUbeqmUzWweFh2zh7v8zs9WBVxGjekmB+OGbnYFSMW0L7OTuC1o5fl3gGWBzV7bhrONbE3R+EJrWgT02cX+slHa2Ndq/gDI7Hs3ae9B6gsYmh3lnwee/FqnlrW3TwLZBW9hGwke2PTAaGIJsIL9tzx6I5YaZ9aKlwNoI8Qmm+unpUpwezGxfFIh8OXBJWztwZIKZrYlUfVu3hS01vKcXA3sCuxXicGRmFyJvvcPN7GxgXXc/rgxtWpPmnIGD3P2THMdPAp52rZJyjm9N4J00GR9NCUwXbY5cgVJ1XYpg7F0EjT+SA8JyuepHAbfrVP0+iyjAN2iZYmQmMiSPRi7vXQqoqyuyL72A1KSjKJLnMJalnm2uFCPLFlFnP7SK/huwarXvMbTpp8BdbXzNsxHR8NoFnLMsCgLeNnw3w8rUlhOBBUg45Tp2d+Dtr+GkQsc3rxLpa9HPpdoNqFgpIefJPFi0P1yf9lI8igJ6q39vlXgRlk4xMhu5556FVkNLMRQgPfeZ4YN9BM1I64a8tB4LLWm0/hkE1lPAL4MgyyvFCKJ5+hnybMtKn9OG99UTkZx+q42ve0J4fzcp4JxDgnB6nTLk50Lk0zNQqEuuY7sBr58Jpzt8fRvKhLsMeJ/w+7fgTeCXgG8Kvhz4WuH/hJCqCzb8qjcgR2fkPYtfqpSQNbIJFk8SX9V6iba0awGV4dn3QUzhVyDVQzL9+oHA/4VV1y3AVtVub0ctYWDfCSWo+0cQWM8jtc/eQK8c5+8QBuhfU2Im2TLcy37I06z477646x6BvOfyzYZgQbCXnFk2TBQeQilUDs7j+NOByU1w76+hqS/4XeBfBqH0IvhhmmT7r8BfAF8I/jr4muC3NafPaLOsuCU9n2o3IEMHvB9m7VOQkBgYhMOsMLPfO3HsDWHGPzl8mE8Aq64IVzWAbxg6LCV4LgZfJ8woNga/N7FvIvi24KPBG8D7g68kjyrCbHUxMC9c58qw3VNCLAwU41Aaj9nIlbhVyvt6KmGW9+Mwc1yI1BFPhoFxp/Z0r/VckDPMcMQzmUox8nKYaOxHhhQjiMftfuQAsG4V226I7PSkKlx7H8Rlt30exy4bJmyf55oA5FHXr0M/vQNsmOPYVYDpI2HoF9C4DPjdBUy8TwY/qfn/Roc+1X5fcz6fajcgQye8Hz6ofigvy9soMWA3RLj6Vaojac6kujUyEj4CvHcq3L4Q5p4HvmOig+4E/xB8MfjtYVn8UUJAdQG/WnYo/z9YsKyE0h7hWo+StoJKE1C/DcesgUhHh9EOEhmGezkgCKN3EdvyclTANhJLRfqvGzAU2Vsmh8nTq+F9PYhggwrC4ZQw6B5axfZuGgRFm+fqAnYO1x6R47ijgD+jeKXxJVzv8CCY1grfT6sM74hX8NcOP34A5nUOq6M8tUK+Bfjvm7fN9RJS27dZn1S7ARk64X3gmPB7OFp6d0rsvw0YE37fgIIPU/tOBl5zuNnBp4Cv2EqnbQ4+KSGg1k3s+5oldPRvhY88q4BC3jONwObVfn5l7Iflw4D1bhBO+7f2AZE5xUjKNtKu0q/XcyF3ipHd0Sr5+mpNMtBqb0KVrj0EZefNqm4LY8H+NDOIDyjiOluHycDAMM49leP4bVC6jRUcbr4ZfJW08WxoGO96gD+Wtu+n4JsF1V9i+03Vfh9zldbcrquJaeHv6sA0b+kKOxWtUlJIujA3hv8bQDq3OYmdNwFbhJ0NaBo5PbF/1cTvZZp/fogEX2tYGa3g3slxXM3DzL5hZr9CBuvhiEF5mLvf461Qqbj7HHf/q7uf5+7bAX2RvWoecqT4yMyeM7NLzWzv4E4d0cZw90Xu/py7X+rue6F39xDE6n8Ayr3VFfX9W2b27So0cwywbyBBbVO44sN2BS4zs++n7w+xUJsih4bpwFjgiuC6nhfMbBUUGzjKxUO4BdIaZTu+EzAehb98CTT0RmNXkoLkSWQH6Y2i9FO4Eo19DyD9bwIN+ba5WqhVAeXh70dAv9BBKayJhEZrWCpSfipwHOqsGeGAAYkLtYKLUBBc11aOmY4G4nVzV1ebMLOtzOwWZPvrjtxdD3T3p4qpz93nuvsj7j7GRQPTGxl4ZyGB/4GZvWxmV5jZfoFxIqKN4e6L3f1ld7/C3fdDE4u9gd8gRv0HzGymmd1kZsea2XqFDMZFtukL4HxgfKWvleX6U5DzyHmBTTyJo1EOpVQQ7QT0zPbLp+5A2Ho3IndNZb1tVUABRyIVbIqtYtZQ9JHmSht8PVJhPIziStJQM4wi2VCrAiqFZ4C5wJlm1jXQouwF3J7jvCloNbUEX6Me7hP+n4hWUK0gdf57qJ9XA9bJdGBY4V2PZl2rm1lnMxsa+OdqFmbWycz2NLN/oHf9FRTXdaq7v1fOa7n7fHf/p7tf6O67IoH1Q+QNdRzwjpm9ama/NbODwiwzoo3h7k3u/qq7/9bdh6D0Ih8h+pxvo2Dh/5nZrWY2ysw2qpAQuR45IxxSgbpzwt3fRkHnx5nZWBM6IwE1MXHcIqQKHxf4/nJhPPJ+HZPYllVABdLfi4GTE5qkKQ3QeAHykb8bGeabaI70BS2Fz0UeGBkGrkY0TtY2qq1jzKBrfR/YJfH/puijmI3UEN9N7LsBuDDx//eBRx36OjS+Bd45oXM9F7wXeG/w08C3B78mzYsvoZ9tpNnGtALSF3+AdPbjw/XSvfh+g1Z3s1EgZE16tiEN5ihke3gB2R6qmomVOk+/3p4LssGOC+//8PBNHItm9FORnfhOFGw6kDLFwqFg2GnkETRfwXvvg1guxiMb64tZjrsT+GmOukaFMWyFxLauaBKeMbAdcfNNbLE9jG8Ofgv4IPCe4CujOKirwOej2Kcu4MsmyqjoxVcjpYQ4KM8QJ4BIGJ8p18dXlc6WmW0sMuz+CakxSg40rFBbM6VffwfNrL8HrFXtNna0grw2PwEuIOEwA/RHnm3XIa/b6aHPTg192Kp3Wo5r3gJcVOX7bkCOP+8Cp2Q5pn94R/tn2b8dso+vn7Z9IPBalnM2DM9yabaPMo9vtVraLxefCGIfpYW/Q96YC+yA+/PN1VknZIec4O43lKOJbQUzG4ASA+6H1KOXex0lBoQlz38TWvIJzqMlY/s73m5f6NpAIEi9BZkHjnD3/2U45hs0Z4feAcXvJFOMvOR5phgxszWQ6nmwu1fNCSnc91TgL8ABnoHA18wuADZ194PStvdDk9tj3P0vafuORG7th2ao70HgYXcfl75vptmQXvCwlWl8q1lUW0JWtBTBxeetcFUhFZRcPat9bzkKMrnthj6oj1HgZpvHllT4/jZCapNbkWr1w/B7VNhXk6vDei9odXseWk3tlcfxq6KYq98i0+9sFJN1NorRalW9HI6bVOV7/iEy99yNGFWWcsFHav73ERN5ctsLwJlZ6h0HnJVh+3eQirtbYlsfpD34O7D4QnimnONbLZaqN6DipVlI5VoOL86n80gFy1X7vrK3rzswElHG/Dv8rvuA4Tzu22gj20gsS575sDAgX1HIO4Zc25M0Wl8hW/5PkI2re9rx3ZHqcPcq3uuzKO17F6RmfgJoyHDc/sj5oEt4J28Jk6aMkyXkYLd72rbuKP5yj8S2cYi9ZQ6yfS8GNi73+FZrpeoNaJMC33S4x2UYnJvWcXPD9ns8Dy4upK74HNio6vfVsl29w6z2I8TttVtHX0GgCP3vhQHlHcpoG4llyTPuFVYVL5GDqidHHXuhfFXPhUH4UURm+y2kxtorfUXRhvc4AKV17xz+74TSlbwM9E071hCjzYkoM8CLZHCAQMHQHyA6t7HAkMS+s4D70o4fghz0PJT/LtlfxvGt1kr7tUFlgpLDfY+lM07eiPvn+VdjpyOX2xeQPnq9CrQ237ZsgAbcQ9Hge5nHxIAZUU7bSEQzgpv5KODnyAvzJi9hYAmu1dvS3E8DkTBYFam3znD3OdlrKC/MbBww393PTWwzlCLkMGBXd/8gsW8gov0C2XtGuHsLN3IzexgJX9Bq6CPkKT4Wee4Ncbm6p45fF72rqTjcM9z9yrSGlmV8qylUW0LWW0EzpMNREPd89HK16awutGEH5In3GRoYaiKvTz0VSrSNxLLU8xyI3KhvAZYvY73LInaH3yE119eI//FXFJBipMhrdyWD911i/6lIzblBYtteYVxYhMJV9s1w3g+Qk4+H+zk3/L8QrdY2SRybWsGNAvZFKtF2Y09urXSsFVQZYGaHoRi4FOYhl+eKZ401s66IjmY04sq7HM1W51b62h0BbZV+vT3DzJZB8YA7AYe4+wtlrv9SZMO6keZ+GoTUf6l++qeLjaIc19sXGO3uw1s55hhEkvxtpFq/JLG7ETjR3SemndMPua07cBKyO12KYs4cCbfewMYokehp7n5bOLeruy8sx/3VPKotIeutIOPnWWjp3oRWURW1RyHm8DOQzvpRNEOLBv/K93UvRPszDuVYmoN3+tsGAAAY0klEQVRyLo1B6pmaDMSuhYJWpp8hequyvasoaP4jYJvEtu4ozug85GH3FXJNH4+cFooOSEVaimPyOO5ApFJbHMp0tNprAk7Pcs4cNAEyFFvmYTz5H1JF7xSe4Z7V7s9qlbiCKhJmtj4ifByAdMyTMetLZh3wDRShAzaztYAfhTonI/tSWWekEfnDzFZEtpGUHStlG0nN3J/0NrSN1DrMbG2UfWAGcLSXyQ5iZkcDx6N0600Z9ndFbOGpFda2aNBfEjPn7p/kcZ1VgdeANd39qxzH7o74WDsjoTQRrYJGAfe6+/7p48MH0K0XPL48XG3y9NsFrT7PDb+vAw5y90dztbXdotoSsp4L8uYZ8xDs4Yrsbs2L5l4XAWt6HT2Ra7Qltg1B7tEzkLqgX7XvNZaM/b8cso1cSHP69TaxjdRLQTaci5GAKEs69/DdPQMclefxXYBvItX4fYhG6w3gamRP/kaW884gnWYo+zXeQYJpMVoJzQ33vn9Iz97q+PA2PHOW1Hggx4tPSKwSO2qJK6hSYXY8UgH1oHXy3SZkrxqN+wSdagbcA3wXzcr7IpXI6mgmdb3nmLlF1A4CWegQ2sA2Um8ws12R3WgicIGX6C1pZoOR1+pGrhQUhZzbGa1+U/20PXKQeQxxaD6GHB9eBX7o7o/nqG9TpHo/G3GHHoY8RL/vElJ5jw83wqSj1aY9PHrjRgFVEpqFUyF0I3MJQsrMzkPL+Z5IeL0S6ptU6gccUX0ENvttaB4Ih6CZdkpgPe7KKdQhEBjqb0Irz8PcfWqJ9U0EPnf3M0usJ0WjlQxBAH3Xp6O+etszDJZBjfkP4Hx3vzmxfcBXsNNyynaR9/gwF/x9+Okm7hcWez/tCVFAJWBmGwJ3IO+t89x9fCsHl8T190P41QTlvOkSti1APF5vh7Y4cm19O1slEfWFkAsoZRvZnma27tSsPS/bSD0jCIPRwI/R6uSeEupaFa1yhnkZuSWDZuNWtOoRb51WP0nex9eRvektoB9yZPhLopKycoF2VEQBlYCZXQd86e6n5XHwvcA+FJdTq+l+WLC3XEo7IWNqZ+An7v6L0JYooNo5zKwLygWUmrUPR15bSWP+UmSs7QFmtg1yoPgr8nJrzHFKtnpGI9vWd8rYtp6I13Fzd58WBNY6tCQq7omcoPojL8JG4Owlk9owPtwOnS5HUnRZYG3kJfFDJL3GIqqJXkinGNAETMJ9/3LdU70iCqgEzOzvwO3ufm2OA/sirrcexVxnEdBFKr01TY4QPdH7+4WH+IYooDoecthGUgLr/ao1sMwIXpETkCfsIe7+nyLq6IY8ZUe7+wNlatdhyAFjj1aOOQW5ta9M8yTVgYddjhdTx0GPS1AU+O5Ir/kyCna6Hunz30CS7Re0EFAQxgfqlQGiXKi2l0atFMSftRi9GHOQe/dLwJdIDTMmdezjep/8evBvgDeA/x78WfCB4CuCn5jw1JkIPgz8VPCVwM8L3jsnSp34GvIqeohELhkSyRBj6ZgFDXwDEK/bnYjRYCqy4xyLyHHrmm8RxQAdg/gtjyvmfhCJ61uUiRQZ0Skd3Mr+rUN7R6L55iykov8U+IfDj7+AucuA350Hw/jfwPsvvX2ui86o6n1U1fej2g2opYJW3d8Pv3ckMGCjuIVPCZQlLyv+yUeBN4I/BN4dfB/wT8H/B94H/NGEgOoMPh58Ifhc8Engq0j4bYzsUD9BcTSptkQBFUuLQuspRo4P71JdCqzQ9leCIF6KJTyP8+8jQ9qKIurpj4Jse2TZvwoKmN8PaezOA3YmSe0EN08O3/zC4gWUu1hiqt431SzF2E86BNz9UXf/t7s3ufsUpC/fAaCzotk5H+n4dkP6uUORn/gayJjwUqK+1YGTkSTqifQap8C77v6ay2PvF8AWZta/Le4vov7gwuvufpW7HwZ8A72TD6PUF38BPjGzu8zsJDMbGJwSah7u/howGNngXjKzIQVWcTrw45BYsBR8D6n556XvCOrEe1Bs1L3u/p67X+TuD3vLcJCG6Uj31yWxcRiK3u+JvGLyQENxt9B+UBcvbzVgZoPN7B9m9rmZzUYz1JUBFmvlwyqJ43tm+D9JKdAvrf6pwFjY1MxmmdksYCaaIa9R3juJaK8IAuttd7/O3Y9y9/7Irf1+lE7kj8BnZvZHMzvVzLYMdq6ahLvPc/eTkLD5k5mdna+Addlqr0Zu3UUhXGskitXKhPFodfWzHFXN6h0OTMaKPIl0gSk68jwwK7/D2i+igMqOW5HaoJ+7p4y5BvClXEwLgqX9vwY0nQR3untDovR09ydLbnlEh4W7T3X3m9z9WFcamM2Bu5AK7TZgupndb2ZnmNmg4ElYU3D3P6Ig5+8ADwV38nzwC+BbZjasyEvvgHj8XkzfYWajkNPKUZ6BXikNU4ZAY3dE5FckGpHzR4dGFFDZsTww093nBZfYw1I7rtMyvyQcD4uugK1CFDpmtqKZHVhqvRERSbj7h+5+q7uPcveNkKC6GdlPJgIzzGxyWK0MC2qsqsOVX2kntPB4MXDd5TpnDiJyHl/kSnEkYm9p4dpsZsORR/g+nh9rxY29wC4ATkDZHL9Cq6aXUW4NaKaWWYgMzvOQp0Xqsoh5o0MjCqjsOAEYa2ZfocRkd6Z23CjXcDzvlfpSaNof/rxIM77bzexLFCrx7RLbHBHRKtz9E3e/091PdPcBKCj9GmA1lG9phpn93czON7PtAxtGtdq6yN0vQJPDa83skjwE6K2IEXxkPtcwswYz+5aZNSDm+j+k7e+HvG2Pcve38mz4Z8DkM6HpMkSmuUoooxBR4zBkh+qJSBs/CL93Uw1NwIN0dBdzYhxU8YiR4hHtEGbWC6WuSMVibYxSjaRisZ72KuQfM7OV0YpvFRQz9W4rx24FPIjUhCcAd3uWLABmtg/yyp2HPCL39MBKEQJ2/4WcJn5dYIPj+FAGxBVUsXB/DlG2FPqxprj4OvzLF1F7cPcv3P1+dz/D3QchB9RfIbaEi4DPzexfZnaRme1mZsu1Ubum07zCecbMDm3l8JeRnfgtRKmUNdkgCh/5CgmSdYA3zOzPZrY8Wlm+gWJrC21wHB/KgLiCKhUlsJlHRNQbgkAaSvMKa0uknk4xgf/L3WdXuA1bAbejFCenuPvXiX2d0IpvY5qZXn7u7j/NUteaSJj1RMJhClp5zUYpLwaVtGKM40NJiCuoUqGXaQdgEnrB0jnFGsP2SWjZHl++iLqFu89x97+5+09cadD7AOeg9/wM4EMze8HMLjOzfcxspQq04UXE5tAFeN7MNoclzOIjkPDyUKBlBEg6PkHCYxFwOXLt7gyshIJ2i/UITDU2jg8lIK6gygmzPmTOqHtjNHhGdAQEp4pBNK+whgLv0jLFSNm+BTM7AgmWCxEl1JoogHkl5NywBfCMu2cN/DWzecA4dz8vxCSuiFY0C4Hb3D0vh4s8GhvHhwIRBVR7QplTzkdElIpypV/PcY31kTt6L7QSmuDupwZ136UolvHAbN/Hm3DzBu6fmNl6yG61EKVv/3lYrUVUCVFAtQfIY+gc5KbuSJ+eQiOKqZgMXByMtxERVUElUoyY2V4oNrFr2DQfWNvdPw4H5PV93AM3HAA/AE4IcVgRVUYUUPWOaISNqGNkSTHyJS2TA76fHjybVsexSACtiTIS9AAecfed4/dR34gCqgooKHNv6xXlTDn/AcplPRtZfkmknC/qmhERFUQi/XoyOWAXJHjOJS39upntDFyHVkdvIlfx4YC5VlStfh8ZEL+PGkIUUFVAQZl7s1eSMRBwLeBaYJfWz46BgBF1gZDN9myUM+pZlk6/vh1impgDfNfdHw4nxkDZdoDoZl4d9AcyZg8tgEPsHIrM6BvOO6fIcyMi2gxhpfQx8Km7H44SAwwH/oY8BA9GNqTlgclmdkE4NX4f7QBRQLUxzOwRRIJ5pZnNMbNbzez3ZvagmX0N7GRm3c3sUjP7wMw+NbMJgXYFgAFmh28O+zZAp2E0Ux4fiVR6e6H00pegNNJGM+3/jsBPoNNQ2LeTrn+/mfU2sz+Y2Zdm9pyZrZVo70Zm9jczm2lmb5jZQZV8PhEdF4Gw9h0z+8rM/mtm3225265EnncPoIw1p9Ay3XpXYMxaZqcilV+n61HEbi+Udn1qskKUomB95NJ3IksCpzoBI/qZnWZmryXas1VoyOpmdk9IxfOeKf17qpHbmNnz4Vv61MwuK+cz6nAoR9bDWAortMzcewMyEW2LPoweKK7jPhTLsTzK73NxOH7LZWDOkzBvEfgNIRvnvJCFs3/I0JnKyvleCFhMZfbcAXxd8Deh8X1l8f0v0t3vgnT9N6GEbKA8jNMQ8WYXxBowHdik2s8wlvZXgAMRtVIntDL6GpHYHo3mWKchIXRw+GY2RywWCxCl0brAapPgUoe5k8K7/t/w/v8cfGji2wD8O+BfgE8FXxl8cth3O8xfTtcYhGTZekjz0Ql4ARFId0M2r3eB3cM9PAUcGX4vBwyp9nOt5xJXULWBP7n7E648M/ORq+tp7j7TlanzF8Ah4dgfHATvDoXunVFQR3fg6QIuNhJYH3r0hw2Q+/k77v53V2bfu5AgAtgTeVBNdDFLv4TceWNakIiyw93vcvePXFms70AxSduE3Z8Bv3H3hWHfGyieaTCyGb0LfOTuH+8j5oieE5CebmM0uzoXkfQlV1Fno9XTmkit8XLYfj10GwVvuPtzLrzt7lORwOrj7mPdfYGLtPYamr/PhcB6Zrayi3WjkE8zIg1RQNUGpiV+90GG3RcS2Xb/ErYD9L8NNmlAH1ZDOPmjAi6W4H1pQHEgnyZ2N6KZH2jGODjVjtCWw4F8E8hFROQNMzvKzF5OvGsDCFmsgQ89LEsCpgKru3j4DkYZrz82sweeFZMEU4Ef0fydrISWTR8mKkm+yMvQnAV7GrBpM1VSEv2B1dO+iXNp/qyORRO/14O6fM/Cn0RECjWXTbODIvkhTEdCYlN3/zDDsdOOhn9PULDjUkjP3JsDuVJKT0OBk7sWVm1ERGEws/5oJbIz8JS7Lzazl2l+pdcwM0sIqTWRGhx3fwhl3u0JXDgSjvgP8qY4D82oCkU/4D+ZP6dpwHvuvn6m81w5ow4N7vL7AXebWW9PENpG5I+4gqoxBDXfNcDlJmoWzGyNREbRa26DtZ+C+Y6U9A+gfAGgaVzWRDktkU9K6T8DG5jZkWbWNZRBZrZxQTcVEZEby6KJ2ucAZjYSraBS6AucEt7BA5Hm7kEzWyWQ0i6L1ONzGrUQajweuJhmd9nZSH+dD46BBVfr3d/ahPWCEH0W+MrMzjKznmbW2cwGmNzaMbMjzKxP+I5TE8BiE5t2eEQBVZs4C3gbeNqUbffvwIYA7v58fxh1MnTrhSy3NyROPAexZjaQM4lNzpTSwf61G9Kvf4SYn1O5gSIiygZ3/y8Kqn0KqZwHAk8kDnkGOdxNR3mpDnD3GWgMOx29nzOBHQbINGvfRR/SIcAKSNpNzrM9B0PTCvqUbkXzv0nASu6+GNlmtwDeC+25FhHMAuwB/MfM5gBXoOSK6QzmEXkiBurWK8zuBfahuElGEzAJ9/3L26gMiAS2HQ+10Of18n1EtIoooOoVtR4pHwlsOx5qqc9r/fuIyAtRxVevqOWU0uIIfBTNYHvQcqAi/N8j7H80HB9Rz6i1Pq/l7yMib0QBVc8QoWXqI8xliG2iLYgwWxLY5nq/OoXjxkUhVVsws/fNLAel45KD8+rzG4DtWulzM9vRzApKtdEqavH7iCgIUUDVO2oppbTUKoWyR0PzgPXNpau0tczMQx6hiFpDCX1u8PvLzPatQKuaUUvfR0TBiB99e4DUEfvXQErpchB0RsN0faGUPmdHBdhOKltrMqF2vo+IQlFtrqVY6qsAWwEvIdfbu1Beqwsd+k6ABeuC9wLfC/zDBO/ZE+DfBF8h/H0isS/BH9joYswYA9wSrvcBMrjPCWVoRe4N+jr82OFmh/vD3x+7aG2q/tzbup2IZ/gMNIDPDv3cI+zbE3jZYPYQWPxKoi8vBl8HfDnwjcHvTeybCL5t+D08cOEtA27q14MRl/H/kFruM8RiPrLqzzyWqpWqNyCW+imIHDPFINMVRcovAC78DUzoDf5CIK49KQxCDj4DvAH8pkDaeWv4f/rSAmqua1BMCqi1goDqUpH7gkEO9wbhONcTA2r4vzHsH1TV59/G7QwC6llE3roS8Bpa7WwZhMfgeXDmdTA/SVZ8Z5iYLBbhqi8D/lEGAZUia31D7T4jXHNHRAo7NrxfI5BdqFe13/1YqlOiDSqiEAxBauHxLtLOe9EgxkOwyzFoedUdRfA/hUa5B1CE5ZHh5EOBjRBFexp6IvVL26DWPM+yoXrtHO8ib52JumsLRGR8lbs/0x0GHgPdkmTF6XTk6xNekCzopHYn+3whMDa8Xw+i1dWGZbqfiDpDFFARhWB1libtnAYwE1bon9i4HNAbEXN+hBg2k+hPS9LOBBrK1NbWkYfn2fssyaVVPW/DAr0ix8Ayh8GVZWrnJ4nfc1G39gdGm9ms5eCQdLLim5AUSxG0voqoFnIg2eczXKz66deN6ICIAiqiEHxMIO1MbOsHsBJ8mUxj8DUwA1gDSbXkPpBhaY3we1laBKvMoiXJdEmR5MFdujEknZtlZk9uZ3bx4jJ7G5bYxnPMbHLatrdWNnuSRDvXB27Po75O0LkS7QyYBlzk7g1z4PZZqO8ORX18HHAl6vsUHXkeHZiLtDiigyIKqIhC8BSwGDjJzLqY2T6EfD27wsMTUT6d+Sj/wGBkQBqBMiLeigwMd6Asiak8BFuggXcBNN4iPrUDEtf8HMWorFNCu/dy9+XR7P+Xb8Mpxy6tJssXlUgH/jgwzMw6A5jZakDXxbDVouAh9zEiZ9y+uu0EERkfb2aDF8OUOdCYIiv+Gq04U3lhJqIVVDasAryt1yUXaXFEB0UUUBF5w90XIMeIY9Gs9wjEeD7/NDj/Z7Bwf5QC9R2aZ/u9w0Hjwu9Lwv+pRD8/D8evBD2Pg02RLEtdcy4iB30irICGlND+2Q5PT4LON4G9iuxjWyIy0X7IOyMbZgIjodPqsF8ntaVk9+iQlmFZ5BSQSqEyfDl4aiB0fSV8o/9E6WJXRx4q/UKbtw77MqATMGIbsz3N7Mnw7KaZ2dGltNflsn0ccGVX+Mn60POGsG8T5H43FAmff6M00dkwBhgJ3TvB+WZ2UCntiminqLaXRiz1XRDLtFyB5UW2OM3DLN+y2OGeCrTvfWCXJdvkkj23H/jvwP8BPiV4nb0C3hf8j6FN7wVPs4Xh/xHgB8krce4cOBPYoQzt2whdZx4y4XQGrjwJ7jwLFl4Wrn0i+Mjw++bgAbkQ/FLwVcAbw74LwA9vbn9jN9V7KBKAvYEtyvqMa7DPY2k/JQbqRhQEM9sBpduejnLBbYYy/oKc93anOILOeeH8SmMzoOfqaEW0Y9qOQ4HHgHR6g48Ry+kMoBf0/DfsDXxiZkeV2J7V0L33QM/zO8D8I+HZmdDlKuA0tEo6PZxwROLk0SgnxBvA5mkV3wY9toZpT7rfFjbNCKWcqIc+j6hTRAEVUSg2BO5Eaql3UV6ejwFwfw6z0RTugNCWBJ0NIA/CldDy72xkK1mADCIHZjhpWji+V/h/ruxZ+XHVtY4V0aoJpJbrBSzYBjp/iWgPZob2pexPlwLXIc85A74ks6fcNGA93VblUB99HlGniAIqoiC4+9XA1a0cMAE5+Y1Dq4LW7JxNaBbdlgSds55DAmo7tFI6Ca2OegCnknmw74cExSwk4QbDP9y91NUTITvxCPQcuiH5sw4wbwVkc7om/F0braQuAR5GxrqURMvkKdcPuF91Vha13+cRdYroJBFRftQoQaeZrXARLDwY/AiUsvUrtDLqgQJKb81y7mooydEJwExo/BpeNbMCnOqyYhoSSpuiBd3RSA5NARq3Ay6jefX0FZpV9qGZcuHLLBUfBvNegL5mdlDwuuxtZltkObw01GifR9Q34goqojKoLYLO+81sEdA0Ft68FBadIKcBfofsOCeh0fUgsgfl3IzsQRtDz8/lSf8IchEvGu4+h+AObmaPISe4fyFOurHDUVxRSkDtjnKKb4B0rKcRAtEyoD+wGRzwPJyP0pLPBn6CogHKj9rq84h2gJhRN6LjoV7SgddLOyMiKoQooCI6HuolHXi9tDMiokKINqiIjod6SQdeL+2MiKgQ4goqouOimYi1tj3P6qWdERFlRhRQER0bIlQ9B7l6Oy05+hpRqNGDwMVVXZHUSzsjIsqIKKAiIoC68Tyrl3ZGRJQBUUBFRERERNQkopNERERERERNIgqoiIiIiIiaRBRQERERERE1iSigIiIiIiJqElFARURERETUJKKAioiIiIioSUQBFRERERFRk4gCKiIiIiKiJhEFVERERERETSIKqIiIiIiImkQUUBERERERNYkooCIiIiIiahJRQEVERERE1CT+H0dJSMsehIu3AAAAAElFTkSuQmCC\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "sm = from_pandas(struct_data, tabu_edges=[(\"higher\", \"Medu\")], w_threshold=0.8)\n", + "_, _, _ = plot_structure(sm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Modifying the Structure" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To correct erroneous relationships, we can incorporate domain knowledge into the model after structure learning. We can modify the structure model through adding and deleting the edges. For example, we can add and remove edges as:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "sm.add_edge(\"failures\", \"G1\")\n", + "sm.remove_edge(\"Pstatus\", \"G1\")\n", + "sm.remove_edge(\"address\", \"G1\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now visualise our updated structure to confirm it looks reasonable." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOydd5hURdaH3zOAIlkUVJSw5og5rQFMaxZzBsNnwBxxxZwzZte0q2KOqGsOq5izoq55FRRzRImKzO/741QzTdM90/n2zNT7PPeZnr73Vp2+t7vOrapTv2OSiEQikUik1qhL2oBIJBKJRLIRHVQkEolEapLooCKRSCRSk0QHFYlEIpGaJDqoSCQSidQk0UFFIpFIpCaJDioSiUQiNUl0UJFIJBKpSaKDikQikUhNEh1UJBKJRGqS6KAikUgkUpNEBxWJRCKRmiQ6qEgkEonUJNFBRSKRSKQmiQ4qEolEIjVJdFCRSCQSqUmig4pEIpFITRIdVCQSiURqkuigIpFIJFKTtE3agEikJjDrCewB9Ae6AROAd4AbkH5I0rRZaC52RiJlwCQlbUMkkhxmqwLDgU0BAXOl7Z0KGPAIcDbSa9U3MNBc7IxEykh0UJHWi9lQYATQnsaHu+uBacBRSFdVw7RZaC52RiJlJs5BRZotZraEmY0xs4lmdmgjx/Uxs0lm1ib8P/r/zG7CG/0ONP07qAvHjQjOomp0MfvpcbiIIuw0s6vM7MSKGxmJVIg4BxVpzhwDPC1phcYOkvQF0Cn1fzfovBrsCMxRYH2pxv91pNebOtjMTgEWlbR7gfWkClh1bpi7zofvGuUG4J/A82l2CgbkY2ckUqvEHlSkOdMXeK+Ik/rUQbtCzxMww4fZhhd6bpEMtzycUw6qaWckUhGig4o0S8zsKWA94PIwfHeYmb1lZr+Z2fjQe0kd28/MZGZtMevZFbqnGv5TgPTuzTh8x5/h/4HA8cBaeLdkLNRNgM06m91iZt+Y2VdmdoaZHRteTzKzKWY2GTgZ2DW893awZZyZbZhm2ylmdnPa/4PN7PM6s59Phy1T738b6v8pzdY3gR7Au8BQ4CW8m9jNd9cNgUHzmF0Yyh1oZl+a2TFm9n2wfWsz28zMPjazn83suDQ76sJn+tTMfjKzO82seyH3KBIpleigIpXFrCdmwzC7CbMHwt9hmPUopVhJ6wPPAQdL6gS8DQzB2+fNgQPMbOssp+5RaFjQTcA1wES8y7YHtFseFgYWBVYEtsKHG1cFHgCuAJYHzgSekNRJ0vJN1WNmSwNXAoN/gfN+Ar4M++bHneWdGXbtDCwHXAWsCUzC485T/NXD0VPMj/esFgROAq7F/fPKwDrAiWb2l3DsIcDWwACgF/BL+FxNU6F7Hml9RAcVqQxmq2I2CvgcOBVvCLcIf08FvsBsVAifLhlJoyW9K6le0jvAbXjjmkn/ugK/93sCy+ATtj8Dj0Kbh+EzSZMlfQ9ch3delsY7X13xaLo/gULWJm0PPCjp2a6wzJnQNt3QPYBUV2sG/gEHN1JYHbSZB3qmvTUdOFPSdOB2YF7gEkkTJb0HvI87VvBO2fGSvpT0O97Z3N7Mcs9bV/meR1o+MUgiUn6aDotOreEZBGyMWclh0Wa2OnAOsCwe/DAncFeWQ7sVWnbvtNef4618L9hxstnm4e064Hu8EV8WH4nbKrz/aQFV9QLGp+zsCMyTtnMQ7jXGAh/hXnC1Jgps5/cgxU+SZoTXU8Pf79L2T6UhmKQvcK+Z1aftnwHMB3w1W0UJ3PNIyyf2oCJ5E+ZxFm3ioFRDlXdYdF/4x9lmF5do3q3Av4Hekrrio17ZAgzSR8DoCExJ+//bLCekF9Ib93w/we2SuoWti6SFJK0N9AFeBZ7Ah/tWy7hmk/Frk2L+tNff0OAPJ0xh1jmn9njo4c348F567ylXJMV078kVw3hg07TP2E1Se0mNOaes93wcs8zrZQ3ZD/N0Cxdpa6SFEh1UpHz40E2qocr/NLCVYX/MVimh9s7Az5KmmdlqwK45jnun3he0ArAC8CzwBfArcHYTlSwAbAgzNoV+ZtYlBBNsaGaHm9mc+PyX4XWkppDSp73GADubWTvzz7t92r67gS3MbO2J8N4J8Gd69wV8ku0G3BOnO6j5QmV/pL1XDzN+8p5divnTAzSa4CrgTDPrC2BmPcxs0GxHFXHPBwL/bHBSqwCEebrP8i0j0jqIDipSToYz65BS3pgPy5USFn0gcJqZTcQDAO7McdzI9N7GRsBOeCTByviESVPcBNNfh//hcza/AJcBBwM/4vNRg3DHMwT4DHjdzN4Mp58ILBLOOxXv+QEQ5oEOAm7tCsfMDSyUUfda+I92JXwMLsX6+DzZ/PjEUooXXaevSbLMLV2C+8HHwzV9GVg9y6lF33NiKHykKSTFrZVuwN/x+YSJ+LTGBkAb4Dh87mQi8AY+bAbeExgKfIIPlV1BkMv6DuY7Fab3AfUADQZNACls94OWBnUFDQC9n7avL+gJfz11Lfgb8DrwGz4/cmGoeyDwZYb944ANw+tT8B7IHcHuN4Hls352GCWYoTQbCthmCO6pyj3KYed6oGsLtBMfFazH55km4VGHAv4P70A+G467Cx/p/BXvXC4T3l89vN8mVeYSsOeyUC/QDNDZoIVB3UE7gH4Ktoz1ejQddByoDjQnqCPoAPhTHi0vfFEzeCfxH7i24CTgBdz3Xow79g+BFdM+Wy/gHjwgZSxwaNK/rbiVZ0vcgLgldONhCXyeoVf4vx/+ZD8MX1qzBD5UtTwwTzhGwIN4oEGf0CBsIomD4c6Fof5T0ETQNqDdQwP1EagD6HHQH6BzQYuAfp/dQU3p7U5ncKivE7BGeJ2Pg5qO91zaAUeHxqrdbJ8fVhVMLtJBTRasUpX7lMXOV4OT/60IOzOuV79wP2/Ep+LmCu/vjQ+Xzhkcwpi08z8FNkr9vzy8fQb8IdDFoNVB40HTQPuBds7ioBQeUNIc7BT5vcp0UD/indr2wFPhXg7BH6DOwBVEwDuUb+C95jnw8P/PgI2T/o3FrfQtcQPiltCN9zU83wMbpjfieE9qUI5zBKyd9v+dwLGS6A/fXJHWQH4IahsapdPCE3Vq3wxQL9DTszsoLe29plOBeTPqzsdBvZy2rw4POlgn6zWAoUU4qcmCoVW9V2l2DgF1AV1fpJ05HNTCjXxHuoVjuob/zwCuC687zwl/jgt1Lgl6Ms2Gr9PufxMOSnInmemgrk2z4xDgg7T/lwMmhNerA19k2D0cuD7p31jcSt/iHFQrRdL/gMPxhv17M7vdzHrhUWSNhUanB7pNIYQlT4BOfdN29MWjtr4DvmbW+ZK6UMns4WBwE/wXWBz40MxeM7N8poVSpEK0kZQKUuiV9UgPcT4qfIbMWIRZqAdNAd0Jt1Pt0Og0O0dC/a/4uqxszADqC1czn3nNzKyNmZ0T1CN+wx0aNExr3QpsG4JBtl0cfk3d18+BbXCP1g1YCu/qpMewN0K20P/M8PfGwuF7mdmE1IYPUc+XX9WRWiY6qFaMpFvlodF98SfYc/EGa5FCy+oGkz5P+/8LfJHdfLiHSN+nUMmCWcpZCb6StAu+wPRc4G4z60hGeHZQJs9UJuidtr8OjzH4OqfR3ogPAO7DG/apGUdMFfz+IPxxAvx9J9jKzFbOWV6lyMNOYNoH8PZGMMbg6lwlNfHerniAx4b4Mqt+4X1zM/Q+fis3BXbdwgNFAL/wj+ATk6ltGtnvcZaQ+Amzv5U344GxmjUcvrOkzUooM1IjRAfVSgmpKtYPT8OpRq8eF8U+3cwWM6e/mc3TaGHA2vD8hT6cwyT8EXYn3EntCDwE/AefJBqBT3D8dfZiph4Jbc2sR+gBpRqueuBjoL2ZbW5m7YATQjHprGxm24aItMOB3/Hos9xIryNth8+pnYQHEzwQ/p5k0HsQHHkRbIYHiIwyz2pbXZqwE+izHKz6FHTBFwln4zt8jiYXnfFr9hP+MHBWlmNuBQ4D1h0CDxOc5VBcszD1IPIDcH+OSubDJ4kCU8kz0jAHrwITzezvZjZX6AUua1GtomWQ9Bhj3Mq4QU/BMMFNggfC32GCHpnH4pHVr+IRbz/jwQ+98JGZE/BJ6YnAa8BC4ZyZ8wTh/xuAMySP4jsZpi8Emhe0G+jntHmGUaClwhzKuqD/pu1Lm4Oa2tGjyL7H/dx7wNZp9e2Jzyt9j0+sjyN3FN9bwErluK64n30b2AE4HY9umyPx+53d1o3w9r99ln2D8M7tBBoCE9qm7e+E+5WJuK8ZkuWe98EfGB4K37epqXnFEaDFQZ3waL7hOYIkXgQtBuoGOih3FN8ZaXXuA4xO+39R4M+0/3vhyk/f4lF+L6e+F8X+PuJWG1vMqNsSqJV04K7DNojieub1wH14L6GIqkvMvdR0+QPwyfyl8cbwS0kHVqKuUjG/D29IOrMKlSV2z/OmVn4fkYKJQ3zNHZeLGY03Eu2Z9cdH+L992D+6whlhz6ZIaZ0/PaaiKSGHxJD0DJ7RYhgufrqeme2XrFU5OQo4wswy1/lWgqLveTivsve8tn4fkQKJDqo5U4TuHZVMW+5Pn6nIuLyph6lHw2SDNSpiV/kYhoc8d8cbtDPMbK1kTZodSWPxtB3nVaGyou55OP4oKpnxt9Z+H5HCSXqMsbVv+ILYMfi4f/4r4Gt5sWnD2p2m1BpmpNbsAH/Bw9uPJ6hT1OKGSxXdHV5vgkcJLpS0XVns7IjPN2VfB1YD97zC9hT0+zg5zJtW5fcRt7y32INKnmPwVfGdJV1awHm1q4GWZ1h02D8A6Sr5U//awC7AuWZWbKrzSnMBHi24vqRHcc26e80sc+goUSRNxnt8l4WQ/EpXWPA9L1fVIUvx1KCIPsnMJn3ii71r8/cRyZsYJJEwZvYknrrhnwWc1BOPsir2BwjeWPRBKiShXuF4FtU98KjBbngE2TvAyGx1h5D2R/AovAPVkL+oZjCzbfBovhXw9bG34nNoQ1RDP6jg5EcDt0rKtTaqEhUXdM9Lr87GAftIejK8UfDv4xR8UdfNDW9V5/cRaZyku3CtecM1xmbgP4ZJ+PqSt3Ch1PHAKWnH9sMjkPbqAhO6ga7EtdmWw/XZDkobsvgkhHN3Ac0D2jEj5PePoIEWyh6N/8DBQ7lfAC7HBUM/BDao8nXpDDyNR8vNrqWX/H0z4EngkPB/B1yc9sikbcti6/L4+qfuSdtSwc84jvSwcg8dn/ISaM3w2+hPg7SWQJ+F30cn0Ibht5Ma4nsa1MtFcI/OWUfcqrLFIb4EkbQ+8BxwsKRO+FqbIfhT5+bAAWa2dcZpq38JD92Br0Q9E28p38OF8Z4JB52Iy4L/guv9HJJRiHn0Uv8cpq2OzwfNC5yML07tXuznLBRJE/GFsR2pzeEz4Q8TJ4VFxVNwpZ9hZrZRstbNiqS3caXvU5O2pYr0/wrm2hxf0PczPi67Hb6AGFwyY2VckfZEYGRGAeYPIbl+H5EqER1UDSFptKR3JdVLegfvQQzIOOz0ztDlb3jrndIEWhBYB+9+gct5f47P4LfHJ3eykCv9+ffAxZKmS7oDF5DdPMexFUHSVLxN+Q14xMy6VLP+ppDnbroFF1BF0ufAzsDNZlawVFSFORHYycyWS9qQCnJfSotvXdjwZvwJZzO8kdsIWAWXvvgCX31+Oi5Fsi6wZfYyc/0+IlUiOqgawsxWN7OnzewHM/sVj26bN+Ow7wgSQHMxqyLmXPg4IXh8sYDV8ER212WvMpcG2lehl5Dic3KJrlYQSdPxxLEfAP8xs8xrkTSnAIPMbEWYuVbqNOB+M+ucpGHpSPoJt/WSGg4+KZWtFbT4noUnP8clSbqlbc/jMiRfA3PjD3gp+mYvsxSNwEgZiA6qtrgVz2LaW1JXPO12tgblHWaPkpqF+YFr8R/j1Xi62f/R8KOcPKsG2vwZpy+Y0ZD1oTHR1QoiD5I4EJ+veyYortcEkibgvZPL0q7XP/AFvSODYG2tcA3+sFNZ1Yba4J1eMH0ws4rXTgaOBRbAh74np53wRdrrjsxc1PUO5BQmjlSBWvoBRTw44GdJ08xsNXyoPBsjye64ZnIXPvcE/rRo+M3ugQ8H3gRtx8JNZrY3s6uX9wQONbN2ZrYDnj3h4WI+UDmQ83dcGPU5M2tM8LTaXId3XneBmfNTB+NO/4QE7ZoFSX8ChwIjzKxDU8c3c0YOgfoHgMdoiEIajf8m+uLDfScDf+A9qwfSTl48HL8MfNWIMHGkCkQHVVscCJxmZhNxheo7sx4lfY+HYufkNTzSoRMubX0JDTLW10D9GTB9YZ9bWgZ4MeP0V4DF8DnkM4HtwzBRokg6B1cGeNbMlknaHpjZwzsUOM/MOoX3fsd7Kvua2aAk7UtH0mhcSPWYhE2pLNL3feDh+6D+LPyhrDdwPg2Jv27Fv+Td8eiRIWmnd4X6k+CN9+FCPG3ZZBqe9yJVJK6Daq64AOZo0nIkFcAUfLHkbDIzZrYnHnKeI64iecxsdzwwawtVUiqnAMzsJjyz6/Fp762GZxoZGIIqEsfM+uAh8SuHwI6WSYV+H5HqEntQzZVa1kCrMJJuBvYHHg4q47XA34H90yP4JL2K36P7zGzuxCxLQ9IXwKW4g2+5tOLfR0siOqjmTAFpy8P+1I+vumnLK4Ck+/F5n7vNrKoh8Dns+Rpv9C/MeP9GPNfW7VWRHMqP84FVzGz9pA2pKK3499FSiEN8LQGzVXDtsM3Ine/mYTzfTYt6MjSzNfAke4dJuj1hW+bE10wfJOmxtPfbAo8Cb0qqifkfM9sWn35ZMQRQtFxa8e+juRMdVEuiyhpotYKZ9ceDRk6VdE3CtmyB96T6S/oj7f158NiVEyXdkpR9afYY8ARwn6TLk7anKrTS30dzJjqocuIildl+ADfEH0BlMbPF8Ab3ckmJza+Ehv8h4ElJF2bsWw5fz7WJpDeSsC/DnmVwzcOlJf2YtD0tntg+FEx0UOUgppSuCcysN+6k7sZ7Kol8uc1sCVxwd1lJ32bs2w6fp1pVvlwgUczsEmBOSTFJX6WI7UPRRAdVKg1ZO9vTeNBJPb7+L07CVhDzYZzHcAdxmKSmJscrZcf5wDyS9s6y73RcY3HD9GHAJAjRhR8Am0p6q6njIwUS24eSiFF8pZBHSulNmamUPFtKaTPrZ2YKk+iRMiAfKlkfWBG4PsFrezqwSVgLlcnJ+PDOJdU1aXYk/cLsck2RclBEynmDKx82O7HitjUTYg+qWMqwENBcqWEsnvOoZUdSVZkg5zMKH0LZOag7VNuGPYADgL9m9uSCOvvLuGp80oEdbYBXgQsk3ZakLc2JkCixF9ArfQ7PzN4CVvgUpi4863Be02UCH8HUxWHdGFEYe1ClULsp1yOEHE2DcCm2B1MyRFXmJrzNGZy5Q9JvwNbAGWa2VrUNy7BlNrmmSN6MJegwwsxAmA4AVqR+Xzgvtg9EB5U3ZjbOzIab2ft1ZhP2gK2mQd0vwBa43tfc4XW6aNdAIJXLfQZwNDAv1P0FtukFO1XxI7Q6Qq9pZzw78ePVVnMIvaZDgLOz5bOS9DEuA3dXCPBIDEkv4PkuY8NYGDcxq5TfHj08QSQGdb/jv/k+eGqcocyahuB8XF29Fw0pcczb5c3mNHvBzPZJHWtme5rZ8xX7JDVIdFCFsRuw8Udw4cdgZ+Azm3vhCZO+wPvzB+c4+VpcUuAt4HWY1h72rYLNrZowdLoPPoQ12szma+KUctf/Kr5IN+u8gqRH8bmoWsgcPJtcU6RJXga6mNlSYah051EeqQd4eo+PgTF4upuv8IRh4F+KC/Cw00/wzNhpaMHZ0+C0OqKDKozLJY1fDBY7EepuA+bBZas74Lkyjqch7Xomd+Jp2nv7eXOd6MMDkQoTejJH4HNSzwXB1GoyHNgzhJ9n4zy8jbomyUAFSV/hbeaIpGxopqR6URsBH6wa8h8KT8J1Ea6a3hk4DkjJndyJP9wui+egOmXWMufqOGtOxVZJdFCFMT787dYXz+A3BVct7Qt0wdNHT8CH8zL5GndOKZaJ179qhJxSp+IJBZ9rxFlUou7vgHOAi7M5oLBe6//w1CdHVMuuHFwELGtmGydsR3PiJjx3257AjRZSxf+Atw8r05DVd5PwPszeHmRm9W0DrT66NzaQhZH6Pk34Ah83HoEnVXoF+A14NhyQLTZyARo8HMAHTQtYRsqMpItxDbqnzWyFKlZ9GfAXIKuwbQjq2AYYZmYbVdGuTDt+x53kJWY2R1J2NCdC2pKxuNbfKIVU8fPiQ/7v0ZDV91dgUjgvsz1Iz+oL0M7XRaVHCbe6Ib/ooArjIDNb6BP45HSo3wmYiH8JuwE/4y1fLnbE8xx86cdOPQ36VdrgyOxIug44DHjMzP5apTr/wEd4Lw6istmO+RwP6rg54XmgB/EGN9d0amR2/g9YX9LkengXvHHdF/f2KcmQr/BV5ODtwQ3A+3hPK6PtmDoffAhsa2YdzGzRUEerIjqowrgVeHwJOGoR0Al4izMVf1paA+/C52JfYGNgeWAlaD+1IcAvUmUk3YXrot1frR5LCIh4D//a5DrmGbytut/MOlfDriw2CLdxeLWDSporkj5NJc/8HG5MvX8usCjeNnQBNsRHXMAX8R+OrypfNPxNw1aCg/Cs9N/h6/0TFxmuNnGhbp6ERXn7SHoyvDEKX2dTsJOvB9XBvUjbldXISMGY2Tp4WPBQSaOqUN8i+Ihw/5BDKtsxhs+vzwNsn6Bc0wXA3JJa3ZN7yZTQPuBD//fF9iH2oErhbHyMuGCmAdvA1DjGnzySnsM7vlcE5YdK1/cp7nzObeQY4cNr8wMnVNqmRjgN2DSHXFOkcYpuH8J5Z5fRlmZLdFDFUkJK6V9h2H3QCXgxjC1HEkTSm8B6wOlmdkgVqjwLWK+x+a8QrLAdsK+ZbV0Fm7LZ8BseGX2pmcW2ohBiyvmyEL90eSKp38zhvYY3i0opvYA0Ao/Yuh54ycx2L7/FkUKQ9CG+SuAwMzuhkuuRJE0CjsEb/pxp4CV9A2wLXBtyNyXBjbhcU/yOFkpMOV8ycQ6qHJSQUtrMlgfuwFekHxwar0hCmNkCwON4sNWwSuWUCg7wOeAGSY0Gy5jZEFyJYrWgPl5VwhDffcCSoVcVKYSYcr5oooMqJ0WmlDazjngE+trATpLGVMHaSA7MrDueQO4dPHgi27rrctSzEt4wLSlpQhPHXgQsDWyehPK9mV0P/CDpmGrX3WKIKecLJjqoGsLMdsF12c4ALksqI2wEQoj3fXhKlMGVSixoZlcDUyXlDD0Px7XF5dvekjSsErY0Uf/8wH/x1CEfV7v+SOskOqgaI4Qh3wZ8C+ydnmcmUl3MrD0+/NoOD/cudMI7nzp64GujBkp6v4lj58FFb0+SVPU1MWZ2FL4YNasaRiRSbmKQRI0RwpDXxtfzvWVmAxI2qdUiaRqwPS4S8ki2lBllqOMHPPvupU0FZkj6Cc8hdbGZrVxuW/LgMmARM4sOKlIVWlcPyqwn2ceAb6jFMWAz2wRPE3MtcHrMupsMIcT6cmA1YJNy92rD8N0Y4ERJ9+Zx/HbAhXjQxHfltCWPujfBHdWySWQpjjRCM2vf8qF1OChPzz4cVxfJFUXzCB5F81r1DcxNiCq7Ec+yuZuk8U2cEqkAoXdzFq4OsFFITVHO8jfApa+WljQ1j+NPw9dubVCp+bFG6v438IKknIuNI1WkGbdvTdHyHZTZUFx0vD2ND2nW4yu4a24dQniCPwbXndxP0v0Jm9RqMbNjgf2ADSV9Vuay7wbGSDojj2PrgHuBryUdUE478qh7UXxZRE65pkiVaAHtW2O0bAfVcPM6NHVoGjW7WM7M1sADKB7E1+gUK6USKQEzOwDPTbmJpP+Wsdx+wOvAivn0lMOc2MvAxZKuKZcd+WBmZwELSRrS5MGRytDC2rdstEgHZWZ96uDD30Ftm7h5Q4EFmS0f9xRgANLrZnYcsLCkfSplbyGYWTdcy21xYOeggBCpMma2G944bKkyDpuY2anAEpJ2zvP4xYHngW0lPV8uO/KotxOeDmJHSS9Wq95IwIf1RlOYc0oxs30rq02VQFKz34Bx+JBLw/ueOGyGQPluT4MW9NczBPck/bka+byGDzP9gGeNtqRtao0bsCWe6mdgGcvsAHwODCjgnE0ICVqr/Pl3w3t8bZK+F815C+3XVDyX4Xd4mqhOjRy/5zLwY77t21ifl9L0hvdqun1L31pmmLlHs2xK8WH0dcBmYeV3zSHnGmAgrvV1SyVCoCONI+kBYCfgTjPbokxlTgGOxsPO80r5Lc8zdTFwr5nN1dTxZeRW4Hf8ISlSGltK6gSsBKxCIyr2faFzV5ibFtq+pVNzDsrMjjWzT81sopm9b2bbpO3b18w+SNu3kpndBPQBHjCzSWZ2zHNwuEH7P/FVlqtk1HERsFV4vSf+TZiMe7SvcZnxTtD+DTjEzE4xs5vTbFjDzF40swlm9raZDUzbt6eZfRbsGxuGgSqGpPeAVfFM0m+Zd/sjVUTS03hP6l9BCaQc3I2vvdq3gHPOBz4Grqmk0G068sf5Q4AzwtBzpETk0aGPAMtma0/MbKkv4cKXoa4THksO8BCwIp4UsTdwSlqZ64a/3fC27SXgRGi7iiuTAD7/aWZKPRRVuy3LSdJduCzd1x2AXrjz3An3HQuE97/CG2TDk1D2TesizxziGwOjCF3ayaBOoI/TuryrgG4Lr/cAHT/7EF9quxG/1zeHehYEfsJFH+uAjcL/PYCOwG/4/AHB5mWqeN22w4ebjgbqkr6PrW0Dlg3fz/3LVF7/cD/nKeCcDsCbwJFV/uzXABclfQ+a65befuH+5T38gSNre3I6vLBWlumJd0AzQG+DeoLuzT3Ep5NB68NnaTb0w49rm3Rblr7VXA9K0l2SvpZUL+kO4BN8geQ+wHmSXpPzP0mfZyujjT9IAP6LHYSHvhEK+5CGHlQTZD4V7g48LOnhYN8T+Bj8ZmF/Pf7kM5ekb+Q9nKog6R7ceW8DPGQ+zBmpEvJovgHAsWZWsqCqpHeAO/GkgfmeMwVXmhhmZn8r1YYCOB7Y3cyWrmKdLY37zGwCHvDyDP5gnLU9mSNLYF1g4zIAACAASURBVMRAYDn8qbk/sEsopDHauYRXLhJry9KpOQdlZkPMbEwYQpuAP5nOiz9ZfJpPGTPc+89kVxoc1K34LzjP0JdMhem+wA4p24J9awMLSJqM9/iGAt+Y2UNmtmR+1ZSH4LAHAG/gQ34bVrP+1o6k/wHrAHuZ2ZllGGo7Cdg+pGTJ14Yv8O/hTUHXseKoQa7pkmoNL7ZAtpbUTVJfSQc21p78kSUJ4iv4qu0eQFfgKlzluDGmw/Rs79dCW5aiphyUmfXFZX0Oxoc2uuEKygaMB3L94GaJlf/NO0kz2QgPdxuDO6pdc9U/679TcZmQdMYDN4UvUmrrKOkcAEmPSdoI7xJ/GD5LVZH0p6QTgCHADWZ2tpk19qQUKSOSvsSH/TcBLrMSMtFK+hk4mQIbfknPAqcC9wdV9mpwJf69H1Sl+lo8udqTH2F8fUYCxF3xUaHx+IT0UBoaxWxfnPYw/XufPkkxfz51V5uaclD42Kdwf4KZ7YX3oMBlYI42s5XNWTQ4NPDQzIVThfwL7kkvtB0+gTUMn3neKEfl8+ETSr/6vwaMzDjkZmBLM9vYzNqYWXszG2hmC5nZfGY2yDy30+94yGhTWTQrhqT/4BFBywPPhkWgkSoQehTr46MtI/ONxsvBtfhQ8w4FnnclPh9+YylOMl8kTQcOAy6sciRhiyRXe2Jm7e6C77+CunR9q4lAd1xO4lV8pChFD7yhT5c9WRH0HvQysz5m1hWXSmq07kp8zqaoKQclTzcwAv9hfYcPq74Q9t0FnIlf+4l4rp7u4dSzgRPCsNvRI93PoLSLuivwJP4rz9VaLImP3S4MdIQ2ljFGK1/dPwg4Dnei43G/Vxe2I/FAwJ/xobaqStBkIul7YAvgLuBVMyu0kYsUiaRf8V7UvMDd5qk7iilnBnAocEFoMPI9T/hIRE9mW4deGcJD0Vv40odIaWS2Jxvgvuezr6H//DBhfvzLBfAPfDy4Mz5puWNaQR3wScK18Cedl6B+Y3hQcDs+SvQGrk6Tq+7E2rIWqSQBtJ6V1nlinnb6NuAp4AhVILdRZHbMbA68590dn2eYVGQ5twGfSDqpwPPmB14DDpF0XzF1F1hfP7zBW0FR2LhkzOwveM90CJ59+SJJb7SW9q2melBlxeVnjiLLhGITpLSqav7mFYL886yMD6O+ambLNnFKpAzIlcZ3wUOJnzBPJ18MxwAHhgarkPq/BbYFrjWzZYqsu5D6xuGpSc6rdF0tmbDe8k48SvgPYHlJu0t6A2g97VsSse1V3WCoYHIesiAzwnFDE7e5ghs+t7YHPkS5P1EmqZrX/UJ8SGX+Iss4gSIlavAn8P8B3avwWVNyTesmfd2b0wa0wdczvgCMxXtOnRs9L7Rv9S20fWu5Q3zp+PDWcHy9ksieL+VhPF9K83iyKBEzWwIfg/4U2FfSLwmb1OIJkXgn4uvpNlKOdXyNnN8eeB9PufJkEfVfCCwDbK4KJ780sx3xqY+VK11Xc8dceHdv4HB87n0EcF/e181slbdg5LKwRDvvbbWY9q11OKgUrj21B7NnnBxJM804WQqhwTsXD/zYVVGVuiqY2WH48MxGkj4q8Nyt8WChFeSRc4Wc2xaX0RkjaVgh5xZKcMZPA3dIurKSdTVXzGwhPJBlH3w+aYSkl4oopw7435aw379hBVpQ+9a6HFQkK2a2FS5XcylwrjxyLFJBzGxPPEPv5pLeKuA8Ax4DHpR0aRH1zoNHg50k6ZZCzy+wrv7AE8BS8jVdEcDMVsSj5DYHbgIuUQnJL831QC/DE0i2qAY9OqgIMPNp7hbgT2B3Sd8kbFKLx8y2w9crbSPphQLOWxpXsllaRTwZm9lyeDTnJkpNulcIM7scQNLBlayn1gm9nM1wx7QY7lCukZSpVlNM2TfiveILSy2r1ogOKjITM2uDzxscAOwt6ZGETWrxmNnGeBj67pIeK+C8i4COkvYrst7t8KCN1SR9V0wZedbTHfgAH87MVGZp8YRFy0OAI/AIuhHAnYUOzzZSfhfgC2Bx+brHFkV0UJHZMLN18UbzLmC4PFQ6UiHMbC3gXuAAuehvPud0wyVoNi+2F2Rmp+ESbhtU8h6b2QG4ttt6LW0IKhdmNh9wEK469DL+MPBMuT+/me2Dfwe2afLgZkjLXQcVKRq5ltuKuPbhC2a2aMImtWjC8N7GwOVhbiqfcybgvd1LSxBoPQVXCrikyPPz5RqKk2tqdpjZMmb2L/zhoSewjqStJI2ukHPeG7i+AuXWBNFBRbIi6Sc8dcdI4CVLKmFZKyEESqwHnBai/PLhemAOcusfN1VnPTAYGGBm+xdTRp71pOSazjezYpQPapqgDbqRmT0C/AdflL2YpKGFRmkWWO+SuDJbix2Kj0N8kSYxsxXwNVMvAwerSLmeSNMEAeQn8eiu05t66jazNfGh2KUkTSyyzsXwPETbSXq+mDLyrOd24CNJJ1eqjmpiZnPiKiFH4g/7FwK3SppWpfrPAdpUeslAkjQvB+VJ+LKtY7qhucb5NxeCUOlluObkTpLGJGxSiyXo5z2GO6qj83BSNwJfSRre2HFNlLEJcB2wuiqkoWdmvfGsN6tIGluJOqpBCNUfis8x/RcPfHi8mvNrYU3bF3gm3vfLUGBttq1JS1nktcGqglGCqYIpGRIeU8L7owSrJm5rC9/wJ8bvgUOIMkmVvM5z46r+/8Sfkhs7theen27REus8Btd+m6uCn6touaakNzw8/ArgF3x4dbkEbdkceLnksmq8ba39HpTZUPwJpT2Nz5nVA9NwIcSrqmFaa8U8U+vtuBz/3vL5qkiZCRI49+GBDLurkUg78zTza0vaqoT6DF8LVw8MVgUahzS5pn3l6TlqmnBN1sGH8dYCrgauUMLrBM3sbuAJSVeXUEjNt621HSTRcAE70LStdeG4EeG8SIWQ9Cn+Y/0YGGNmAxI2qUUin+vbAg+EuL+JAINLgKXMbNMS6hMuu7M03iCXHfn8zFF4luCazfRsZu3MbBdcdeOf+JBrX0kn1IBzmhfPu/q6mY0xs4lmdmiBhRTVtrYzm2ZmCzdxfNmo3R5UK8l30twJDeJ1+JPlGYrCoGUnNOTXAX2BLeXJELMdtzk+Ub9cY72tPOrrA7wC7CHp8WLLaaR8Ax4HHlARck2VJGSX3QePOhyLX88H5RGPNUGI8lwVz3b7m6QjCiyg2bSttdyDGo53PQEoUByuPWkpjNOx0tJvRzKQq02sBKwNPBUmwiNlRK46sAc+af2UuehxtuMewlNqFPY0PXs5X+ALa2+qxBq40FM7DDgx12epNmbWL6i9f4Z/n7eTNFDSv2vMORkNa5/6Au8VUcwsbWuB5GxbK0IVJvPGAUfjP65fgTvCh9wTeD7jWAGLCnoOhj+HgjYFdQA9AXoItBSoE6gX6Py0Cb0HQMuDuoLWBL0F0wQ90mz4e7DhdzxN+z0ZdV+KizYmMunZ3Df8YWc4ni5gq6TtaYkbnjbhTFw6aMEcxyyOB0wUlXMqo6wD8Aaw8ZxExZd/EXB1wtd0deBO4CfgfKB30ve5CXtXCu3ZU/hz+zRgEu7w3wJ+A8YDp6Sd0y+0rXvVwVfdQFeCXgUtF9rMg9La0k9A64K6gOYB7Zi2D9DHMO0gWDbUm9qmEJ49Qp17h+/pLzQMjxb+eatwQcfh47i98LTXH+Ahmo05qGGD4c8uoOdBM0BTQfODng0X6mfQG+H1m6AeoJdBf4JuAPWB+gnw9zQbxgC98VwpCwCTgW5hf1s8Mm3lpL+AzX0D1sSHRi4F2idtT0vc8Gi7seSI2sOz2V5fhnoMV4G4F6irwOfoBnwLrFTl69cGzzL8fLiOh1fKCVfA9stSzgcfptsnvB4ILBceFPuHB8Wtw76Ug7rqZxj+CEybEzQI9B3oy9B+jg7t6c6gM9La3ecyHNRHHtl3dIZdtwC3hdeD8J78UqFtPQF4sZjPW60hvkslfS2X3H8Az1nSGP3roM0gfCa+Du9ytcPDf37DY3BXCgdfg6eGXR3/5u3hx9ujsH6GDeMlTZVPcj5Lg/TKJsCPqrCyc2tAns9mRWB+4OWQGDFSRiSdB5wDPGNmy2Y55AxgYzNbvcR6hOcr6oknWiwrcrmmE4DLSpBryhsz62Rmh+DBPcfggSWLSbpYRS5yriYhAnIXXN1lFuRSSu9KqpeL8t4GZAYvnT43LL0JzNkxFNQTWBAPU0zlfGmHp0P+Gm93184opM7f7p9m19+BJfFeE3gH5GxJH8jnpM8CVgiL0AuiWg7q27TXU4BOTRzfDby7k849eFrIvviVT2X2+hwPR+mWto33SudNOz1z8eFIPLMp4e9NTX2ISH6Ehmcn4B/A82a2VzUaoNaEPLz4aOBJM1stY99vwLF4w1/Sb1webLEdsE9IllhurgfmpEi5pnwwswXN7Gx8JGUgHkK/hqS71LyCerYC3lGWRc5mtrqZPW1mP5jZr7iTmDfjsO8IbetcwHxpO+bCx+nAu98CVsPTL1+X3ZZuod5N8eHFrSVNDfv64lGaE8xsAr5MwnBfWBBJBklMJi2KJKyeTzEB/BOlsypwPz4WtzWwY3i/N66aOSFtmwIcNusEYma44n1A//AEugXeRY2UCTnX4PpyRwE3h9QAkTIh6TY84uwhM1svY/fN+BzFkDLU8y0+JHatmS1TankZZad0+s4N677KhpmtEFQ23gU64ioZ26n5Zo7ei5z+gluBf+NzaF2Bq5i9CYXQtjbG/MC1eA/qauBAfLwus5wwOjIS2FGzqo+MB/aX1C1tm6uY656kg3obWCZ8idrjysop3qnPCNz7A/cgv+Jd0C40GL8vfjdewb3QZOAB+P0ryCnUKF+PcTd+Y1+VRy5Fyoyk/+IPYxOBN81DXCNlQtKD+FD1HWa2Zdr79XjDf1YInS61ntfwtVH3hxxPZSM0XE8Bx5ValpnVmdnmZvYf4EH8IXURSYfK1+81S8wTiq4OjMpxSGfgZ0nTQo86V4/0HWBqjn2ACzt+GV7PjXu5dEdRD9PGe9t6P3C8ZtdvvAoYnnqYMbOuZlaUkn1iDkrSx8BpuN7YJ/iEZYrZxljBx+D64c7pKhq6PKvgHv9g/IIuCtwA7R5uulc0Ep9YjMN7FUTSFElD8WGnh8zs6FKHniINSBqNS99ca2a7pr3/Gq50XZb5I0k34U/pt1VgucaxwH7FhrWb2Vxmti/ukM7Ahw4XlnSupF/KaGdSDAHuljQlx/4DcSX8icBJeGRiNkaSvWc1k9dwT9gJH1O8BJdMT2FgW7sG4RLARWY2KbUBSLoXOBe43cx+w48tagF5LS/UHYVHgxTTkNUD9yFt13gV1gfP2zJ/GLePVBgz64f3Wn/FF4K2uCygSRGGqx/FF0xfFd6bD28g1pH0YRnqaIs7vbclHV1qeRll/x1YSwXINZmLnB6Ih8S/hk9Hj1bNNmyFE+ZvP8bnzl4uQ4EVb1vLRS0/xZ6Nx/gXw7Rwfk7CE/yRwO3ROVUPSePwGJc3gbfMbINkLWo5hOHUAcAxobFHns79LODicgSqhKCCnYFtrPw5wi7G5Zo2aepAM1vazK7Fh5rmBwZI2kLS0y3JOQXWAv7EZzHKQUXb1rJS7jj9sm4wVDA5Q2G3qW2yYGhj5eITppPw4YCaXpjXkjdgA3y4+yygXdL2tJQNj5Z6PzQkhk/bfoDLJJWrjmWBHyjz2kF8qPJDYI4s+wzYEA/m/RYfyuqR9PWuwv28DjimrOVWqG0t91a7Q3wpmoHibqR4whDNDfj04S7yHlakRIKg6KP4IvmD8Yb9H8CyKlNCPTPbFleDWE3eUysLZvYw8B9JI8L/c9CQGLAtro93S7k+Ry0TIhvHA0ur3CK1zaBtrX0HBWC2Ci6jsxkeqDdX2t6p+JPVw8DZRIHYZkcYbj0Cl6M6SNJdCZvUIghh/Q/gDdxeeIDWK5LKNkRjZqfhSwk2UAkCtRllLgG8gK8R3RZ3sP/FHdNjahaNVnkwsz1xXcAtmzq2yApqum1tHg4qhQtL7sHsWR9HEjPqNnvMfyy3A/8BjlDuiKVInoQUHXcD0/EHgOeB5SV9Vaby63AppG/kkZrlKHNRPAptGVwR4UK5OkKrw8yeBS6WlCu8vFwV1WTb2rwcVKTFE576rwSWx1PLF6PWHEkjDJHdCPQA3gB6Sdq98bMKKr8L8DIutlxUAr0QwLE2Poy3Nj7suzswSNKrZTK1WREc9YvAQuXqnTY3ajmKL9IKkUdU7g5cAIw2s/2jTFJphMZtNzyVxEBgPTNbq4zl/4aHLZ9mZpnSbY1iZm3NbGc8Qu064Amgn6RhlEmuqRmzJz7X1iqdE8QeVKSGCXMRt+NKK/vKNf4iRRIc/fm48sSvwIpyqaFylb8J7mRW16zSN9mO7Qr8H67jlpLTfDDdnuCYXgSuknRDuexsDphZG/y6bNZahzch9qAiNYykj/D0Hd/ga6bWTNikZk0ILhiGC68shit6l7P8R/GovvvMbK5sx5hZXzMbgffmVgW2l7SupPsznaVcrukQXK6ptek4bgh825qdE8QeVKSZYGaD8MwqlwDnlvPJvzViZufgzmrNcs7xhF7aLXho8uBUxF3QhzsS2AiXIbpUeepfmtm/cJ25YeWys9Yxs9uB5yRdkbQtSRIdVKTZEAQzb8Ej0gaXfV1IK8PMnsJl19aR9GYZy+2ARwveig/PHoUnHbgE+JcKVG4pt1xTrRPEeMcCf5Hn0Gu1xCG+SLNB0pd4EsrncGX0ogQoIzPZEZfQeaLQ4IYmMFx1+5ywXYZn/72oUOcEM+WazqZMck3NgF2AR1q7c4LooCLNDEkzJJ2KJ0S82swuCGHUkQKR9COeSu0LYFQ+GniNYWa9QmLAz/GsyofiCiFvqvTEgJfjifC2KLGc5sBe+DBoqyc6qEizRNKzeCO4GPBCsWkaIlyFywddBIw0s+0LLcDMljezkfgwXCcaEgP+A8/zdr+ZdS7FyBBqfRie3mHOUsqqZcysP57s9smkbakFancOyjXasq1sviGqRkRShCGfg4CTgcMk3ZqwSc2OkI33erxXei9wgqRcmVtT59QBm+CBD0vhw3hXKyP3Urg/V+OLhLcLkXml2Ho/8JKkc0opp1Yxs4uAyZJOSNqWWmiDa89BecbV4XiCq1zaUI/g2lCvVd/ASC1iZivga6ZeAg6RNClhk5oVZnYXnhr9dnyx7EWSLs5yXHtgMK6d+Ae+fumOxhaThiHYp4EnJJ1Sop2L4It6yybXVCuE6/Ql8FdJWbKsV82Q2mmDqymd3uTWIAE/ownZ9xlJSL/HrbY3fHjpejxdwwpJ29OcNnx+5yegT9g+xnulqYfYnuH/b4GH8GAVK6D8+fG5rm3KYOuZwM1JX7MK3INtgWcTtaPG2uBKX/BxwIZZ3l8H+CjHhZnlQjwNWrCG8pPErfY3YFc8V9EhhTSirX0LDuiO8Ho+YAyeIvwa4Jfwd6kSyl8l3JdlSrQzlYJiraSvWZmv/wPAnlWqqx/eO2o78/20Nvhk0G41kCMqkSAJSc9JWmLmG96lHAF0KLCoDsCIIBkfiQAgn4daAxiCqxrMk7BJzYXzgdXNbCCekPA7POR5Fdwx7Sfpg2ILl6drOBIPmuheQjmTcGX2S4MkULPHzBbARXLvTsiAmmyDayWKbzieNKsY2ofzm8TM2hZZR6SZIelTPFX2J7hM0roJm9Qc+BP4Nz6/cBmeP2oBfOjvsnJEz0m6KdRxe4m/x9vw+ZC9S7WpRhgM3Kvk5k7zboOzrBfIuw0ulGo4qBXM7B0z+9XM7jCz9mY20My+BMCs5+uw2YpQ1xlXsdwJyAxhGYEPgi/ArAsEfoe6I2GrNmZfmtl3ZnZVSgcsVY+Z/d3MviWuLWhVSPpD0tHA/sAdZnZyS3niLidm1t3MjsXVC5YCPgL+Iemfkn7C1x61wXs+hT5hZ+MYfKK96Eg8+TjVIcDpZjZ3GWxKjBDpuBcutFtqWcea2admNtHM3jezbcL7bcKawR/N7DNg87STen4Gmw0IbfBGwI9pZY7Db9a/8MnJ9cP7LwN/BbpBXX/YZmWXI0vZsaeZfRbsGGtmu4X3FzWzZ4I/+NHM7mj0A1V4nHMcnnK6F9Ad+AAYikv+fymJ3+DY3lB/MegP0D2gdqDj0+ag2oBODPsfAs0F+jnsPxy0Ofz5MZwIdMbHcc8O9Q/EHf65wJzAXEmPM8ctmQ1/tnkSeAbPr5O4TUlvwKJ4T+lnPP/S8uH95YDvgXnSjm2Lz0c9B3QtQ93dgU+B3Uos5yo8oV/i17OEz7AGHpRS8nwp/ozfC+987ARMDt/9oXjwUO9w7Z8mNQcFw1aHGUeApoGeAXVKm4Ma68dpMGgSaAroS1D30B7PAD0K09p7XT2AjsBvwBLBpgUI8454z/f4YF97YO1GP0+FL/w4YPe0/88LX6iZDuoOeKwXqD5t4m2tDAfVHjQ9bX8P0EvhnA6g//n7N4Y61gTGhtcD8VDY9kl/CeOW/Ib3AobjcytbJW1PQtcglRhwFB6wcBaewDDzuEvxXlT6e3XBob0J9CiDLcsGG1YpoYx5KUPgRcL35BrguAqVPQbP1fUUacEMwN9SDupduKdNcD6pNnaXLA7q07T954B2zwiYWBG+wtdNdcTXTG2X2SnAE2dek+9DYjWG+L5Nez0Fj8BJ3znvgvivJkXvjALmwR/fUnQAJuHfyinAyl7oLmY2AXgU9+IpfpA0raRPEGkRyGWSzga2wSfYLw3relo8ITHgTvjIzA3Af/DEgMdJ+jrLKScD24X1ZcDM9BeH4mHmzwbx3qKR9F98+HVUEIQtpowfgdOAS5qjTl8YMt0eb7jLUd4QMxtjZhNCe7gs7sR74ZGPKT5PvRgPPefGvUqKvlnKTm+XP8cnKLulbe975OcCkibjvbehwDdm9pCZLRlOTQ3vvmpm75lZo3OIiQdJzAc/foW76BSNZjpLY158Bdl7wG0+bHohsFXYlUJZTo20YiS9CKyADz28HBIjtkjMrIuZHYkPpx2Ei64uIemK0JBkRa4IcRLuyC3tfUk6EZ+SeK5UiSlJo/C54XtK0FS8El9ntU0ptiTEtsArciHkkjCzvniur4Px4dluuPyU4TnV0n1Mn9SLBeGHX/DxuRTZ8qBkdiIG492k1DYNblVQ+JD0mKSN8N/Yh8EuJH0raV9JvfCHk3809h1K3EFtBk/XgS7HJ4vuxyet8qEO2Bc4DGb86l3YTrim2C9m9gw+8ThnmSZ2Iy0IeXbeHYF/AM+HSd1m9wSeCzPrY2YX4IEPqwE7yBMD3qf8c2n9E5/X3Slzh6QL8OHBZ8xsuRLNPRV/wLy0mJPlQrSHAiMsR6LEGmZvyhe81RF/IP8BwMz2wntQAHcCh5rZQiGo5NjUSf3hpZWh/mR8PuR5fCK/MXYPxzwGzACmwtTLYHIofz4zG2RmHYHf8QGv+mDTDmk971+CvTnlrxJ3UJ3hunvgj3/h3cSb8ZChfONZzwUWBe3pIcVD8U7VSfiPpw3QBfjezF4ws7PMbONShSsjLYPQG7gGWA84GrjZmnnmVjNb1cxuA97CH3pXkrSzikhKGBzZIcB5obHJ3H8tvq7pSTNbo1ibw9DhEGBdMxtaZBlPAW/g97FZYGZ/wXXu7i9HeZLexwOeX8LnWZcDXgi7r8X9ydv4HOKotFNH3grTX8GjJ07Fb0Zj9A5Gn4XPp/SBuY7xoJu6sB0JfI0H4AwADginrgq8YmaT8OUGh0n6LFc9taHFZzYKn8irA8+gNhTv/uRBPXAf0na5i7cOePDEgLCtjI8MPotHdT0fnqgjrZTwHbkQT7W9s3xRabMghM5viTcKffHEgP9UEbmXcpR/K/BpGNrLtn9zfF5rp+Aoiq1nMfwBfntJzxVxfj/cSa2oPLP1JomZnYIPxR2StC2ZbXCBNNkGF03SESyS2Bv2+xqmTAfdEKL2vs5PZkNBaqOgKCA8vHEA3tP6DzARf6q4CNiatPDauLWuDQ/T/R7PAluXtD1N2NoROBBfjPwqPhTXtgL1LIQv1l24kWMGhOs2qMS6NsafvHsXef4pBLmmWt5wRzAO7+Embo9gVWWRmqtUG5z3dUr8wvjN2m8u+K0jaDnQg4VdmJJ1oIA58CHC4XgU4G+4rPxleIRNz6SvUdyqt+E6ZS/iigo1d+/xiecz8bmGe/Gw8YpqDgLH4UoHjR2zCh61W+rapmF4T6jgdYt4kO84YEDS96kJOzfAQ8BrRivyOzhyMtQX4ZwqpsWX+EWZZctTSXcG1P8O0yt1YfCo9tXCD+VBPEjlfTxaaGc8lDL56xW3im1AO3yI/Stgg6TtCTb1x4fSfsYzzC5axbrb45GAGzVx3DJ4IO6BJdRlwK34lHTBDXjoBb9did5kGa/nzfj8S+K2BHvaAo+NgCfzaYPVEtTMi9pgFcE9gqmCKRkXZYpg6iR4cHWPAFm8SjevDbASngPnvjDc8TE+8TgY6JP4dYtbpe79BsFJnQm0S6B+w/PyPBHsGA50T+haDAoPao1eB2Dh4MyOLaGuDqEXdXSR1+zpUpxkha9j1/DQO2/StqTZdD6utNI2nzY47K/IsF76VhtBEtkw60H2bI4jkX4ws6OA9SVt3kgpFTLN6vDwzQFhWxdfRvBM2jZWNXtxI4Vgnln0Bvx7uKukcVWosz0ezXsEvgJjBHC7GkkMWAWbDB8Cf0RZkhlmHNsLd6oPAMOL+S2YWR88OeGekh4r8Nzl8PnlpeR6gjWDme2P90S3T9oWgKCTdxqw2izXqok2uCq2Ndc2NCzqewc4StJDCdtiuMjmujQ4rRnM6rA+iQ6r+RIeSo7A0zwcKKkiaRHMG4UD8bDcN/DIwqdq5btjZkvh0a/LSPq+iWPnwR3a68BBKiLdu5mtg6egWEsFZpk1s8vwQJeDCq23kpjZy8DpSbdbwZaV8Xu0vqR3k7Ynk2broADMbBM8NN2ybQAAIABJREFUkGFZSb8nbU+K4LAWpcFZDcDnNNId1ge10uhE8sc8b85t+HDIEZKmlqncJXEHuCPeIF8kX9dSc5jZCKCLpH3zOLYLvt7lK7wnNL2I+g7A1RHWkDSxgPNSAtV/k/R2ofVWAjNbGv/u9JEvME7Slvnw6M8jJd2TpC25aNYOCsDM/o2vYzovaVtyERxWP2Z1WJ3wJ9HUWqx3i3nCjFSf0OheCSyPr/15r8hyDF8kfCS+gPFKXKC10Z5J0phZV1y+ZkvlsV4sqDvcha+X2VEFamOG63QVrvW2bSG/k7DwdxdgYC08EJrZ+cAMScc2eXBl7ZgDHwJ9WtJJSdrSGC3BQS2KC2D2V3bRy5rEzHozq8OaF09lkOphjVH+kjSRKhMazT3wyeXjgWvzbQBD47AT7pja48N4N5erN1YNgsjnPni6hCYdhpm1wwVR58PXSuXdEwrnz4HLmT0p6ZQCzmuDD5WeLanx3EMVJlyDL3Bn+VHCtlyJC8huU8sPxs3eQQGY2dnAgpKaUuioWcxTPqcCLgYAC+IyJSmH9WYxwyORyhKG5m7Hozr3UyOKJEEDbT9cPugjPPDh0VpuIHIR5uReAS6VZ8nN55w2uPbhCsCmkn4usM758SGpwyTdW8B56wC34AETOQVyK42ZbYkHjPw1KRuCHfsBh+NDpmVRG6kULcVBdcKHHHaQ9FLS9pSDEDm2Dg09rL/gGlsph/VakhFdkQZCxN15uNzQrpnfQTNbBDgMj8p7ELhQ0piqG1pmgv7ePcCS+faIQs/zXDx0/m+SvimwzlXwBdTrydN15HvercD/khzOMrN7gYflGoZJ2ZDKBba2pI+TsiNfWoSDgpmhkkfgoZLN7om0KcKEb7rDWhx/mnwGn8d6udCx/Uh5MU95fQ1wMd4Ir4kP4w3A18xdLumr5CwsP2Z2A/BtIXMqwUkNx5W8Nyw0bN/MBuP5qlbLtxcWFLTfxtfujC2kvnIQHjg/xoMjEum1hGmFV4C9JT2ahA2F0pIclOFCk9dJ+lfS9lSaMFG9Ng0OaxlcTzDVw3opyeGM1koQLH0YH6L9GR/Gu0HSpATNqhhhaPpdYE1JnxR47kF42oe/SfqgwHNH4Grdm+UbDWdmxwMrS9q2kLrKgZkdgYvYJjINEQJVngPurOWAskyal4Pyp5BsC8duCIt3V8IbhyUbmwtoiYQUIn+lYR5rBfzapBzWC4VOTEfyJ0T2/R8+lDceD6seAOzVXJ5Wi8XMhuHad1sUce5gfHh0c0lvFnBeW/y3/o6kvFJshKHY94D9JT1ZqK3FEh6e3wEOkTS6WvVm1H8jroizW9Zgniba1upZm0G+khOJbq60O6oJ6Y1R8lDda/A1JMnbneCGS8Wsj6d3GY0nDXsVjzrbAuiWtI0tYcMzk16Ay1/dgQ87pfYNwKO2LgDmSNrWCl6DOfCgj82KPH9rXAl9nQLP645LKg0u4JxBuJOqmmwVLqL7GQmp4+PDzG8CHWbbX0DbmojtSVRa0JangGxKvPBLT1j2A7B04rbX0IaHM68LnIgvFJyIJ7W7GE+VHVOMFHY9V8EFTX/Gw8T75ThuHnyh6mvAIknbXcHrsSk+x1KUI8bzcP2AR/cVct6y4by8dOFwnb7HqaJQK3AFcFJC9+VveLr32fVCC2xbVWFh2Kz2V+ECvYfH/Rd+fsMFbOziZW6T/+VPsk9QQ1L2tbaFp96/MmuKkXdxlewdgPmStrHWNjyHzyB8yPQLPGdU1zzOMzy0/Adgl6Q/RwWvz4PAsBLOXxPPBLtDgedtE+5HXt9ZXJbsB6qQSiU8GP4E9E3gfiwSrue6s+0vsm2ttpOq6gVr4mKOwyN6Uhew6ARa9TB5Le/6b11A/aOBfZK+Dgle/2wpRj7AV/DvAvRK2sYEr00HXB/v49AT2pkihoiAFfGhsOuAjkl/rgpcp8WAHykhHQ2uzvE18H8FnncqHiSVVw8u9HqvrcI12Rl4IoF70Rn4L9kU3UPbehtoNVAHUI/w+gpvP3Uh6C+gzqAFQIeDplc4OWHWz1HtC9fIBc10UKPy6HoqOCTNyOiSfuGh158B7fOsv1U7qCzXo01oUA/Hk+L9hGdu/SetJMUInhjwjPC0fR8e5l9SrxyXuLoeX7e3QtKfsQLX7BxgZIllLBbagyMKOKcu3KOr8zy+Kz70tXKFr8dj+Nq4at6DOnyt07VZv68w6nyo7wm6C/RbaEPfBO0Kmgb6H+iX0J7+BFoPNKJhuO+eqn2WKlyscfj48inAnXg0yUR86G+VcMxNuE7XVGDSfHCqYOpLoDVBXUH9QU+nOaEBoONAf8VTxH8S3jshvNcJNIdPvJ6ZZssaeKbUCfiaiIHh/TNx9fFpeDDB5dX8QjWHLXzpl8NFO+/Cr+04YCS+nmWRUhvvWtnwSKbr8ZxjVwCLVaCO3YLjO7ilXLfwuTrjEYxrlFhOH7y3eWq+1wfoEtqVvIah8KjLFyt1/cNn+IkiMgOXWO/JuArNnLPth56/wNQOoLvzHJH6EbQB6ICG96YKelTls1ThYqU7qGnAZvjT+dn44tJZjgsXcdh4mNod9FDoHT0O6g76Ps1B9Qb9N3Q9/wjvLQz6CDQZpvSDz4EpQG98XcpPof46YKPwf49Q/2hiD6qQ+5pKMTIUV/f+GvgSl5TZD1iiOTW84fNsjE+gf42nOK9o4AiueP86/uTfYoJU8B72a5QYtQb0pCGQJ6+ywjX9jjwiAkM78Bqwe4WuwwnAlVW+9lvjyxzmz3oMDHsIprVpGLLLud0ShviA/2/vvMOsqq73/1kUYawgghWxxgJ2EUERjUYNsXexRDQGe+zdSIwlMWLhpwlq7Ap2UaPEr713ReyKChKxIQIiQxHW7493X+bMcGdun3vvzH6fZz9z57S9z9nn7LX3Ku/y5cDH1u2b5XkkksyntKF58YK7P+oiQb0N6ZvTYcM7oONA6kuTzVHgQwqHoejUdiiXBcBgRLGwONQcoCX8ZBRncTCiGXnU3Re4++NocBhYzBtsLXDhQ3cf4e4HognAdiiTaX/kpDLZzO4ys2PMrGeIx6gomFnHQHz6LnLBvwNY3d0v9hInunPlN+oHjAfeNrNtSllfM+IOYB76RPOGi9V9OxQ+ckOIfcp0znjgUOCukPCwqWMXACcAfw9xhEVD4CocjFbizQIz64nUenu5+zeNHLbhVOiwHBo3U+iHgp9qkG0EYBDynPoEzUKXrzu8BmkZSo7mFlDJhzYL6NjIS9dpItIjdUqUF5DESaF7mhNXSF4EOiBV1FZIvbevmU1LFcTEsGK+NxNRhyCwPnX3f7v7Ie6+KvLKehTNLR4CvjOz+8zsBDPbKHzEZYGZdTWz84AvgH2QrW0jd7/FmzG3mLvPdQWaDkGD6vmBVLVqkRj4LwqMJ4VcaxpylV4JuNPMOmRxzmPAFcADZrZ4hmNfRmEXZxfSzjToj0wWrxf5umkRqNBGowSuTdXZqQvyZEnSb6TsHl2QrSWJtdFC4JgG1ymsxdmhbANEGnji97TuSE8wLVF+RrwoKWSajv8gg3R7FBvVB6U06JQoS7j739LUH1EEuPuEMOAf7u5rApsi420vZI+cYmYPmdkpZrZ5NjPkQmFm65rZtWhi2AOplQe6+xMedCTlgLuPATZDsWpPBu64qoUrT9R/gILJWV2UXbuhT/4hM1sii9MuQ44o12excj8TONLM1i6spfUwGLipOd6p8N2MAh5291szHD6tL5q5P5hDHb8gt+jkdXI4PW9UkoD6Flgj/B53EMx+GLnApLwXnkFGjmwwD+Z9plifDREfWnvgIDM73MzaBtXOtomBIFl/RAng7pPc/Q53/6O7r4MmZncgB4tbkMB61MzOMLMtQ/6cgmHCdmb2MIph+gbRYf3B80w2WAq48pntiOxgb5jZbmVuUqE4BzjUlCa+IIRV7f5IZf+YmTU5gw+C4Q/Auiheraljv0ZmgMsLbScspB3bA7i9GNfLApegsfz0LI4d1wlqz0cronuRx9oCYCxaBIBcdVNZMz8IFWxfd41aRINUejSD0W4CdU4Stye2r4ZWLe3C/7ujYLtpy8NQh9pXwLcB7xyMdAPBJyacJK5vYNRrsK12VQVHvoBWqUORnJuD+mM2sjtsj2ZmfdGs+keU46bZDJuxLHwnugJ7A8ORl+UMNFifg9Sxi3olNX299sj2+BaaTf+RZvaoKuBZ9AvfznCyDJWoxIJUp49RJIcZNBBfhZwnMgbaIk+6r4GdMhzXIXz/OTFZNHKtI4DRzfR8D0KLm+ycbKBb8MLz28F7g9eE8XUL8GvB54AfBt4NxUj1AD8VvLYlevHlXXKIg0pT0vrqI5fhq1FupcOQAfNz5O57H9Kbb0SZOLNiWaS/lg0Tl2HIoeUnlFX1fGDbxoQN0BnNJv8Xjv9dNfZpuI97w2C8Trnbk+c9tEeT8N2LeE1D7ucfAd2zOL4/0pCsleG4gci1vSDexDApLtr9NlHPZmHs2iCnc0swtpbsHpuropxLAUwS3ki0M7IBfofSwye3dw8zkevCC/oDUtGeHF6CtmV/HrGAgisHolxLr6CYtedRMO1vkMrwKsSPdxtKb1D2dhd4z4YcKL4Pk6qqcd1P3MNvwiy/qCvB8H1OIIs4NeSI9j6wVIbjHqEAF2rkRPwtJSajRU51E4G9cz6/BGNrye6zuSrKq5SALwo4GpmzGv3QkWff/ig99fvIIPhImJX3KfXLF0t2BTnB7AjcjByTHAWJXo3IS5cudxuLeK+9wrt4e6ZBthILco45uwTX/UPo8w2zOPZa5OnW6Go6CJi86ZqQueayEj/LxcLE7IK8rxO5+IpUisy4i4KE3yEHQkrS20YeQ66pOdtGYim8oDCOfYCXw+z8+NBPDVOMvI48unalylOMIE7AaxHlVLPNYovU9tWRZmKVElx7v7BqaZK9IgzsLwBDMxz3d5RkMtd2tA3CsmeJn+W/kIanILX1fDhqgYJuWy+beVEKbO5wnzeds+S+bJeeKE/PRNLlR8nu/JRt5HLgTRa1jVStUbvSC6LS+ROKX3oR2ItGVLDUTzHyOIumGFmu3PeT5zPYF6mqTy50kGrmdv8VGFmia/82PJMdMhy3PHLG2ivDO/YV0CePNrxW4mc4BNn0CtIOICe1T7aAGcUcW4tdqi2jblfSZ328hRyzPprZncDH7n5+4c2yZVAw8IBQehHTrxcVZtYdrZKOAJ4ELnf3V3K8xmLIppjqp35osHqO0Ffu/m0x210qhNTyo5DX6WEu1oWKRohf+hBldX2+BNffBjmVHOnujYb5mNnmwBhgO3d/r5FjDkU8iVu6Ao+zqf8e4El3H5Fz47O7/tbImWtrd/80z2u0Qfd1CVqRf+Tu6xVzbC0qyiEVK6Egx4gpNJJorsBrp2wjFyGVwkwUrH0JsDNVaEMoYz9thmKlpiJ2gKL1F1IT9kaB3A+jwf4jpEYbBKxc7vvP0P72wMVotr99uduTZZv3R6vYkjgehfflazJk2UXhB+OBZRvZ3wapjwdnWW8XNKiXRI0cxqvJwM4FXucxFGrjofxfud+Jpkp1raCKDDM7F6U82KfE9SyOqJZSM/fN0TI9NXN/3kXpEsHCWd4uSIW1BrL9Xe/u00tcb1s0g0z10zZIaKVWws+6+8RStiEfmNkOKND5ZmRfmVfeFjWOwOrwDFL1XVuiOtZHA/El7v7PJo4bhhj6B7r7L2n290YUXetmevfM7Higr7sPKqjx6a9dg5wi7nb3Swu81m7oPemEPERHuvtBBTeyRGjtAqoGCYo/uPuTzVhvR5QcMDUQ9kGzudRA+JyXmKi0EhEE+e+Bk5AjyjDg3nINuEFQ9qS+wKolIbCAz70CPiIz64ZS2SyNsvZWnCBNwcw2RgJkPXefWqI6Vkd2xxuAv6Xro0AR9CjwrrunZZsws38D01x8iU3V9zbKJvxEwY2vf11DIRNtkGq04HfNzB5FgcnbIFKCJpk2yolWLaAAzGxPZLzdpIwD4WJoVbUN9W0jSYFVFbaRfGBmKwDHIgPwy0gwPV8JA38SYbBYhzqBNQCpSZ6lbjX8cbnaHQTqySgc4mh3v68c7cgGZvZPYL67H1/COlZEQuoR4MxGhNSywGvAX9z9tjT7uyH3/v7u/lEj9WyMvOpW9yztVdnCzE5BMZpbu/usIlxvIFKVb4C8Xud5Bdsvo4DSoPN/iGhxeLnbAwtndptQNwhujfjjkqqmyeVrYXFgZhug1dKeyOB/pbt/Ut5WZY/w7qxJfYHVgYTTBfBBsQetLNq1BXqej6OstLXNWX82MLMuyGFie3d/t8T1PIrsXse6Uv00PKYXShPzWxfJbcP9JyHb8c6NCLmrgOnuXjAxboPr7ohUt33c/csiXG8xRO92krs/mun4SkCrF1CwUGf9LLC+l9NjpRFUq20kHcKgviOa6W+IgmpHtBSVZvCuSwqspZH9INVX45pDYJnZ0sAINFM+wCuIFDcFMzsWxRduX8pVZyBvfQg5T/w+naYkaFKuArbwBrmUAmnxO2gV9lCDfR0QpVYfd/+8iG1eC4VR7Ovuz2U6PstrnoqyiO9SjOs1B6KACjCzKxG325BytyUTqsk2kkL4kAchweQohmyUN2PupXIgsOUnBVZX5NmZ6qex6Qz0RarbED3SpYhw9/oKeyfaoXCMC9z93hLXVQPchWw5+6ZbVZrZUERs/Wt3n9tg329QgGwvd5+d2L4PWpltV8S2LoWovK5pyskjx2uuiFZPfT1PF/VyoHUJKOmT0/n632zKAPoR8uh5q3yNzB1hIFqX+gPhAuoLrE/KMTiZ2XKIB+1YNAsdBpQ191I5EQaK/tT1U3c0U06pBd8oti3UzNZFg/PHwB8ryWPUzLZFaqz1imFjyVBX+1DXisBu7v5Tg/1tUJzRd+kmqmb2AArEvSSx7RHgLs+chynbNqba8D0wJKfvpOnx7R/AN+5+ZhNXqDyU0oe9YorIEe/PEC19/x9FOvoCVUjImSzIfXQtFNR6K2LN+AYNUsegQOKSMhAgZ4J/IVXkDWjmWfZnU2kFraj2QuqlsYjt4nHgXCTIikKjhVg1rkYMHE1SApXhGdwFnN9MdbUN7+VrpElRgVgk3icNrQ8KeZhCoGtCWX6nAksUsX3nowlL9ozqGca3+TDnYah9HwaUu69zfh7lbkDJSw5cfgvg59PEjjyo7O0udkeL2uT3wI2Iu+57ROD5J2DjYgisIBgHENK7AxcAy5f73qupoBQbu1GXYmQmMuAPBbajwHxWKJHetyiLbEXQJKGcTVOAHs1UnwF/A94DVkqzf63wjPqn2XchcEf4fSZSmxarXXsAk4AVsj4vy/FtPizwMvHpFfRMyt2AkpY8GHt/gdo/ada/ZBYv1ExgjbLfZz4dD6tQP8XI1CBYTkEu7+1yuFZ7ZF96E6lJh5Anz2EsizzbTClGcp69I7Xic2Gllv1gWNr7/DNwTzPXeWaYrK2eZt9OyKli1QbblwhCZKvw3fQrUlt6hklj76zPqxJG8oKeS7kbULJSQM6T2fDL3nBjgxfoGRTQW/57K8WLsGiKkenIPfcMxIKxSIoRpOc+PXywTyH2h4qYlbfUQn0areeDwHo5rAgGkiWJKKJ5+ksYhAuizynSfdUg9eOvm7neY8L7u36afaeGSdfiDbYfEITTRxTBHIDIp8cDh2Z9XhjfRqFMuIuDdw2/rwFfAH4peE/wJcFXC/8nhFRVsOGXvQEZOi7rWXyaDsw7a+QCmD9afFVrJdrSogVUmmff0DaSTL++L/D/wqrrdmDTcre3tZYwsG+HVIBPB4H1BlIR7gZ0znD+gDBA/4MCM8kW4V72Qp5m+X/3+dV7MLLRbt5gu4X3+46kIArbv6YImWXDROExRH6c/blw/z9gQTfwe8BnBKH0FvggTbL97+Bvgs8D/wh8VfBRdekzmi0rbkHPp9wNSNNhE8KsfRwSEhsE4TAtzOx3Sxx7c5jxjwkf5ovACsvAtZ3A1wkdlhI8l4CvEWYU64Hfn9h3E/hW4KeAdwLvAb6sPKoIs9X5wOxQz9Vhu6eEWBgohiGHhOnI2aIge0EllTDLOy3MHOcBcxEB7lCKYBuJpWj91AE5V5xLXYqRsWGisRdpUowAyyGy3NeANcvYdgOeAI4rQ927I7vpNg2216BV1KmJbUuECdv3mSYAWdT7j9BP2Qtl6PYj1C4Ofm8OE+/jwY+r+7/WoWu539eMz6fcDUjTYRPCB9UdedSMR4kBF0PJ6H4C1gnH3oyMq5shL6WngC9OhDvnwaxzwLdNdNDd4F/JYOh3hmXx5ISAagd+nexQ/v9g7hISSjuHup6hwQqqgYC6JhyzMvIU6kcLSGQY7mWfIIw+B05Aaqai20ZiKUn/LQb0RfaWMWHy9F54X/cj2KCCcDghDLoHlrG9PYOgaPZcXcD2oe6BDbavipjEdwr/Hwr8B3kDDi+gvoOQDWwRb8ImC5z2CMxuG1ZHWWqFfGPwf9Vtm+UFpLZvtj4pdwPSdNoE4PDwuz9aerdJ7B9FyIgZBNT1iX3HAx863Obg48CXaaLTNgIfnRBQayb2/cxCOvpPw0feqIBCwX+1wEblfn5F7IelwoD1eRBOe9NEigTSpxhJ2UZaVPr1ai5kTjGyE1ol31iuSQZa7Y0oU91bIg++/Rts7x+2rxXGgr3RyvM78gihQJPq74ENcm4n3HYb+PINxrO+YbzrCP5sg31/Bt8wqP4S228t9/uYqbSjMjEp/F0JmOT1qWEmolVKCkkS1drwfyfQ2nxmYuetiL5gQvh/Jlp+pbBC4vfidT+/QoKvKSyHVnCfZTiu4hGYD1KJAZ9GDMovZzrP3WciG9X/heukUoxsgxwpepvZh9QRqz7v7j+W5CYiGoWLteL1UC4LNFobIFvUPqi/ZqAB+VMzO8LdxzRzM4cCH5rZte7+dnNW7O6vBNaIMWa2lLv/O2x/3szOR6vQzoi7c66ZXQBcZWY7eJA8mWBmy6MQjyGeHw9hpy5o7PoFFg7iL4W/q6Ao/RSuRmPf80j/m7xOHnU3K9qUuwGNINXRk4HuIbo6hVWR0GgKi0TKTwSORJ31QzigV6KiJnARcBZypW4MU5B9as3Ml6tMmNmmZnY7sv11QO6u+2YjnNLB3We5+1PuPtRFA9MF0RxNQwLwSzMba2ZXmdlegXEiopnh7vPdfay7X+XuewHdkHPFlYhR/xEzm2pmt5rZEWa2VmAuKWWbfgTOA4aXuq5G6h+HBPY5gU08tX0EEt7TkWwA8R12Q/a9jAiErfcCt7j7/Xk2cVpf9JE2mjY44EakwngSCa6G18mz/mZDpQqoFF4FZgGnm1n7QIuyK3BnhvPGodXUQvyMlOxdw/83IUV8E0id/wXq5xVRJPkiCCu8G4HLzWwlM2trZn0D/1zFwszamNkuZvY0etffQXFdJ7r7F8Wsy93nuPvz7n6hu/8GCayjkTfUkcBnZvaemV1jZvuFWWZEM8PdF7j7e+5+jbtviRhBJiP6nN+iFfD/zGykmQ0xs3VLJERuRM4IB5Tg2hnh7uPRavJIM7vAhLZoCJmBGB9SK9ITgGGB7y8ThiPv16EFNG9cJ6g9H/nI34sM8wuQ8f7ncNAdyHj/OGkHrlo0TlY2yq1jbFiQBm6HxP890UcxHSUX3DOx72bgwsT/fwCecejmUPspeNuEzvVs8M7gXcBPAt8G/PoGXnwJ/WwtdTampZG++Euksx8e6mvoxXclWt1NR2qsivRsQxrMIcj28CayPSwS59TMbarq9OstuSAb7LDw/vcnPY3W3YhvcQOKFAuHgmEnkUXQfAnvvSsitB2ObKxvAcuHZ7FX4ri7gT9nuNaQMIYVZo8N45uD3w7eG7wGfDkUB3Ut+BwU+9QOfIlEGRK9+CqkFBAH5WniBBBN0KvF+vjK0tkys12ADLsPIjVGRfIOIu/BTYATgQeQZvYzNLP+PbBaudvY2gry2vwGrR7aJrb3QJ5tNyCv2ymhz04Mfdioc00Wdd4OXFTm++6EHH8+B04I2zYn4eQQnsEPNELXhHK6fQusXZR2FXl8q9TSctnMzXojb5vFMxyZDrOAASSSlwU72EvIu+jmYjSxuRASsp2E9OR3Ald4FSUGhIXPf33qM7bPpj5j+2feYl/oyoCZrYSERhvgYHf/X5pjVqEuO/QAtOJIphh527NMMWJmKyPVcx93L5sTUrjvicB/gX3cfY6ZHYzYOHq7+9TgRNHT3fdrcG53NLk93N3/W4z2TDXbsjM8aUUa3yoW5ZaQJS1F5qpCKqjJVIHLNDK57Yg+qK9R4Gazx5aU+P7WRWqTkUi1+lX4PSTsq8jVYbUXtLo9B62mds3i+BVQzNU1yPQ7HXnDnYlitJpUL4fjRpf5no9G5p57kafqEmH7ZYRAW6TmnwBslzgvFeh7ehHa0BVpD54A5l8IrxZzfKvEUvYGlLzkwGaeTechNcY/yn5fjbevAzAYUca8G35XfcBwFvfdWIqRottGYln4zPuFAfmqXN4xFJaRdYqR8E6PJwTKluleX0Np39shNfOLSPXXLgisYeG4vZHzQTvqqJJGFjpZQjbAuSg6xhGJwHrFHt8qrZS9Ac1SYHOH+7zpfFD3eRYEikhd8T2wbtnvq367uoRZ7WTE7bVja19BsGiKkaLZRmJZ+Iw7h1XF2wSGlzyvsWtYjbweBuFnkPrs10iNtStymml2vkAUkfK/1PuC1JtXBOHajTqy10OCUHoqTIpOQU4VBTP7o5jCn4NwcuCDhfuLOL5VWmm5Nqh0MEstkRtmnLwF9++zv4ydjFxu30T66LVK0Nps2/IrNOAeiAbfy909gwd960QxbSMRdQhu5kOAvyIvzFu9gIHFzJZGHnypftoACYMVkHrrVFdgeLPAzIYBc9z97MQ2QylCBiFqr6VRYPtAZBt9Phw6C1EnjS2wDWuid7UL8ig/1d2vbnCaxjJ/AAAZ5ElEQVRQUca3SkLrElBFQHgxB6HU0fMJumd3n9vMbdgGBb72Ra7Y17j7N83VhpYAM1uB+gKrO3KESQmsoqdfb8kwsw1Qdty3gKO9QUr1Aq67BFIn7olCSeYh9XWqn15w9xnFqCtN3e3R6mlrd/80zf4T0QRxRxQScxVi+r8UCZK5iNtwdAFt6IVsyX9FnoC3oRxWU5o8sQUgCqgcYWaDUAxcCrORy/O3jZxSzLrbIzqaUxBX3hVotjqr1HW3BphmoP2pE1hrIiLc1ED4mrvPKV8LKx+B4upKxHB/gLu/WeTrX4ZsWLdQ10+9kfov1U9Fo9Eysz2AU9y9fxPHHI5Ikn+LOAA2SeyuBY5195vyrH8LlEj0JHcfFba1by0TpyigcoSZtUMC4nzEvzcPkcR+VMI6l0FsCyny1mHAI16fozCiyDCzztQXWOsiG0mKT/Bld69t/AqtF2a2H2IW+xtwZbHe1aD++wjYw91fC9s6ICGV6qe+6DtJCaznPE8Vl5k9CDzo7jdmOG5fJDQbskk4Usddnkfd26EV6eHu/p9cz28JiAIqT5jZ2ojwsRfSMY/BrBvpdcA356MDNrPVgD+Fa45B9qWizkgjskeYKGxFnVowZRtJDYQvNadtpNJhZquj7AM/AIflKyTSXPcw4CiUbn0RwRc0DZtRJ7C2Qmq6hTFz2ajDgwr4Q5T2vUl1pZnthDJQJ+njHDlN3O/ue+cyPpjZrshjeD93fyZTW1ssyu2lUc0FvYxDH4OdXZHdTXnR3O8K6Gt4jRrkGp3M2Lklco/+Aemyu5f7XmNJ2/9LIgP5hdSlX38F5cjKOv16Sy6IZPkSJCCKks49fHevkmWKdGQn3hxpPh5CNFofA9ehnEyrNHLeqcBNWdbxGbI5zUeCaUH4O/x0ODmX8QHZuL8Btih3/5W7xBVUoTA7CqncOtI0+e4CZK86BbEip5wd7kPG322Qy+rJKM3IlcCNXiRDc0TpEchCt6QZbCPVhpDC4hZkoznfC/SWNLM+yGt1Xc/RQaJBipEB6NubTp3q9lkU3/UecvZ4LsP1eiK3+DORo8Qg5CF65dcwaQU5N2Q1PtwCow9Tm3b26I0bBVRBqBNOudCNzCIIKTM7BxEO1yDh9U643uhCP+CI8iPYRragbiDcEs20k7aRFu+JlUJgqL8VrTwHufvEAq93E/C9u59e4HVSNFpJj07Qd30y6qvxnmawDGrMp4Hz3P22xPZeP8F2S8oGl/X4MAt8Avx5ffcL872floQooBIws3WQUXJN4Bx3H97EwQVx/R0Nfx+hnDepfGNzEY/X+NAWR8SS4/O4fkQFIuQCStlGtqGOrTs1a8/KNlLNCMLgFOA0tDq5r4BrrYBWOf28iNySQbMxEq16xFun1U+S9/EjRPn0KQpP2MWTPHtF5gJtrYgCKgEzuwGY4e4nZXHw/cDu5JdTa8HDMHc3pTFog5KftQXOdfeLQ1uigGrhCB6hG1M3a++PmOaTxvxFyFhbAoL79ChEE3Sy5+kNGRIK/trdf1fEttUgXseN3H1SEFhrUJ+ouAY5OfRAVEy1wJkLJ7VhfLgT2lyBpOgSwOrIS+JoJL0uQEFjnanL9I3UfaNx37tY91StiAIqATN7ArjTQ5rnJg7shrjeOuZTT0jTPBtY1eQIUYPe3x89xDdEAdX6kME2khJYE8rWwCIjeEWOQJ6wB7j7+3lcYzHkCXeKuz9SpHYNQg4YOzdxzAmIWmw56iapDjzpcryYOAw6XooYcndCes2xiM/pRqTP/xhJtoupJ6AgjA9UKQNE0VBuL41KKYg/az56MWYi9+63UfbMScDQ1LHP6X3yG8FXAe8E/i/w18A3AF8G/NiEp85N4P3ATwRfFvyc4L1zrNSJHyKvosdI5JIhkQwxltZZ0MDXC/G63Y1YBCYiO84RiBy3qvkWkRv24Yjf8sh87geRuH5KkUiREZ3S/k3s3yy0dzCab05DKvpvgacdTvsRZi0Ofm8WDOOPg/dYdPssV/xU2fuorO9HuRtQSQWtuv8Qfm9LYMBGcQvfouBAxir+yYeA14I/Bt4BfHfwb8H/B94V/JmEgGoLPhx8Hvgs8NHgy0v4rYfsUOeiOJpUW6KAiqVeoekUI0eFd6kqBVZo+ztBEHfK4/yHgDOK0I4eiFS4YyP7F2bTRRq7c4DtgaUWHge3jQnf/Lz8BZS7WGLK3jflLPnYT1oF3P0Zd3/X3Re4+zikLx8A0FbEkJyHdHw7Iv3cgchPfGVkTHg7cb2VgOOpSxozAjgBPnf3D10eexcDG5tZj+a4v4jqgwsfufu17j4IWAW9k08irrr/At+Y2T1mdpyZbRCcEioe7v4h0AfZ4N42sy1zvMTJwGkhsWAh+D1S889uuCOoE+9DsVH3u/sX7n6Ruz/p9cNBOk1Bur92iY39UHRuDfKKyQKd8ruFloOqeHnLATPrY2ZPm9n3ZjYdzVCXA5ivlQ/LJ46vSfN/klKge4PrTwQugJ5mNs3MpgFT0Qx55eLeSURLRRBY4939Bnc/1N17ILf2hxEf3APAd2b2gJmdaGabBDtXRcLdZ7v7cUjYPGhmZ2YrYF222uuQW3deCHUNRrFa6TAcra7+kuFS07qEA5OxIi8hXWCKjjwLTMvusJaLKKAax0ikNuju7iljrgHMkItpTrAG/68MC46Du929U6LUuPtLBbc8otXC3Se6+63ufoQrDcxGwD1IhTYKmGJmD5vZqWbWO3gSVhTc/QEU5Pw74LHgTp4NLgZ+bWb98qx6AEqe+FbDHWY2BDmtHOqZeQXHbQm1HYAH82wI8p0Yl//pLQNRQDWOpYCp7j47uMQOSu24Qcv8gnAU/HIVbBqi0DGzZQLhZERE0eDuX7n7SHcf4u7rIkF1G7Kf3AT8YGZjwmqlX1BjlR3u/iViRH8JeCtw3WU6ZyZwBjA8z5XiYMTeUs+12cz6I4/w3T071opbOoOdDxyDsjn+hFZNY1HWQaijlpmHDM6zkadFqlrEvNGqEQVU4zgGuMDMfkKJye5O7bhFruF41iv1RbBgb/jPL5rx3WlmM1CoxG8LbHNERJNw92/c/W53P9bde6Gg9OuBFYF/IoH1hJmdZ2bbBDaMcrX1F3c/H00O/21ml2YhQEcCc5CwyQgz62RmvzazTsBu1E+lg5l1R962h3qafFCNNPw7YMzpsOByRKa5fChDEFFjP2SHqkGkjV+G3zvqCguAR2ntLubEOKj8ESPFI1ogQoqRramLxVoPeIO6WKxXvAz5x8xsObTiWx7FTH3exLGbImbx3oRFjDeSBcDMdkdeubORR+QuHlgpQsDuC8hp4h85NjiOD0VAXEHlC/fXEWVLrh9riouv1b98EZUHd//R3R9291PdvTdyQP07Yku4CPjezF4ws4vMbEczW7KZ2jWFuhXOq2Z2YBOHj0V24k8RpVKjyQZR+MhPSJCsAXxsZv8xs6XQyvJjFFuba4Pj+FAExBVUoSiAzTwiotoQBFJf6lZYmyD1dIoJ/AV3n17iNmwK3IlSnJzg7j8n9rVBK771qGN6+au7/7mRa62KhFkNEg7j0MprOkp50bugFWMcHwpCXEEVCr1MA4DR6AVryClWG7aPRsv2+PJFVC3cfaa7P+7u57rSoHcFzkLv+anAV2b2ppldbma7m9myJWjDW4jNoR3whpltBAuZxQci4eWhQP0IkIb4BgmPX4ArkGt3W2BZFLSbr0dgqrFxfCgAcQVVTJh1JX3GzFuiwTOiNaCU6dcbqe9gJFguRJRQq6IA5mWRc8PGwKvu3mjgr5nNBoa5+zkhJnEZtKKZB4xy96wcLrJobBwfckQUUC0JRU45HxFRKIqVfj1DHWsjd/TOaCU0wt1PDOq+y1As476NfR+fwG2/cv/GzNZCdqt5wCNINbhITFRE8yEKqJYAeQydhdzUHenTU6hFMRVjgEuC8TYioiwoRYoRM9sVxSa2D5vmAKu7+9fhgKy+j/vg5n3gj8AxIQ4rosyIAqraEY2wEVWMRlKMzKB+csAJDYNnG1zjCCSAVkUZCToCT7n79vH7qG5EAVUG5JS5t+kLZUw5/yXKZT0dWX5JpJzPq86IiBIikX49mRywHRI8Z9Mg/bqZbQ/cgFZHnyBX8f6AuVZUTX4faRC/jwpCFFBlQE6Zexu/SNpAwNWAfwM7NH12DASMqAqEbLZnopxRr7Fo+vWtEdPETGBPd38ynBgDZVsAopt5edADSJs9NAcOsbPIM6NvOO+sPM+NiGg2hJXS18C37n4QSgzQH3gceQjuj2xISwFjzOz8cGr8PloAooBqZpjZU4gE82ozm2lmI83sX2b2qJn9DGxnZh3M7DIz+9LMvjWzEYF2BYBeZgdtBHt0gjb9qKM8PgSp9HZF6aUvRWmkjTra/22Bc6FNX9ijjep/2My6mNkdZjbDzF43s9US7V3XzB43s6lm9rGZ7VfK5xPRehEIaz8zs5/M7AMz27P+brsaed49gjLWnED9dOvtgaGrmZ2IVH5tbkQRu51R2vWJyQuiFAVrI5e+Y1kYONUGGNjd7CQz+zDRnk1DQ1Yys/tCKp4vTOnfU43cwszeCN/St2Z2eTGfUatDMbIexpJboX7m3puRiWgr9GF0RHEdD6FYjqVQfp9LwvGbLA4zX4LZv4DfHLJxzg5ZOHuEDJ2prJxfhIDFVGbPAeBrgn8CtROUxfcDpLvfAen6b0UJ2UB5GCch4s12iDVgCrB+uZ9hLC2vAPsiaqU2aGX0MyKxPQzNsU5CQmj/8M1shFgs5iJKozWBFUfDZQ6zRod3/YPw/v8VvG/i2wD8d+A/gk8EXw58TNh3J8xZUnX0RrJsLaT5aAO8iQikF0M2r8+BncI9vAwcEn4vCWxZ7udazSWuoCoDD7r7i648M3OQq+tJ7j7VlanzYuCAcOwf94PP+0KHtiioowPwSg6VDQbWho494FfI/fwzd3/Cldn3HiSIAHZBHlQ3uZil30buvDEtSETR4e73uPtkVxbru1BM0hZh93fAle4+L+z7GMUz9UE2o8+Bye7+9e5ijqgZgfR066HZ1dmIpC+5ijoTrZ5WRWqNsWH7jbDYEPjY3V93Yby7T0QCq6u7X+Duc12ktddT933OA9Yys+VcrBu5fJoRDRAFVGVgUuJ3V2TYfTORbfe/YTtAj1Gwfif0YXUKJ0/OobIE70snFAfybWJ3LZr5gWaMfVLtCG05CMg2gVxERNYws0PNbGziXetFyGINfOVhWRIwEVjJxcO3P8p4/bWZPfKamCSYCPyJuu9kWbRs+ipxkeSLvDh1WbAnAT3rqJKS6AGs1OCbOJu6z+oINPH7KKjLd8n9SUSkUHHZNFspkh/CFCQkerr7V2mOnXQYvDtCwY6LoGHm3gzIlFJ6Egqc/E1ul42IyA1m1gOtRLYHXnb3+WY2lrpXemUzs4SQWhWpwXH3x1Dm3RrgwsFw8PvIm+IcNKPKFd2B99N/TpOAL9x97XTnuXJGHRjc5fcC7jWzLp4gtI3IHnEFVWEIar7rgStM1CyY2cqJjKLXj4LVX4Y5jpT0j6B8AaBpXKOJcuojm5TS/wF+ZWaHmFn7UHqb2Xo53VRERGYsgSZq3wOY2WC0gkqhG3BCeAf3RZq7R81s+UBKuwRSj8+s1UKo9ijgEurcZacj/XU2OBzmXqd3fzMT1gpC9DXgJzM7w8xqzKytmfUyubVjZgebWdfwHacmgPkmNm31iAKqMnEGMB54xZRt9wlgHQB3f6MHDDkeFuuMLLc3J048C7FmdiJjEpuMKaWD/WtHpF+fjJifU7mBIiKKBnf/AAXVvoxUzhsALyYOeRU53E1Bean2cfcf0Bh2Mno/pwIDesk0a3uiD+kAYGkk7cZk2Z79YcHS+pRGovnfaGBZd5+PbLMbA1+E9vwbEcwC7Ay8b2YzgatQcsWGDOYRWSIG6lYrzO4Hdie/ScYCYDTuexe3UWkQCWxbHyqhz6vl+4hoElFAVSsqPVI+Eti2PlRSn1f69xGRFaKKr1pRySmlxRH4DJrBdqT+QEX4v2PY/0w4PqKaUWl9XsnfR0TWiAKqmiFCy9RHmMkQu4DmIMKsT2Cb6f1qE44bFoVUZcHMJphZBkrHhQdn1ec3A1s30edmtq2Z5ZRqo0lU4vcRkROigKp2VFJKaalVcmWPhroBa/NFL2mrmZmHPEIRlYYC+tzgX5eb7VGCVtWhkr6PiJwRP/qWAKkj9q6AlNLFIOiMhunqQiF9zrYKsB1dtNakQ+V8HxG5otxcS7FUVwE2Bd5Grrf3oLxWFzp0GwFz1wTvDL4r+FcJ3rMXwTcHXzr8fTGxL8EfWOtizBgK3B7q+xIZ3GeG0rck9wbdHE5zuM3h4fD3NBetTdmfe3O3E/EMn4oG8OmhnzuGfbsAYw2mbwnz30n05SXga4AvCb4e+P2JfTeBbxV+9w9ceIuDm/p1f8Rl/D+klvsOsZgPLvszj6VspewNiKV6CiLHTDHItEeR8nOBC6+EEV3A3wzEtceFQcjBfwDvBH5rIO0cGf6fsqiAmuUaFJMCarUgoNqV5L6gt8P9QTjO8sSAGv6vDft7l/X5N3M7g4B6DZG3Lgt8iFY7mwTh0Wc2nH4DzEmSFd8dJibzRbjqi4NPTiOgUmStH6vdp4Y6t0WksBeE92sgsgt1Lve7H0t5SrRBReSCLZFaeLiLtPN+NIjxGOxwOFpedUAR/C+jUe4RFGF5SDj5QGBdRNHeADVI/dI8qDTPs8ZQvnYOd5G3TkXdtTEiMr7W3V/tABscDoslyYob0pGvTXhBGkEbtTvZ5/OAC8L79ShaXa1TpPuJqDJEARWRC1ZiUdLOSQBTYekeiY1LAl0QMedkxLCZRA/qk3Ym0KlIbW0aWXieTWBhLq3yeRvm6BU5FBYfBFcXqZ3fJH7PQt3aAzjFzKYtCQc0JCu+FUmxFEHre4hqIQOSff6Di1W/Yb0RrRBRQEXkgq8JpJ2Jbd0BloUZyTQGPwM/ACsjqZbcBzIsrRx+L0G9YJVp1CeZLiiSPLhL14akc9PM7KWtzS6ZX2RvwwLbeJaZjWmw7dPlzF4i0c61gTuzuF4baFuKdgZMAi5y904z4c5pqO8ORH18JHA16vsUHXkWHZiJtDiilSIKqIhc8DIwHzjOzNqZ2e6EfD2/gSdvQvl05qD8A32QAWkgyog4EhkY7kJZElN5CDZGA+9cqL1dfGr7JOr8HsWorFFAu3d196XQ7P9v4+GEIxZVk2WLUqQDfw7oZ2ZtAcxsRaD9fNj0l+Ah9zUiZ9ymvO0EERkfZWZ95sO4mVCbIiv+Ga04U3lhbkIrqMawPDBer0sm0uKIVooooCKyhrvPRY4RR6BZ78GI8XzOSXDeX2De3igF6mfUzfa7hIOGhd+Xhv9TiX7+Go5fFmqOhJ5IlqXqnIXIQV8MK6AtC2j/dIdXRkPbW8HeQ/axTRCZaHfkndEYpgKDoc1KsFcbtaVg9+iQlmEJ5BSQSqHSf0l4eQNo/074Rp9H6WJXQh4q3UObNwv70qANMHALs13M7KXw7CaZ2WGFtNflsn0kcHV7OHdtqLk57Fsfud/1RcLnXZQmujEMBQZDhzZwnpntV0i7Ilooyu2lEUt1F8QyLVdgeZHNb+Bhlm2Z73BfCdo3Adhh4Ta5ZM/qDv5P8KfBxwWvs3fAu4E/ENr0RfA0mxf+Hwi+n7wSZ82E04EBRWjfuqie2ciE0xa4+ji4+wyYd3mo+1jwweH3bcEDch74ZeDLg9eGfeeDH1TX/trFdN0DkQDsAmxc1GdcgX0eS8spMVA3IieY2QCUbnsKygW3Icr4C3Le24n8CDpnh/NLjQ2BmpXQimjbBjsOBJ4FGtIbfI1YTn8AOkPNu7Ab8I2ZHVpge1ZE994RPc/fAXMOgdemQrtrgZPQKunkcMLBiZNPQTkhPgY2anDhUdBxM5j0kvuosOmHUIqJaujziCpFFFARuWId4G6klvoc5eX5GgD31zE7hdwdEJqToLMTyINwWbT8OxPZSuYig8i+aU6aFI7vHP6fJXtWdlx1TWMZtGoCqeU6A3O3gLYzEO3B1NC+lP3pMuAG5DlnwAzSe8pNAtbSbZUO1dHnEVWKKKAicoK7Xwdc18QBI5CT3zC0KmjKzrkAzaKbk6Bz2utIQG2NVkrHodVRR+BE0g/23ZGgmIYkXB942t0LXT0RshMPRM9hMSR/1gBmL41sTteHv6ujldSlwJPIWJeSaOk85boDD+uapUXl93lElSI6SUQUHxVK0GlmS18E8/YHPxilbP0JrYw6ooDSkY2cuyJKcnQMMBVqf4b3zCwHp7pGMQkJpZ5oQXcYkkPjgNqtgcupWz39hGaVXamjXJjRyIUHwew3oZuZ7Re8LruY2caNHF4YKrTPI6obcQUVURpUFkHnw2b2C7DgAvjkMvjlGDkN8E9kxzkOja770XhQzm3IHrQe1HwvT/qnkIt43nD3mQR3cDN7FjnBvYA46S7oj+KKUgJqJ5RT/FdIx3oSIRAtDXoAG8I+b8B5KC35dOBcFA1QfFRWn0e0AMSMuhGtD9WSDrxa2hkRUSJEARXR+lAt6cCrpZ0RESVCtEFFtD5USzrwamlnRESJEFdQEa0XdUSsle15Vi3tjIgoMqKAimjdEKHqWcjV26nP0VeLQo0eBS4p64qkWtoZEVFERAEVEQFUjedZtbQzIqIIiAIqIiIiIqIiEZ0kIiIiIiIqElFARURERERUJKKAioiIiIioSEQBFRERERFRkYgCKiIiIiKiIhEFVERERERERSIKqIiIiIiIikQUUBERERERFYkooCIiIiIiKhJRQEVEREREVCSigIqIiIiIqEhEARURERERUZGIAioiIiIioiLx/wGGY61b2AYmIQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "_, _, _ = plot_structure(sm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see there are two separate subgraphs here in the visualisation plot: `Dalc->Walc` and the other big subgraph. We can retrieve the largest subgraph easily by calling the StructureModel function `get_largest_subgraph()`." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOydd5icZfWG7ycQSEILXUoITZHeq0BCl957R0Kx4E8pigJSlACCAioiAalGekeRGgHpHWkivfcgJaEkz++P8y6ZLJvdmdmZ+WZm3/u65kp25itnZme/873nfd7nyDaZTCaTyTQb/YoOIJPJZDKZrsgJKpPJZDJNSU5QmUwmk2lKcoLKZDKZTFOSE1Qmk8lkmpKcoDKZTCbTlOQElclkMpmmJCeoTCaTyTQlOUFlMplMpinJCSqTyWQyTUlOUJlMJpNpSnKCymQymUxTkhNUJpPJZJqSnKAymUwm05TkBJXJZDKZpiQnqEwmk8k0JTlBZTKZTKYpyQkqk8lkMk1JTlCZTCaTaUpygspkMplMUzJ10QFkMpmENAewO7AUMBgYCzwKnIP9dpGhZTJFINtFx5DJ9G2kFYFDgQ0BAwNLXh0HCPg7MBL7vsYHmMkUQ05QmUyRSPsBJwED6L7kPhEYDxyIfXojQstkiibPQWUyvUTSIpIelvShpAO62W4+SR9JmgpgqPSfP8EpwCB6/lvsl7Y7KSW1TKbtyXNQmUzvOQS41fYy3W1k+yVgegCkFYfCQlNVfpPYkaTux76/qmgzmRYhj6Aymd4zFHi8wn0O7Vfl399EGDAh5qwymbYmJ6hMdUhzIB2MdD7SNenfg5FmLzq0RiLpFmAt4PepfPdDSQ9J+p+klyUdWbLt/JL8lDQXIYj4kiOBXUp+foFQRnyRfh4O/Bz4FjAd9HsONj5FWlDSWZJel/SqpF92lA8lLSzpn5I+kPSOpIvq8gFkMnUkl/gyldGz4uxopD6jOLO9tqQxwAW2z5Q0HNiNGFEtAdwo6WHbV3bsMzRer1iddD4h5VsEmAgTz4SrgDuBhYHpgGuBl4E/AccANxDJcxpghWrfYyZTFHkElSmfmJwfA2xOqM4GdtpiYHp+c2BMX5zMtz3G9mO2J9p+FPgrMKx0m36wJF/97HpkD2Bx4q7yfRj4FCwG/J/tj22/BfwW2CFt/jlRepzb9njbd1T9pjKZgsgJKoMkS1q4h4065NBlK86GwmkjpZNrE2VrIGllSbdKelvSB8B+wGyTbROLcCtmSMn/XwQmxOf8uqSxksYSI6c50iaHxKm4V9Ljkvaq5pyZTJHkBJXpmSjrdSSn8ncDLQ/7IvWl8tJo4GpgiO2ZgNOJRPElDoeIyZgO+KTk5ze6OHDpQYYAU8MEYDbbg9NjRtuLA9h+w/YI23MD+wKn9XgTksk0GTlBZcrhUKJ0VzGK+Y++pDibAXjP9nhJKwE7dd5gIjxGzNd9yTLAbcBLwAfAyB5OMheM+zo8BZwkaUZJ/SQtJGkYgKRtJc2bNn+fmPOa2Js3lsk0mpyg2hBJP0mqrg8lPS1pHUlTSfqZpGfT8w9IKq0arSvpmVQu+oMkAbwlzXk0bDoU+s1BzO5/ULLT1cS8yGBCafZk51jiO7bR6tL6ku5P6rY3Jf0mxTpc0iud4n9B0rrp/0dKulTSRSnuByUtXcOPq9Z8Fzha0ofAEcDFnTd4Ec6j06hqPWB7woRveWCTns+jfWAz4gbgCSIJXQrMlV5fEbhH0kfEr+mHtp+r6h1lMkVhOz/a6EGIvF4mJscB5gcWAg4m7twXIS6OSwOzpm1MKMAGA/MBbwPfts334eIFYeKz4A/BW4J3ARv8NHgQ+AbwZ+DjwQuBP02vDwXfGP//ZEgop3dN55seWCX9fzjwSqf38AKwbvr/kcSE/zZAf+Ag4Hmgf9Gfda8ecLlhgtNnVeFjguGywt9DfuRHnR95BNV+TACmBRaT1N/2C7afBfYGDrP9tINHbL9bst9xtsc63A5uJapO3AZrHAhakMgqI4ELifU5FwEbE3f/HZljHKF77sTAGUK1trCk2Wx/ZPvuCt7TA7Yvtf058Bui3LhKBfs3IyMJb71qGE/PVcBMpuXJCarNsP1f4P+Ikcdbki6UNDcxr/5sN7uWzst/QrLkGQvTDy15YSiRnN4EXks/d9AvneTVLg5+Pvwb+AbwlKT7JJVRxfqSlzv+Y3si8AowdwX7Nx+xRuxAJtdGlMMnhGFstjnKtD05QbUhtkfbXp3IHwaOJy7yC1V6rMHw0YslP79ErMOZk8gQpa85nWSeLo6zHLxqe0dCBn08cKmk6YCPKVEHJieEzm4UQ0pe7wfMS+TH1iZcyQ8EPpnY88LdiUxKTtnNPNMnyAmqzUjO2mtLmpYoBY0jLm5nAsdI+rqCpSTN2tPxVoc7fgN+HvgI+BkxmT81sB1wHXAzMUl0ElFbXO2rhxn3Y5ha0uxpBNQhs54I/AcYIGljSf2Bw9JhSlle0laSpiZGh58ClZQImxf79Gdgg2vgc8f7Gtdpi3HE7/FKYFhOTpm+RLY6aj+mBY4DFiXyxp3APkRVblrC/mY2QqK8ZU8H+wV8f3bYYk2YejywAfC79NoiwAXAD4iy3jLANYSsrBM6I6apHpc0iBh47WB7HDBO0neJBDoVcAJRwivlKiIvngv8F9gqzUe1Bd8Iq6IbDXsCuz8KW46HISuFa8ejwLnkjro9kzsStx25YWGmZ6TLCfuiakbcE4Ersbeu7tQ6EljY9i49bduqSPoXcILtq9LPewJr2t6z2MhahNyRuG3JJb5MOWTFWZ2QtDixFOC6gkNpTbI/ZFuTE1SmZ7LirJ6MAM62/UWPW2Ympwp/SHJH4pYil/gy5TPpgjCAbi4IhomKkVNWnHWDpAHEfNuKtp8veT6X+HoiOhLfcxaE5UhlfDI/XP8iPGz7mJrHlqkZeQSVKZ9INsMIRVmHQrCUcZ/BxP/Ag2TFWTlsTSxCfr7HLduMZGF1QS8Ocag62UVNiXOA1Sd/asALQE5OzU9W8WUqI8p1W6fOuV9RTA2HN++C/ZzLeuUwAvh90UG0HKHW27DH7aZMP2AjpNmzuq+5ySOoTHXYb2OfiL0b9mbp3xPvigZ9C6TJ/8wUkLQI8E3CyLWt6cK8eGPSkjpJH0l6JG33pUlw+nmyUZakXSW9OAiePzqWJABhgTIIKPXtepBY7f0Y0ZDrLsIapaMR127Qf9P4rn5pWCzpEElvSXpd0haSNpL0H0nvSfpZSRz9JP00GS+/K+liSbPU9lPLQE5QmRqTJvv/TIwOMlNmb+Bc258VHUg9SYn4+8Q82wzEUrqngGOBi2xPb7tHd3pJiwF/BHZ9G64cC/07Fst9jXAcLrWNP59oLbwk0ZBrVWKheccK8X4w1ayTmjt2HGYAYYRyBDAK2IUwl18DOFzSAmnbHwBbEOXuuQkn+T+U9YFkKiInqEw9OAvYOYkAMp1ILh+7E4uT250pmRdXyjbAtbZvmw5mPIbJL167E4vGO074V2DXHg7Yf/IeZ58Dv0oLwC8kFrOfYvtD248TLU06Eul+wM9tv2L7U8L3cpvkdJKpITlBZWpOmvR/ENiq6FialM2Bf9t+puhA6k035sWVMjeTTIPHTgeU+nRtTmSQ54EbgZmAlXo44OeTr+171/aE9P8O8c+bJa+PIxkoEx6XV6TeaWOJNmgTCIvKTA3JCSpTL0YRFkuZr7IP8fn0CaZgXtzV+pbJjIOJslsHrzPJNPjRj2Fc6ZzTAMIb8gKivFc6eupK6jcRJrwLb1X0RibxMrCh7cEljwG2uzLyz/SCnKAy9eJqYFFJ3yg6kGZC0kJEqeiKomNpBN2YF78JzJ/c6Tt4GNhBUn9JKxBlvQ4uBTaRtPrdMPpw6N+5f/1uhKT8aiZPUHMSi806T/bdGT591XA68CtJQ9N7nF3S5lUeK9MNOUFl6kKa/D+HLJbozHeA821Xax3VanSYF79DCO7mIHzzLkmvvyvpwfT/w4mWMO8DRwGjOw6S5oG+B4xeFR79HP4zb6cTfYu4oC3H5H3K1gYWJ4Zjs8VTE9+BV96r3Bmlg1OIPHiDpA8JZ/2VqzxWphuyk0Smbkj6OnAHMKTd1WrlkNqJvASsbfvJbrbLThI9EQaxY5i8JMjawE6ERLIbPiEWkue1ek1OHkFl6kYSATxOzGFnYBPgv90lp0yZdOEPeR+hzNm++z2zP2QLkRNUpt5kscQk9gHOKDqItqGkI/FuwLrAycAMXW+dOxK3IDlBZerNFcAykhYsOpAiSRPqKxGT/ZlaYZ/+Eay1HXzyPny2R+5I3FbkhWU9kbt09grb4yWdT0wL/Kyn7duYvYDRqYtwpobMEEuinjBsRNd/q7kjcYuSRRJTInfprBmSFgVuAeZrp1bt5ZIcBl4g1s48Vsb2WSRRAYqOz9fbzuXTNiOX+Loid+msKUkU8F9CJNAX+TbwSjnJKVMZkuYC1iIZv2bai5ygOlNGl84NgXPjv1/p0ilpfknOvlxfYRR9d01UFkfUjz2AS21/WHQgmdqTS3ylTGFtRZl8AgxTLEh8Huif23hPQtJAYkH/srZfKjqeRiFpHqLrwxDbH5e5Ty7xlUFyoXgG2NH2vUXHk6k9eQQ1OYcyucNxJQxI+2e6IIkDRhNOCn2JvYCLy01OmYpYG/iQWAKVaUP6dIJKDdIOlfREP2ns7rDZeOj3PjFZMjswc/r/KyX7DWdSn4QJwEHAbNBvAdhy7h7XCfZpzgD26ivlz3SH/x1yea9ejABGOZeB2pY+naASOwMbPA2/+Q/ol8SKvj2BFwlfmoFEx7WuGAVcCzwE3A/jB/TdeZYeSSKBVwnRQF9gPaKNw4M9bpmpCEmzE80P/1J0LJn6kRMU/N72y1+Hrx8O/f5KLKrYmpiImgH4OfDPKex8MdHsZkjsN/DwmH/KTJm+JJbI4oj6sTtwpe2xPW6ZaVlygprUBG3wUOA1Qu2wL+GIPCOwJrHib0IXO7/GpCY1AIvnz7QnLgLWSOKBtkXS14g5kix/rjGSRCrvFR1Lpr7ki+mk/DL2JaJt50nA08A9wP+A29IGXRW652JShgN4MiqEmSlg+yNi4NnuCrU9gMts/6/oQNqQNYEvgDuLDiRTX3KCgu9JmvcZeOYYmLg9IQsaSHilvEc0ppkS2wGnEiKK92Dc0TB/vQNuA84A9u7UrK5tSO9rb/Idfr3I4og+QlteICpkNHDDInDgQuDDiDmlcURzs1XofkZ/BDFTuzSwHAwYN0ngl5kCSTTwLiEiaEeGE5XivDanxkiahRDWnl90LJn606cX6kp6Adjb9k3picsJ+6JqEvdE4ErsrWsWYBujcN5Y1/Y2PW7cYki6ELjD9u+r3D8v1J0Ckn4IrGR756JjydSfPIKanJGENX81jE/7Z8pjNLCOpDmLDqSWJPnzt8ny55qTxBH7kEunfYacoErpoktnmeQunRWSxAOXE2KCdmI34Crb7xcdSBuyKtCfKa/6yLQZfTpB2Z7/y/LepCe/7NJJz4q83KWzd5wBjGgXsUSWP9edLI7oY7TFhaHmRLIZRnThHE+nLp2GceOB/8E/yF06e8O9RIIfXnActWIN4qblX0UH0m5IGgxsyZeNBDJ9gT7hiVYVUa7bmphTmKxLp+DRxeGbz0WPn1zWqxLblnQGcWd8S9Hx1IB8h18/dgJusP1W0YFkGkefVvH1BklLEzZ889vuymQiUwaSZibsoRa2/U7R8VRLkj8/Rw3eR1bxTU4qnT4EHGz7xqLjyTSOXOKrEtuPEE5HfcX4tC4kMcHVhLigldkF+FsrJ9kmZgXCdezmogPJNJacoHpHR3kq0zvOAPZJd8otRxZH1J0RwJm2s41YHyPPQfWOi4BfS5rb9mtFB9PC/IsQF6wO3F5wLNWwCtGwckzBcbQdkmYAtgUWKzqWwpHmoNN8OPAocA7220WGVi9yguoFtj+SdAlhfPqrouNpVZJYYhSxCLMVE1QWR9SPHYAxtl8vOpDCkFYkunVvSHhWDyx5dRxwNNLfgZFpLWfbkEt8vaetjU8byPnApkk00TJImoksf64nfbunVliCjSEs2AYweXIi/TwgvT4mbd825ItqL7H9APA+sG7RsbQySVzwN0Js0ErsBNxk+82iA2k3JC0LzAHcUHQshRDJ5iSid2pP1+p+abuT2ilJ5QRVG7JYojaMooXEEtkbru6MAP7c6ss4JL0gaZykjyS9KekcSdN3s/0eg6WHmZScuuUFQESDLCYlqRVqEXvR5ARVG0YD67ab8WkBjCHKFSsXHEe5LE9MVt/U04aZypA0HTH/9OeiY6kRm9qeHliOkM0f1t3G88K8xN9CNQwg5qxanpygakCJ8enuRcfSyiSRQYdYohXI8uf6sS1wp+2Xe9yyhbD9KvB3YAlJe0h6TtKHkp6XtLOkRYHTn4RZp4d+g9N+1wHLEovBhgBHlhxzzfTvYGB64C7odwRsNn0IuACQNL8kS5o6/fyVc9f1jVdJTlC1YxRhfNoS5akm5lxgyyQ+aFpSiWY74OyiY2lT2lIcIWkIsBHwJNGMe0PbMwCrAQ/bfvIAuGplmPgRoSMHmA44L/18HfBHwigU4Lb071jgI8Ly3cA3YcEpxDBdV+eu6RutETlB1Y57CGPZ4QXH0dIkscFNhPigmdkB+Gde/1Z7JC0BDCVEM+3ClZLGAncQ7UKOJNb+LSFpoO3XbT8OMDvM26/TtXk4sCTx5FLAjnTfc2QqmHom6E4R2+W5m42coGpEKk+dQeuUp5qZUTS/6CQ7R9SPEcDZtr8oOpAasoXtwbaH2v6u7Y+B7YH9gNclXSfpmwDTdCGMuAdYC5gdmAk4HejJU6t/9M76Ct2du9nICaq2XABsKGm2ogNpcW4CZpa0fNGBdEUyCp4buL7oWNoNSQOAnYGzio6l3tj+h+31gLmAp0g3PJ930TB1J2Az4GXgAyKzdKwK72pOYTrgw8n72X2tnHM3GzlB1ZAS49Ndi46llUmigzNp3tHoCOCsVpc/NylbAw/Yfr7oQOqJpDklbZ7mgz4lpo8mAvSHJ18Gf1ay/YfALIQ8715CNtzB7MSF/LmS55aCTx+C2STNl+Zzv1T1dXfuZiMnqNrTUmt5mpizge26Wy9SBJIGEVMA7SJ/bjbaUhzRBf2AHxMdEd4jGqTuL2nFP8DgxUBfAzpKMacBRwAzAEcT6pwOBgE/B75FKPnuBjYA94cLCa++B4jWQN2eux5vsrfkflA1JiWmJ4ARtu8oOp5WRtKVwLW2zyw6lg4k7Q5sZ3vjOp6jT/aDkrQIMfc/nz3ZAKKtkTQNMXI8gCi5/WEsrDsDrNev6wpeT0wErsTeupZxFkEeQdWYFlzL08w0o1giiyPqxwjg3L6SnFKp7XCiYec+wAnAwsBDG8Eyn0O1JeTxwMgahVkoOUHVh/OAzVrN+LQJuR6YO4kSCkfS4sTakuuKjqXdkDQt0bSyaUbL9ULS8pLOJcQJ8xHrkdYiFvCeAJxzJ+wyLfyALgQTPfAJcCD2/TUNuiBygqoDyfj077Se8WlTkUQIZ9E8o6i9Cfnz50UH0oZsAfzb9jNFB1IPJPWXtL2kfxGuM08AC9seYftRSUsB9xEJa2nbN2KfDhxIJJ2eRAwTmZScTq/fO2ksOUHVj+wsURv+DOyYxAmFkeTPu9AH7vALYgRtKI6QNLuknxNlvP0JA9iFbB9v+11J/SQdSLSzPxHY1va7Xx4gks0wwjhiPNH/qZRx6fkrgWHtlJwgNyysJ2OIXi0rEevsMlVg+yVJdxPebEX2XNoKeKjd5c9FIGkhYGngiqJjqRWSliNKdFsAlwEb236k0zZDiO/0NMBKU/xuRblua6TZ6bqj7rm5o26mImxPlNSxlicnqN4xCjiIYhPUPsDvCzx/O7M3cJ7tT4sOpDdI6k80rzyAsGr6A/D1VPLvvO0OhB/eycDxZa2piyR0Yi1jbnayzLyOpPYbTwFDk+N5pgrSH/6LwHpFeIZJ+gbRin5IIxRmfUlmnn63LwFr236y6HiqQTGyGQF8l1gveypwZVdWTZIGEzc6KwI7u03EDPUiz0HVkWR8ejPNb3za1CRRwtnEnXYR7E0fkj83mE2B/7ZicpK0jKQ/A88Q8vBNba9p+9IpJKdhhGv4/4DlcnLqmZyg6k8zruVpRc4EdklihYaRFlHuThZH1IuWEkdImlrSNpJuI9wZ/kuU8fay/dAU9plG0nHAX4HvlZjFZnogJ6j6cyMwa7Man7YKaQL5IUKs0Eg2B56w/Z8Gn7ftkTSUEBFdWnQsPSFpNkk/JUp4PwR+Byxg+1h3I1CQtBgxB70osIztvIauAnKCqjMlxqd5FNV7zqDxn2Nf8YYrgu8Af7HdWTrdNEhaOomdngEWIdpmrGH7ku7Wwyn4AdFP8LS031uNibp9aA4VnzQHXcsnz2kT+eTZwGOSDrL9UdHBtDBXA3+Q9I1GjGgkLQgsQxvJn5uF1Hp8L2DDomPpTIptc0KNtzCRYBYpN8FImptYvzczsGq7Lj5uBMWOoKQVkS4nFFpHEQshN0n/HgW8hHQ50ooFRtlrbL9KqMC2LzqWViaJFM6lcWKJ7wAX2B7foPP1JTYEXrH9WNGBdCBpVkk/Icp4PyYS0/y2f1VBctoKeJAwFV89J6feUVyCkvYjFrNuTrQ5Gdhpi4Hp+c2BMWn7ViaLJWrDmcDuSbxQN5L8eU+yMWy9aBpxhKSlJI0CngUWA7ay/S3bF5VrayVphqToOwHY0vaR2RKr99QkQUl6QdK6XTy/hqSnu9hhP8LyY1BHDGOAeacc4yDgpBZPUtcD8yTPrUyVpNLeE8SNSz3ZGHjO9hN1Pk+fQ9I8wOrARQXGMJWkLSXdSvxtvkSU8XavVP4taTVCPj6BEELcVfuI+yZ1HUHZvt32IpM9GeW6juRUCR1JaoUahddQ0rqIP5NHUbWgEWKJLI6oH3sBFxUhtZY0i6SDidHSIcTveH7bx6R1i5Ucq7+kYwjz1wOT8WueY64hRZT4DiVKd9UwgJLWxVMiTXI2I2cBOxVtfNoGXAEsm0QMNUfSfMDKtID8udWQNBUxt9fQ0qmkJST9iUhMSxKmrKva/ms1C7CTu8i/gBWAZW1fWduIM1DbBLWMpEclfSDpIkkDJA2X9ErHButI6ywDW8wA/bYlFAOHdTrIScAcRFvJs0ue/xQ4CPoNgS2nkt6SdLqkgQAd55H0E0lvdNq1abD9ErEmYpuiY2llkmjhAuJCVw/2AkbbrrQXT6Zn1gPesf1gvU+UynibS7oZuAF4FVjU9m6276vymJK0L3AncA6wke3XaxZ0ZjJqmaC2A74NLEDIxfcofVHSNA/ApbvBF+8BO/JV7e4bwAfEt+gs4HvA++m1nwL/AR6G8S/BKcA8wBElu38NmIUwaWzmbrZZLFEbRgF7JjFDzSjqDr8PUfeOxJJmTi0s/ktUXM4iynhH236jF8edk1jqsA+whu3TnM1M60otE9Sptl+z/R5wDbF+pJRV+sG0P4L+/Qk7gJU6bdCfyDj9gY2A6YGnAROF4t8Cs8LAeWLB3LHADiW7TwR+YfvTZl74R9ijLJxWmGeqJIkXniPEDLXk28Brth+t8XH7PJK+BqxNWP7U4/iLSzqd+F4sC+xgexXbo3vroyhpU0II8RixtqnlvANbkVomqNI7k0+I/FLK3HPCp6Xd+4Z02mBWJl85PAj4CHg7HXB5YhXv9DEAux6YvWTzt1thvUoTGJ+2E2dQ+9FyFkfUjz2Ay2rp7J/KeJtJuokwZn4DWMz2LrZ73eZG0nQp6Z0KbGf7Z9k0uHE0UiTx+pswbel4+OUyd5yNWBT1OGEx8RH81fZMtkuTYCsNtc8Edm208WkbcimwchI19Jokf16DAuXP7YqkfsRNWU3Ke5IGS/oxYUF0ODEfNDStP6rJnJBCcfwQcflZxvbttThupnwamaDumgCfngKffwFcBdxb5o79iML1j4A3o8Xxo5LmkbRBnWKtK7afI8oFWxYdSyuTRAyjCVFDLdgTuDhLhevCWsDHlP9n3yWSFpV0GtFCfQWilc1Kti+oVcPD5Fh+OFGOPyytjfqgFsfOVEbDEpTtz5aD7c6GqQcTEqxNgGnL3P94whRrFRjQD34B3ETMRbUqWSxRG0YB30nihqpJd/hZHFE/RgCjqhEVSOonaRNJNxBr+t8BFre9k+27aylUSO3nbwOGAcvbvrhWx85UTuM76ob33uZAv5WB/Yjb1nIwTBRcib113eJrEMmq52WyX1evkXQPcHRvWhlIWh8Yabvwtijt1lE3dZx9hmhP8X5P25fsNxNxefg+Ud0/lVjgW/PW8JKUznU8IcA6JXUiyBRIQxfqShp2CZzxOYw/l7Ar/3YF+48D/RT+WafwGkqaaD2PLJaoBbUQS+xDHj3Vi92Aq8pNTpK+Ken3RBlvZWBXYEXb59UpOc1GzGf+H9F6/rc5OTUHjXaSWGQ7OHcQTHUi+FJiQW6ZfHIn/PZ4OFDSH9rEjWEUsEe9jU/7ABcBaySRQ8Wk9S3rEPNZmRqSRiY9rn1KZbyNJF1P3IS+Dyxpe0fbd9VrvVGax36ESIYrNZO7eqbBCcr2Gbbn/Nwe8Bh8d+NQj/d0pzKR2O7Ade0DgaWBmYAHW71LbTI+fRLYrOhYWpkkariY8qvFndkDuLyW8ufMl6xB/A3/q6sXJc0o6QBiyeMviTVSQ20fntrU1AVJAyX9jkicu9o+qBWWqfQ1imu3YZ9OTEReCYwn1Hlf8hlM+AK+SK8PS9tje6ztXYAjgb9L+llvJ8gLpoguse1Ih1iiou90usOvmfw58xW6FEdIWiQliBeAbxE3CcvbPrfeiULSssADxDrKpW3fUs/zZaqn8SKJLqPQ7HTqqHsfjN0MNnkDFprS8F7SEKKB3TTEXdDzDYu5RqS1UC8T5YWWi7+ZkPQAcKjtGyrYZy3COmvpZrGtaReRhKRZCFeHhW2/k24eNiA61S5P3BT80fYr3RymlvFMBRwEHEisWhndLL/zTNc0h+t3tHU/sfSpleLOdi1gOHBr17v55dSH6kfAvZIOAs5rpS+d7fGS/kJInDt752YqYxQhdig7QaXtq5I/Z3pkFwELe1QAACAASURBVOBvwGeSfgD8gFgLdSrRFLBhlmSShhKiJAjBxYuNOnemeopt+d4N6YLRY/nL9kTbJwHrAgcDF0uatQEh1pJRwF5N3CakVRgNrJNEDz2S1FsbEsvyMjUklU6/Tyx1fIGYi/oOsJztsxuVnJL7+C7AfcB1hEovJ6cWoWkTVOICYKN0IekW248QK8tfBh6WtF69g6sVth8nVES1Nj7tUySRw+V0ctLvht2AqytZm5PpnqTG2wC4A1iQED8sbXs7RwPTho1UJc1MiC5+Bmxg+wTbExp1/kzvaeoElS4c1xDrIMrZfrztHxNqrj9LOrmjZ1QLkMUStWEUsHdPYoly5c+Z8pA0g6TvAU8Qi10nAkckc9VybTdrGc/ahHz8LUJ88VCjY8j0nqZOUIkzgH3SBaUsbN9EyNHnAe6TtHS9gqshlwCrJuFHpnruIRShw3vYbvX07x11jabNkbSwpJOBF4k5430Ide6ShClyo+OZVtKJwPnACNsHNHn7nUw3tEKC6riAfKuSnVJfqu2AE4CbJB1UqQS5kSTj0wupnfFpnySVkMrxOazaG66vk+Z11pd0LXAXsUxkGdvb2L6NMHC90fZbDY5rSWKuaQGirPiPRp4/U3ua9oLdQckFp2IrGwfnASsS/n83N/kI5QxqYHya4QJgwynNXaa5ic2YpOrKlIGk6SV9lyjjnUisURxq+6e2X0rbiAbbRqV5rx8BtxB9Tbex/U6jzp+pH02foBLnAZulC0vF2H6BKPncADwgacfahVY7ktDjDWKtSKZK0tzl1YQIoit2Af6eL2LlIWkhSb8hynjrAPsTI5Qz08i/lI6+ojc1KLZ5ib/rbYCVk0Iwj4rbhJZIUOlC8ndg514cY4LtkYSs+AhJf5E0uFYx1pB6dInti4wCRnSeu8ziiPJIZbx1JV1NzOt9TkjEt7Y9ppskMAI4sxFmq5K2Ax4kWnAMc/RZy1SCNAfSwUjnI12T/j04mScUTnM4SZRBUuWcTA1W/Cej2ROATYHdbY/pfYS1QdL0hFR+MdeoM2hfJCWiJ4B9XNIJVdIqRAnwG83qWF2kk4Sk6QjV7AHABGJR7V+6GCl1tW/Hd3dx26/VMcaZgN8BqwC72O5VE8Q+SXQLPpS4YTfRNbiDcYCIQcFI7PsaH2DQEiOoxBhgELBSbw9k+xPb3ydKFaMlnSCp3N6JdaUGxqcZuhVLdIgjmjI5FYWkBSWdBLxElJi/Byxle1Q5ySmxA/DPOienNYhu1B8Dy+bkVAXSfsT1dHNgAJMnJ9LPA9LrY9L2hdAyCSpdUKoSS3RzzL8RcvRvAPdIWrxWx+4lZa3lyfTIZHOXkmYEtgLOKTKoZiGV8daRdBXRin0isILtLW3fWkWlom7iCEnTSBpJ3LwdYHt/2x/X41ztgqT5JXkyh5pINicBg46Efrt0f4h+xKDgpKKSVKtdAM8BtkoXmprg8AHckigZjJF0QBMkhgeAD4gJ6UyVlMxddvwd7gTcbPvN4qIqHknTSdoXeIwo4f2NUOMdXK1hsaRliPZu19cu0i+PvShwN7AEIWe/ptbn6BNEWe8kIulUQkeSWqH2QXVP0RfiikgXlpuBmqrwkhz9LGBV4iJ2vaS5a3mOSuMhiyVqxRlMEkuMSD/3SdId9a8JNd6GwA+BJWz/qQajkRHAWbW0EkojvO8BtwOnA5v19ZuLXnIoUbrrkS+++tSAtH9DaakElahpma8U2/8lHAb+BTwkaet6nKdMRgPrSpqjwBjagTFETX03YBYaJH9uFtJFfi1JVxAjcxGtXbawfXMtJNlJdLQD8OfeHqvkmHMRI7vdgdUczU5bQ9FVZyT9VNKzkj6U9ISkLdPzU0k6UdI7kp6j1NtTmuM52GgY9JsBWA8oXWPxAvHFOAuYD1g7PX83sBowGPotBVsuL21eEscekp5LcTwvaef0/MKS/inpgxTLRVW/Wdst9SCS6guEv1Y9z7My8AxwNjBjQe/1z8DBRX/mrf4AfgI8BRxWdCxlxrsncHYvjzGIGNU8RqgZ9wOmr1O8uwPX1fB4WxLrAY8G+hf9+2i2B7AtMHe6Fm5PCEbmSr/jp4AhxM3YrYRCb2rDwSvDhB+Bx4P/CZ4evDPY4OdjO+8K/gj8CfgV8Czg68ATwNfD+AFxrtmB6YD/AYukmOYi1JsQBr0/T/ENAFav9r223AjKIZY4kzobq9q+B1iWWP/xsKSKrJZqRJdreTIVczEhhLmk6EDqjaShkk4g1HibEr3SFrd9ukMhWg/2oQalU4Xh7JmES8VWto+w/Xmvo2szbF9i+zVHq6GLiBvplQhrt5Ntv+ywehvZsc+/YZX7od8xRP+TNYkvR2eOJDLPQFIrifRInSanXRTGpqcgRDVLSBpo+3VHVwaIa+ZQYG6HgXfVfpctl6ASZwPbpXUXdcP2R7b3If7IL5P0S0n963nOTtwNfEqYb2aqZ23ijnytogOpB6mMN1zS5cTC1akJV4XNbN/kdFtbp3MvTnjfXdfL46xKyMdFCCHurEF4bYmk3SQ9LGmspLGEeGQ2YlRV6hz/Zd+rl2GOmYnk08HQLo5d6gP3InFHN7jk8QTMCczlmLPcnhi1vS7pOknfTLseQvwe75X0uKSq/UVbMkHZfpWYON2+Qee7CliGGFHdKWmRBp23ah/CzGSMAE6jzdqZSBokaW+ircTpxPzaUNs/tv1sg8IYQZQju5hX7xlJ/SUdRfj6HWz7O7Y/rGmEbYSiM/AoohnkrLYHA/8mEsLrTJ5j5uv4zzzw9vtEfa6Dl7o6fsn/hxArtseWPMbDaNvHAdj+h+31iPLeUykubL9he4TtuYF9gdMkLVzN+23JBJUox7G6Zth+A9iEmBe6Q9L+DSq9dTRtbLUuwU2BpKWItivHAbNIWr7gkHqNpPkkHUfc5G4BHEQ4j5xWxzJeV3EMIOzHqmqrIenrRLeClYlR0+U1DK9dmY6YL3obvnQdWSK9djFwgKR509q/n6bn+78M45YDfgF8RnzoPWn1d0nb/IOwFBkH434HH6fjzylp8+Q88inwEVHyQ9K2Co9EgPdTvFUtjG/lBHU9ME+6ADUEB38klH7fAa5Rme3Fe3HO94jvyZSMTzPdMwL4c7rDP4sWHUWlMt6aki4lSmEDCHXbJrZvcDHOGFsBD7nCtVPpvewD3En0bdrQ2darLGw/Qaxlugt4k+i79a/08iginzxClHpvTM8/ux8MuQA+v4dQTxxFzxeUIcBVwLGEKmI+GHgILEzkjX7Aj4HXgPeIaYj9064rEsYHHxGmzT90tT6JRStSeqlmOQr4XUHnngb4JTGs3qzO51qTUGKp6M+8lR6Eku1dYL708zzpj6kuarYaxv2lio+Yr96LSEpPExZEMxQdY4ptDLBthfvMkS5aDxGjvsLfR7s9gBUIF5X3idLvErYxXG6Y4KTcq/AxwXBZo99LK4+gIMptO6V1GA3F9me2DyMkn6dI+lMa7taD24k7ltXqdPx2ZRvgHqdeRW7w3GUvGSTpWKKMtzUhlV/U9h/cBHM0kr4BLErcZJe7z8ZEon2CEHE8Uafw+hzJCmpHSXcClwKPAgvZ3s/2v9NmI4nmktUwnhJVYKNo6QRl+0WiFcA2BcZwB+HnNy2xuLfXZrZdnCOLJaqjK/lz0zp0pNLX6kSpZAtivmF12xs7JqSbyeB2BHCu7c962jCJOU4D/gDs4Ghw2ON+mZ5Jc0GHA88Tv5MTiMR0omN6YBLhSn4gUK75bwefAAdi31+DkCuiZdptTIm0ivrHttdogli2If4I/wAc6yqVTVM49uzEeocFHA35Mt0gaTEmqdo+L3l+KmKh98a2Hy0ovMlIYoMdgR8A0xPGrdjuwcuzGBTO/y8Ba9j+Tw/brgD8hXhP37f9QQNCbHuS2OcAojP0JcRUx2Nl7txhGDuA7gcpE4mR04HYp/cq4Cpp6RFU4lpg4XRBKhTblwLLESKK2yUtVMNjv00IQ6pu2tjH2JuYx5lsoafDK64pxBJJDfUrooy3HbH6/puE32QzL1DdHHiiu+QkaWpJPyfsio6wvWtOTr0jSfK3l/Qv4HLgcWBh2/uUnZyAlGyGEdL+8UT/p1LGpeevBIYVlZygDUZQAKlWP8D2j4uOBUDhhv4D4DBC6vln1+CDlrQO8Ftq0LSxnUkjkpeJeY6vqIckzUdM0g9x+b2OahWbiLnEA4D1iWUEv7f9dMk2hTUsLAdJNxLf6b9O4fUFCXXeeKIh6CuNjK/dSNWTfYjS738JB/qra1KhiWPvDixFrMUdS8xfnUvcFBdKuySoBYm5qCG2q50ErDmSliAuQM8DIxztH3pzvH7Af4CdHVZMmS6QtCOwl2MR4ZS2+Rtwoe3zGhTTAEKccQDQ0RH2nK5GFc2coLr7W0vJd3fg18SE+slNNm/WUkhalvi+bEEIH37XLGXpRtEOJT7SXfLDhMlk05DUMx2ms49I2rCXx2uID2EbUI43XEPEEpLmkXQMUcbbETiCaDd/SouWvPYGzu8iOc1KzIUcCKxj+zc5OVVOKuNtK+l2QiH5FFHGG9HXkhO0SYJKNNRZolxsf2r7EGJh9umSft9LWfw5wNaqYdPGdiK5EyxGz/Ln64AF6zF3mdR4q0r6K2FDMzMwzPa3bV/XqhduhQ/lnnTqmitpfWJx6IvAin3xQtpbJM0m6VDgOcLG6GRgQdvH23632OiKo50S1JXA4ukC1XTYvpWQo88CPCBpuSqP8wZwCzVu2thG7E0Z8ucknjg7bV8TJE0raVdCsXYBUQqb3/b3bT9Vq/MUyCbAs7afBJA0UNIphOhkd9sHNlOJvRWQtIyks4gqy8LApraH2b6slirgVqVtElS6IJ1HDS84tcb2WNs7AccQXXt/mmTPldKUo8WikTQNsAfle8OdBeya5od6c965FIanLxL+mkcRZbyTW7SMNyW+7EisaPF+P/A1QrRzc5GBtRJJ4biNpNsIG7P/Al93GOU+XHB4TUXbJKjEKGD3dKFqWmyPJuxINgBulTR/hYe4EZit2lFYG7MZPcifS+nt3KWklSX9hXBGmB1Y2/b6tq91DVufNwNJ+bgycLmkg4nv4Ehi4e173e6cAb4s4/2UKOP9kBDKLGh7ZG8FVO1KWyWodGF6irhQNTXJfmcdwpfsPkm7JhVUOfs2zVqeJmMfOs2PlEFFYolUxttF0r1E59AHiMXT321z657vEN/Va4ledyvaviAvd+gZSUsrGjE+QzTO3Nz2Go7Gg8283q1wmkNmLs1B11r8cyrV4kvaGdjN9gY1j7NOpHLJBcTCu/3LuSOVNA/RznuIo3lYn0bSAsTcT0VLDdJo+2XCUuiZbrabi+htsy/xezqVaHNe85FSs8nMJU1NOGdDSMh/3W4jxFqTPrPNCJn4wkQ/slFugrVFrUSxIyhpRaIL6ItE3X4XYiJ2l/TzS0iXI61YwVEvA5ZPF6yWINWdVySc0R9JC3J72udVoq1LKxifNoK9gQsqnaRPc5fnMoW5S0krSbqAKOPNCaxre13bV/eFi7Sir9BNhDP8eraP6wvvu1okzSLpEOBZoh3FacQI+9icnCqnuAQVflBjCNuUAURbgVIGpuc3B8ak7XskXaAuIEoSLYPtcbb/j2itcK6k35QxeZ/FEnx5t/oV+XMFnAns0TF3qXCG3knS3cBFhOvEgrb3t/14TYJuASStRczRzUf4XT5YcEhNi6QlJZ1BJKbFgK1sr2774lzGq56qEpSiz/zwqs86yaxwUBkx9EvbnVRukiIuVHulC1dLYftGQo4+HzE31V1Dxr8DQyQt2ZDgmpeNgeeqnQNKc5dPEoq+Iwgz2e8QIoCFbZ/kPmTQm+bZfk3c6P2cWMd1frFRNR+SppK0haRbCJ/Ml4BFbO9h+4GCw2sLqkpQthe3Paan7SS9IGndTk+uyKTkVAkdSWqFMuJ7nLAXekhS08rOp0RamLctcCJws6QfJ5ujztt9QfTE6uujqGrEEV+i+E5ORTR3mwdY3/Y6tq/qa+WsZM91LzFvsgwwP3CRG9hKvtmRNLOkg4jR0iGE0GYB27+0/Vax0bUXRZT4DiVKd90yhSb2A9L+5XAGMHcFcTUVDs4FViJaa98oad4uNj2LaNrYuUTaJ0jy51UIm51K9uto8HZX2vdvwAfAcZ7U4K3PIKmfpB8CtxICkK2IbsR704vk305IWkLSnwiZ+FJEN+HVbF/Y08LwTHVUW+J7QdK6ko6UdLGk8yR9mEp/K6RtzifKVNdI+kjSIUhz3AkbrQb9BhN1rDElxx1O1BO+RQyXnkvPHZ6emwH6rQtbHCEtUhLLKpLulDRW0iMlpcfFCNeG36fz/76a91o0tp8nrPFvBh6UtH2n118E7qPApo0FsxfwV5fpSq7JG7ztQzR4W9j2SKKM1XIj7t6SFKH/AHYAVrV9VpKPrwe825fLVamMt7mkm4EbgFeJzsa7ORoAZuqJq+t5/wKwLnAkYam/EVEiGQnc3Xm7jp/vhWNmAV8HngC+ATwL+K3U934YeAj43+DPwZ+l5xYEPw3+BLwGTPg23JKOPw9xl7cRkWzXSz/Pnl5/FbiqmvfYjA9ice/TxIV0ppLntwJuKzq+Aj6PqQiJ+NJlbLs8odZ7nxgRLNXFNosBrwFTF/y+9iR6WTXiXNsQEvLDO79vwkF7v6J/zwX9DmYmjG+fB+4CdgKmKTquvvaoRYnvDtt/c9TqzycGRl1yLmy8EZNnkxWI2koHewCLA1MD/dNzexKr2wYCO0C/t6BjBLUL8Ld0/okOgcH96RQQF5vVq7QTajocLZeXAz4k5OhrppeuAb4uadHCgiuGbwOv2X6kqxc15QZvXTpDO0QWzxGii7ZG0oySzgWOBTazfYxLvN8kzUksJB9dVIxFIGkxSX8kvgfLANvbXtX2aOcyXsOpRYJ6o+T/nwADpqSeexNmuYRYidvxuINY/NPBkC72+1rJ/wcBn02awxoKbJvKe2MljSW62c6VXv8Y+B9hKdQW2P7Y9neB7wEXSjoOEOFy3tfKUyPoYn5E0uyKbq7PE03eTgIWsn2Ce3aGHkUD2nAUiaTVCffx8cBy7rq32B7A5bb/18jYiiCV8TZVNGK8mbimLeroAnxvweH1aeotkpjMpmJ2eG9Xwiai4/Ex0XK2g3K8fiZAx53ey0RvmsElj+lsH1dy/n/Shio329cRd3iLAncT03m7SZq2yLgahaS5gTWBC0ueW07S2URTxwWAjW0Pt325y3eGvgRYJYkv2ookDPkV8R5/aHtfd6HOS4rRLpN/OyFpsKQfEd+Xw4mbvKG2j3J0DcgUTL0T1JvAgh0/7AZ/u4aYjZ1A3L6NASrpBz0BPvskRmoQ6zQ2lbRBugsaIGl4idrtTeBtYHiyqmkrHJLWLYA/Ep/FuzRZ08Y6sidxof1U0naS7iB6QD1NOEPvPaXSX3c4xBZ/JcQXbYOkbwJ3EiX4ZWxf3c3mw4m/sbbs2ixpUUmnESPsFYCdbK9k+y+5jNdc1DtBjQQOS+W3g1aBU6+AT48lrJ+HEMZelXRvE+j15Atm+2XCaeJnRCJ6GTiYSe/rlPT6IMICqe1wMApYjRAN/L4dk3Ep6Q5/H2Ik/TyTGrwt4LDi6a0zdMdC75afu1TwXeB2wjFjU9tv9rDbCMI3rgmMOmtDktFvIukGQkr/NrC47Z2nUOLMNAGNN4sN773NqSI5GiYKrsTeurJTagXgYmKCvCW7mZaDpOmAt4i7331tX15wSDVHYax7HKEiPQ/4ne2H6nCee4CjbP+tx41rf+6amMVK+hqxkHsOYGfbT5exz2xEf6IF3AbuGZJmIkbb3ydmFU4BLrb9aaGBZcqiiIW6I4nqXsWMAx0fdv+V8gCxCLNHE9ZWxuFq/kfCAukESWdJmqHgsHqNJm/wdi2xvOBQ23vVIzklWlosIWlzwkPwQWJtU4/JKbEbcHWrJydJ30xrH58n+ljtSrQIOT8np9ah8QkqFrcdyKR5pHL55Ab4009hpKSjJfXveZeOU9pU2PenhRkFrE+4o08EHpa0WrEhVYe6bvC2MjAvqbNrHbkQGJbEGC2DpOkljQJ+C2xj+zCXaVYqScTfSL0/27qQyngbSbqemN5+D1jC9o6272qnkmVfoRg3c/t0JiWpnkpuE9N2B25h7w8sS1x8/yXpGxWcdTSwrqL3VNuS7pSfJrq7jiA+58srTepFoskbvC0CbOHU4A3YGbjCdW6lntRtFxPloZZA0irEqGlqQgjxrwoPsTrx91bpfoWS1nQdQHzvjyH+1ue3fYTt14qNLtMbimu3EUlqGHAlUfIb12mLcen5K4FhaXtsv04sxD2XSFL7pDu/Hk7nD4AriMaI7c4ZJGm97SupPqk3jFTG20rSGGLt9vOEM/SeTm0e0u95BI27wx8FfEddGPU2E+mz+wWhYjw0fWbVrF8aAZzZKiMNSYtI+h3hWLMa8be9gu3zXGFfsExz0iwddWen646659JNk6/knHABYWm0t3twEpa0KpHYFmmVP8JqUPSReoX4Y30hPSdi0epRhOVhU6i0JM1CLDD+HhHzqcQC0a+UpRT9iU4lbIrqHnv6zB4AfpJcShpCJSIJSQsTfwP/A/aodsSgaEz4PCEk6q0Ksm6km4UNiE61yxE3EafbrmS1SqZFaI47Q/tt7BOxd8PeLP17YnfJKXbzk8CqwL+JuZZNejjT3cBnxMitbUl3j3+hpGljkqOfRixu3Re4ushyp6Sl0lzJs4S71da2v2X7om7mTBoqf07naUqxRJKP7034xI0Gvt3LctYuwN+bNTmlMt4PgKeAXxGNJIemObacnNqU5khQvcD2Z7Z/RrQ+/52k05Pcuqtt+5pY4itNG0uS+mOUl9RrRlpMvaWkWwml4UvAN23v7vAZ7G7f2YjS7gUNCLWU0cB6Cm+6pkBRcbiCkE4Pt31qb5ZPNLM4QtLXJZ1ClPHWIBZQL2/7nFzGa39aPkF1YPt2wvpnINGWYsUpbHoBsJGkWRsWXAE4ehq9yCTj3NLXyk7qtUDSLJIOJkZLBwN/ItbZHFPGotEOdgWusf1eveLsijR3eTlNMncpaUPCR+9pYGXXpgX9yoS/5ZgaHKvXJDXeBpKuIwQbHxOO9dvZvqMZStOZxtA2CQriYmJ7d8JX61pJh3UxgniPcP/etYgYG0y3o8UKknpVaFKDt2eBJamywVsB4ojOjAJGlCPGqReSBkn6A9H1dyfbP6nhep59aII5SUkzSPoe8ARwPNHuY6jtnzlcYzJ9jLZKUB3Yvpjo/zMc+KekBTttMgooS/3X4lwCrKauO/EC5SX1SlB9Grx9i/ARvqPauHrJ3YSidHgRJ5e0PLHgdjAxkhhTw2PPSPg3nlurY1YRw8KSTibKeMOJm5FlbZ9tu7O6N9OHaMsEBZAmTtcn7sLukbRnSUK6nXjvLbmAtVySs0RZxqcpqS9HCEi6SurdImlmSQcSNjmHEq3o57d9tHvvDF2oN1yJWKKhrvgp2R9KzNcd5fCNG1vj0+wE3FxBqbUmJJHH+pKuJYQe44iktK3t24sezWWag+aQmdcZSUsSqrZngH1sv5supkul0UPbovCuu5qY85lQxvb9CAnvz4FDgHO6u1hIWgz4AdEu/DrCG69m5pvNIn9OcvjniL5SPfWU6u259iTmDr9GGOLubvulOp3rQeCntm+ox/G7ON/0hJ3SDwhF7anA6DxSKoNQ3Xa1HOecnhTPLYuboK1vIx7EJPCJxFqbDQhD9bHA4KJja8B7vw/YsMJ9liS+/JcBs3Z6bSpgU+BGot/kkcBcdYr9+8CFRX+GKZbzgR/V+RwiRmvjgYOAfnU81/JE8q/bOUrOtRDwG6IlzGXESF1F/05b4gErGi43jDN8YnDJ45P0/OUOr8Hi463l96boABr+hmFtQt58KjFH872iY2rAex5B2ANVut+0REeUjqQ+GPgRIXq4l7AdmraOcSslybWL/gxTPGsSE/h1ubACsxD2Sq8AVzbg/fwJ+Hmdf3/rEiP4twnhw9Cif48t9YD9DB8bJnRKTJ0fE9J2+xUecw0ffaLE15lUNvojsArwKbEWp20/CIWj+UvAYg6rqEr334Mwau1PWE/91g3ooSNpZaI0+w03QZuUNIf5JOFaUlPBhqR1gbOJOdMngNXcy3YbPZxveqJ/2uKusV9dWrKwK1HGm0i0uBjtaAaZKRdpP+Akop9duXwCHEiyhmt12lYk0R2OVgI7AocRpYdT1QbN6aaE7Q+J0eIe5e6jyRu8HQecBlxPlP4a1XW0QxxReHKC+oglFF2gf0u0G9/L9o+Iead6swNwWy2Tk6QFJJ1IrL9bn7CvWsr2mTk5VUgs+TgJGHQhsVBtOqKx18rEH6MJy/oFgRmBuYEfwaDP4SSiB17rU/QQrugHcfF9HfgnbVx+IMxin6OH+QZgJuD/CDXe/cSd8LTpNRFlvbeAnwBT1THeGYH3gTmL/uw6xTUbMXc5cw2OtTRh03UJMEvJ83sCZ9f5fdwDbFyD44jos3Yl8A5wAqHeLPx31dKPmFOacCJ4DvAl4P+BJ4IfBO8EHg/+L/j9VOZ7F7wW+ESYaLis8PdQg0efHEF14mRCQHETcJ+kndt0fdT9hKHo2l29qDIavDn4C5HsNgJukTS0TvHuCNziBsufe8KhJLyeSNRVkUanBxHfuROA7dxAhwxJSxM33Nf34hjTSdqXsMw6hXCgH2r7ECeD4kyVhFpvww+g3xHEaGkbYAbibmBZou49LVH+GZx2M1ESezY22yiZcLc0fT5BOdbo3EKMCjYg5NWj0zxV22D7Kz6EmrzB2z+JEcuS7qHBm+0XiUR3HfVL6k3pDZeoeqG3pCFEYtoCWMnRGqLR858jgLNcxrKDzkiaX9KviTLehsSShCVtn+FYd5fpPbsDvouYIN+8h41HE+WG2QgPrH3jadMk9ly9oc8nqMQoYISjffjy32PNFgAAIABJREFUhOLoEUldjjZamL8A66e5gs4N3obaPtz2q+UcyPYE2ydQh6SenBNmJWTszcitxMT1SpXsJGkHon3HjcAw28/XIbaeYhhEjE7/XME+krSWpCuIkbiI5LqF7VsKSLDtzlLAwHeIpFNq67IaMWIaCNyWntuJKI38B9gPSK7GA9NxWpqcoIIbgdkkLWd7nO0DiLvM8yWdKGnaguOrFV8DXgMep0YN3uqU1Dsa5zWFOKIzKa4zKVMsIWmwpAuI9WIb2h5ZzeilRmwL3O0yFv4m/78RhNT/90RJcKjtg2w/V+c4+zKDIe7Q3mFyxcydxATorHy1FfnXib413+10nFYmJyhiNEBY84woee4fxCT2AkQZa8mCwusVqYy3oaS/ExZP9xKikB1t31mLu99aJvUkf96OkFw3M+cAWycvuykiaThRefkAWM72A/UPrVtGEBWDKSJpqKTjiTLeJoRoZgnbf8plvIYwFqInzrREm+Ry+YJYpFh6nFYmJ6hJnA1sr5K2E2lCfBtCzXmLpB+pydt/d6CuG7zNRyjExhOLTmtKp6R+b5VJfTvg9nJLjUVRMne5Y1evS5o2XeRHA/vb/p4LllonW6oFibnDzq9J0jBJlxHGtFMT7Tw2t31zLuM1lEeBcYOBXxAjokuBD4lR08NE/xGIYXxHG/EngJGEpJLwNny0UQHXjaJlhM30IFa87zWF1xYketPcBMxbdKzdvIevE6qq9whXgtXp5HwA/BD4Sx1jEJEI3ybuvsu20iGcwzcp+nMsM9YNgPu7eH5x4CFCej17Fceti8ycuNH6VafnBhKdlx8hFiHvD0xf9Gfbpx8wR7IvssEXgFcEDwTPBl4J/Cfwp+A9kgx9EHgo+CDwuNhvnKv47jXbo/AAmulB+Mvd1c3rUxOLe98kpMGFx5zi6pcultcRN1THAkO62X4WUim7znFVlNSJSd1XgKmL/kwr+NxfIEp3HT8fQEwd7N35xqCC49Y8QRFLKd4GFkw/z0fccL9F9Edbr9p486MOj7QOqgd7oyk9JuR1UO3J34EhUypN2f7C9i+JRPZLSedJmqmhEZagKhu8OdbcXEudmzY6JtKHEZ1aH5C0bQ+7dMifG+Gk0GscYomziGaGcxPfn52AVRzuCc1UFtuKGNXNK+nS9P8BhKXSprZvbLJ4+wxpnnhRSVtIOkTSP9aIUnm1Le3HEzcfrU/RGbLZHsDRwKllbDcd4ef3PLBGg2NcmFhg/C7hQrAGFd79Eonj8Ur360XMKxKy9nOBmbp4fVB6P0OL/g5U+L7mIaYH3iKmDHo9+qPGIyiijPdk+q4+RVgQzVD0Z5cfX/5+diDWLX1A6BwM3F5iFOsKHm1lGJtHUF/lLGBnSQO728j2x7b3J0o6F0saKWmaegVVhwZvtxEly1VrHWtXOLrpLkeYWT4saY1Om2wD3ONYBNwSJAXfL4nS3hm2j3ITjf4kDZF0LFE2XZBoXbKY7T84/BkzzcHlhA3ZjEQrm/HAToTh64HE30xPSy4m0mZGsZBVfF8hXSDvB06SdL+kbldj274GWAZYArhb0qK1jEfS9JK+S5Txfg1cAcxn+1D3ooldSmijKHGWqDclSf0HRFI/tiSp9yh/biYkfYsQVH1BrCdbp9iIgnQjs7qki4n4BhEXwFNsX+cmXVvWV0luJLsAMwOfp8ef3FGij2QzjBDcjCduTEsZl56/EhjWTskJyCW+0gdRkz+N+IV/QXxZDihzXxEX+7eJEkqvSmc0oMEbBTZtJBa8X0M4K2z8/+2deZicZZX2fycrTcCELRAEA4ZN2US2AQIBEdkEJiwCDjuJsgmKBCc6iiwOfgq44CAfEtEAQ8YAIgwB5wOJQwgS0AnBURARArJNhjWQBUzu74/7KbpS6e7auyrdz++6nivp6nd56q3q97znPPc5BycQD271d6CCeQ/G1TdeAg5Lrw3CXsq2DTpH1SG+9N09CUvEn8De0prAECzq2aLV1y6PlT6zdfHDwzzcJeAsHC5eu8t9YD3BeYKpgtvTv+f1BbVet9eo1RNop4Fd7GeTYRLwFnBMlcfYAifD3kWVXWZpQYM3nB/VkqaNRUZ9Ea5S0NYqsvTZPowLo25Q8ruK1i4rPE/FBgqvgV2SjNAM4ACKZP24csR9rb52eaz0ue0PPI+jIsXdAuqukt+XRg7xFSHpTbxO8kfsQQ2mMw+u0mP8CdgD38j+KyLGl9snVYY+Dbde+A42UKMlfUnNX5P5EVah9XoFd/mvcir2WEcBd0bEBr09j3KksNlpWDJ/HW5T8VLJZhWtXTZwPntExDRcTfx9WKhzkKS7tWIYb5UKnfZ1IqIjIq7En8nxkiZpxW4Br7V2hu1FNlAlyNUj/g57QUNwwmu1x3hX0tewtPeyiLg2lfBZgWiPBm+/wqGgVjU4G4/DfDvhtb+5EfH3LZrLSkREIRQ5ERuBq5JhXYH0IDEHiz2aNZfV0proI7jU0mzce+ns9GBUuv0HcXeGW5s1p0zlRMQO+Lu+LrC9pF+1eEptTzZQXZCMwzhgyi/hVSImEXE9EXekfydV0mtF0mwsoAh8490tPf3uGxG3YS9rOS7YerikmV3d/JqJOguf9ppYooRC19xio355d0a9N4mIQ7DQ4FFgN0mPl9nlGhrYbbdoHhtGxMX4QeZY4KvAlpK+n7z+7jgVuEE1FgLONIaIGBgRXwJ+CVwit7PJnlIltDrG2LYDdk7Z3IsFi0pyDRal12+Vm/qttD/OTTqy6OdjcVX8/8GhvM8Aw1r+Pj23UbgXVK/mxuCyTC8DQ0peXxOHzP6Mk157+3oMA67GeUNjq9hvMC7E+6E6z38yDiXuBtyEvfgfAFtVOZcXsKy85d+x/jqA0bjX2kxWsRy/dhjZg+oKrzfMxL3CVsOJjsV0pNcPA2am7Yt2jw9jkcSPImLzcIO3K/EaxjO41uN9apPK0JJexD2Ouix82kQmAFMlvVMyn4WSTgXOB26LiK9HxODemFBE7EJnlYXtJc2qdF9J7+LQ24Q6zj8UG6ZPAjfg1uybSjpL5T24Yg4G/iLpD7XOJVM7KVJyHI6S3Ansq1Uox69taLWF7I0BbIlDNQspJxuvM3sbS7dfxKG7d7HXdBmdNdCE+wItwOGgtlCu4e6oD/fi+YZgqfaWZbYbhRV+vwE2b+J8BuHQ2cvAUXUcZ0z6bIdWud8o4MJ0TR7DPcoG1jGPO4ETW/296o8D5zRNw5Vadmj1fFbl0V88qPOxx7KmpO93u1XEzsDlOLmxGlYHLp/v5M1HcI5P4Jvec1q5wdsNeI3rDOwhjKzyfM3gP4CRaSG3NzgUeFzSEz1tJHt3B+JrNjsiGq44jIgxuLLGXrjw6/RajyXpKbxmVVa9mc69a0TciG9m6wL74Py3v6rGpoYR8QEs9Kn5fWRqI9yw81H8oLOT3NAzUyP9xUCNxjeAckzGoZ1aWA1XER+O13MWYqn6lhGxbunGcuhlVyxpnxsRB9d43oagLpo2NpmK5c8yP2BFo15WpFKOFIY5BXtn/wbsr8b0obqGHkQnqVfUcRExB/eLegR72GdK+mMDzn8KcJNa3H+qP5E+08tw2sQESedIKq36kKmWVrtwzR5YRr0M59q8hXsh/RcOvT0HfF0SgpF/8Tb6MWgj0AjQD0FzQNuChoPOLArtXQfaHfR50NqgyfCuHOI7BRue17BnMrpoPgI2K5njXnht6ipg9RZeq43wgnxTxRu4oeH/AqvVsO8Q4JtYAHBQHXMoZPE/irvFNvL9DcVimNLPeQMc3n0Rh/AOoYswHnUUi8W13J7D62ct+R71t4GrQMzDFV+a2sKmv40+70FJ+hhudX6WpDXwDekEYAReSD495d2cKBsPHgKexI/Un8ftaO/BLtjPsCSnwEO4CufLwD/BuxfCt4AvY7n0ejh0dFOZOf4nLq+/JvC7iGhJTpKkv2Ihx6eafKqa5c+S3pH0j7gC9FURcVVEVBWSjYgD8PfgKWAXSb+vdh5l5rgUV22fkM63S0TcgB9a1scL5vtJukM1hvF64ADgBUmPNvi4mRJSm4wv4IfgK7Bq95UWT6tv0WoL2RsDK/ImdPO77wLfEVz/tA2U/lrkJa0Nmlb08+Gg7xR5UBuXCCZ2dPmSU4uOPwCX8hmdfl7JgyqZz9HY3n2FOhbJ67hWhwKzm3j8Qfgabd2AYw0HrsctJHaqYPsOrKZ8FtinyddxG1zn8CEsV/8iFZaxoT4P6rbi718eTft8N8LPrQ+QBFB5NH70eQ+qlLQofV9ELIiIN4DTcLhnRGGb9Yu27+ji57eKft645PgLXHbmexHxekS8jkNmgWumlUXSvwE7Ah8Dfh0Rm1b2zhrGDGB0RGzTpOMfDDwjqZI1wR6R9Iak43HYbEZEfCUiBna1bUR8FGfxr4PDX/fVe/5uzrN+RHwNh3aF5fubSbpcTU7ODDdN3As7/5kmERGfwt+l+4BxWlEAlWkg/c5A4UXp23FL9OE4ITPw027VlMrJ1vHa1mcljSgaHXJViYqQQ2374TWSORFxYm/VypP7Gf2Y5oklGl4bTtI0ujHqRVn8d+Ms/k83w1BExM4RMRV7cxvi0lVnYZlxo8N43XEyMF3SW2W3zFRNRAxPn/HFwCclfUNt1P+rL9IfDdSawKuSlqSkzE+n1+ctr73FcoHFh3jxe3JEbA3vfanLtTpfCUnLJV2B+wydh/snrVPn/CqlKYVPI2JjnIT6s0YeF0Dun1Nq1DfBT7kH4hDgvzbynBExJCKOjYgHsaR7HjBG0mnyutYtwI5pHk0lIgbgNa9rmn2u/ki4weZcnGT/UbkBZ6bJ9EcDdQZwUUQsBL5G583yp7GyQ1QtcSFMwm0ypkXEm7is0YG1HlDSPNwu/Vng0Yj4RJ1zrOScz2Dp8xENPvQpwDQ1Sf5cYtQvwX2R7sWihJqbO5aSwnhfxWtLE/HnPUbSZZLeKy4si0BuwKKQZvNx4DVJv+2Fc/Ub0kPIpThsepak09UmFWD6AyH1am3S9ibiVly+qBbDvRy4DanRN/X3iIh9cSmdW4DJamKeRUQcgatujGvQ8QbiG/ohaqLCLCLWAn4IbIeN7MeAUyT9RwOOvSNwNhaSTAeulPRYmX22pjPVoKJwUEScDOwl6eQq5nYzcI/6WkfVFhLujn0jbkY5QVJVrXcy9dMfPaieuJTaw3xL0v5NQ9K9WI4+Cng4IrZv4ulux0nGWzXoePsDLzXZOO2Lw2wvATtKOgG3Y58SEd+tJWQZEYMj4uiIeACHD/8bix4+U844ASQxyHzgoGrPXcUc18deY0NDmP2VlMB9Jk4RuRp3Ts7GqQVkA1WM48pfxLLwalgEfBHpkcZPakVSCOkYnKx6T0RMSusPjT5P3YVPS5hIk9ZHwn2SLse5R6dK+nzBu6zVqEfEehHxFez1nY5LYI2R9C1Vn+vSY2WJBnAicKt6br2RqYBww8wZOFdyD0nXKIeZWkY2UKU4RFIwUsvLbL2cTuPUa6EVmRvw2tQhwL2p/lqjuRY4IVXYrpmIGAXsjQtoNpSI2BY3CtwEy8dXCuVVY9QjYoeIuA74UzrmQZL2lnRrHYqt6cDuEbFRjft3S1J35q65DSDc/Xou/j6NVRdNIDO9SzZQXWFjMw4nPS4BStd6FqfXbwPG9aZxKiaJGfbBjdAeiYiGtsuQ9GdcWbveDrcNlz+nLP5zqTCLvyejnsJ4R0XE/cAvsFR8M0kTk0ilLtKi+k1YJNJo9sbfxYeacOx+QUSsERHX4q4D4yVdkCIImRaTRRLlcFHSE/Gi+wicLzUP+CnSglZOrZiUiHojrjN4hqSa8rq6OO7RwERJH69x/wG48eCn1KAQaPJEfooL9B5fbaJkEmxMwp7yPcBY4C/A94FfNCO3JSI+gtf1Ni2XF1WNSCIibsKVP65szExXYdwVoKu/1Z9097caEbvhaiQzgS9IWtg7k81URCPLUuTR2oHbflyJF+X3btAxC4VPx9S4/37YaDak7xWuE1goBTWoxmN8BCcjv4lvYjOAEb3w+TwMHFjBdhWVOsIVUF6nwhJKfXbU0P0adxwu9N8a3/L3kEeXI4f4+hCSFkn6HC7fdGNEfKve9SO58On11C6WmAjUvdBcbxZ/RAyKiCMj4j+BO3A94A/iqg9P4RyzfeqZYwU0WixxAnC7mlxCqa2poft1RGwOzMLtbnaQ9PPem3CmKlptIfNozsCV1H+OF33rKswKbIWfNAdXud9I3HJkeJ3n3xOr6X5Ila1AsJfxjzjR+X7gqK7eB06mfh5Xo6+qG24Vc1kzXY9RZbYr60HhpPI/Anu2+rvWslFD9+t3YOnn3KvtLNqkm3Ue3Y/sQfVR5Jj74TjkNzMizqlVji7pcaxqO6TKXU8Efi7pjVrOW08Wf0Rsnxa+nwS2wLkse0qari4WwCXdhUN/mwMPNaNYrry+MR0boHrZI/07qwHHWvVI3a+nweq7AsPw09CuuKmagG/jkvJr4gZk3wYGw5DvwiDBbyTlBfg2JxuoPozMFNz++xjg7lTxuhauoYoCsvXKn1MW/2+ArYGPSLqzgn0GRcThETETuBMLH7aQdIoqaL1dYtTvq8eo98CPgAkNOO5ngB/145vs5Mug4xysdnkJL0xejftfvION1FTsst4N/ADnOQxw08vJrZh0pkpa7cLl0TsD92H6Kv47PrKG/TtwF9xNKtx+b1yHsKowCg5dnQkswAau7P7A2sD5WBwyCwspqgpHdnHMMcCDuEzR+xv4OQQWjXy8h216DPEBa2FxxLqt/l61ZMDI12Dx6qCbqwjvfQ50VufPiwXrtfy95NHjyB5UP0HS3yRdjOvI/XNEXBcR76ti/8W4lM65EfHNiOjyCTQivpd+dzZViiNSQm8hi393ST16CBGxbURcg0UOHwYOlzRW0s9UZx6LpKfw2tcs3OX4yHqOV3RcYS+qHrHEccBdkv63EXNaBTlxNsRSrHyoBOEFyK1XfOnEhs8s01CygepnSHoI+CiOgsyNiLGV7BcRB+NmeJ/D7T+62+8ALN8dDxwWEXt0s13p8cdjz6KQxf9kN9sNjIi/j4hf4cjNs8CWkk5Sgyt5J6N+ETbql0bET6ox6j1wI/CJcN5OVeTKEQBs9yoMXReHBQrsjpOfOnARvWK+jsu+FC3+deB8qUwbkw1UP0TSW5I+C3wemB4Rl0TE4DK7XYrXnAEGYsPQFS/jHBNwNY7zejpoRKyZxAzfpocs/ohYKyLOw97S+XhNbFNJl6jJhTyTUd8BWEoVRr2H472BFZa1PMHvim+uM+uZw6pARIyOiPGpGG4xI9bB8ebiPIPZOO65DivWKPsBXou6Eyf1FR+nwVPONJhsoPoxkm7HyrWPALMjYsseNt8LS9YLxuOv3Wz3Qvp3CfDvWJzRJSmLvyBe2EHSg11ss01E/F8seNgOOErS7pKmSXqnh/k2lC6M+jciYkgdhyyIJartQTYRuFZSuTqRfYH9cL+2+RHxYkTcHRHnA6/vho3NL8oc4Me4AOO9QBeFEBtSbSXTPLKB6udIehnLx6cAsyLi9NRuYNuIuKpwA5VLJ43Ff+vgEGFXFMr4TMFrQkvhPcn41IjYINW+uxB7EZMkTVBRiZkUxjssIu7FIoXngQ9JOkEt7mSajPoOdBr1WtuRPIiN/V6V7pDCi4fjKvN9mogYhh2h5dgWbYBbthwJzBsBiy/A3UdvxolNy+lseQuOo34Zt7j+4MqnWIzLIGXamFyLL/MeyYO6EZc22hpXWThM0oyibQYCt+4NN94Hoympe7Yd6DEXWj2t5Njn4JYVj2Al26u4keCLRdushQuqnoVDhd8Hbu5NT6lSkuH+LK5s8TXg6moEIekY5wC7SPqHkte7rMUXrprwcUkNEWy0E2k9bg8sTBmLv39zcXHfwdigfAu4UE5Cnw+sdiPwPSwXHYYN0anAScCW2M0vDusdh6Xo2MP/AG1UTzOzMtlAZVYgrUXNBnZKLz0NbK5CgVMnSE7GlRfEiqVlFmPjcxdwKcnbSU/+zwHvS/vMAsYVbugR8WEsvjgGhwWvlDSnee+ycRQZ9ZexwX25in3XxqHLMfh++g6W2G8BrI9TeqZhT+t+/Ll8WdIvG/keeptk3D9IpzEaiz2k2fi7MQt4WNLiJIYZCxwr6Zaig7R19+tMg2i1zj2P9hr4prEcGxLhm+NnJRWXlllWJudkWdrutHTM7+LQn4pGoe3F/wNeBC4ANmj1+6/xmg0GLknv49Aq9lsP+B32Ppdj8UfptT8//f9t4A1gq1a/3xquz0CsHD0brym9gMO207C3vD0wsJt9twe2Xel3LhBbVZmjovG2YKdWX5c8yo/sQWVWICK2Br4AbAtshpNgF8mtKS7HFdMrZdEyOG8Q/Av2rJbgLH7wzfbPOEIzXW0YxquWJKm/HrfwOFc99L9KXuM8bLiH4OtxLBZBFDyD/8Ge1QKsqF6Or+NEuUJIWxIRqwO7YM9nT1zJ5HnsBRY8pGdU783HIc+qv5P0coPRTO1kA9UPiYiTgAmSysqlI6LjZtjvCDfcq+ZGUGDR9XDcCZacn0ln6spCYB31scZwKZz5PXxzPk6WqHe13QBsuE/A1/VtfI3exoYrcKv5KVjePhgb+N8D+6lB/b4aQUSsw4rrR9vh91AwRg+oWUnFnUZqNXoO9y3H1y8bp1WIbKD6IdUYqLRDXfH+ZfCLQfBJbJyW4bWWDtwbaZVeT+mOVHniX3Dt0m9I+lu4g68kPZe2CeBLwEU4DLaJpOci4re4Msb7JL0bEQuxEZuBZfZLWvCWKJrzaDqN0Z7A+3HdxFnYS5ojaVEvTmonvC56EN2vi87A66INaZqZ6R2ygeqHVGWgrK6aj59Qa2UJ8IFwbuUawHAsmPiTmtC9tl1IhXl/gt/rBHyTfBXnfKlou9OxMVtd0pIkvNhC0h3p969gWfqh6uX8p6Ta3IZOMcOe2JgWjNEsYF5bfI6rSPfrTBW0ehEsj+YN3AfpKRxO+wOpcyhW4T6Ak+zfAB4H9i3a7ySsLls4Al6dCksLC8xTQFuBRoA+AXqmaPEZ0A9Bm4GGg84ALe/sanoeXl/5Y9F8PprOtyFwC15reRo4u2guu2Bp+ptYKXdFq69rlZ/BACwOWIw9x7foQkgBDBGMFEwSXC+4I/07aQ5s2IvzXQ0boS9jg/o68ARwbfpejCH3Ucqjl0bLJ5BHEz9cN+fbMN0kj8brG6PSjeZvWAwxOP3uDSyIGJaMwZaSeAJu/n0yQLeBxoD+AHoXdDFotxIDdTDoNdB80Lqgu9LvroBf44XynXHIZTMcKhoA/BbnEg3B8uO/APun9/AgcHz6/xrA37X6utbwORyUDFRBnfccxe3qa2hZ3sC5rQUcjAsuzEoGdA5wBa6nOLLV1y+P/jtaPoE8evHDduLjYclAvVD8JJxuSscnA/U6cATQkZ7kJdABoGuLbp7LQB1FXhSg+4t+fxTo0vT/na1IO6eLOe0KPFvy2mRSuwlc9/NCVuHWEsAN6YHgTSx4EPDPkmqW7tcxl42BT+O1scewN3sPlvnvC6zR6uuVRx6FUVwMONPHiIgTgHOBTdJLa+AW6MuA5yUVL0DOx6GktyPiaBySm7ILvDUV93yfD5yD9eYFhN2i0ennDYp+tzp+HAd40d7RU11MczSwYUQUq9IG4vUNcGGAi4DHI+Jp4EJJ/17B228bJB0XEROx17gFLtmzoEqZ9IC03eVEQAVKtKQU/DCda0djsYCgsHZ0HTBXfUxJmek7ZAPVR4mI0bgg6b7Ag5KWRcRcHF4DeH9ERJGR+gBwO4CsrPtlRHSMgLtPhVEPwICNga8A/0DVLO7w+tGYLn73HPC0pM272lFuu3FsutkeDtwcEeuowtbv7YLcT+uxNG5JFTlmToPVv0NnqZ5N8Sr/6bhc+UU4k3ct4BkfqmCkHqFEkRYRQ4Ed6TRGe2BRxv3Ar7An+mTJg0km07bkYrF9l2HYwVkA79V326bo9yOBs1Ph1qOADwEzImL9VKh1GLD0dzBnYNrhNNxz47/Tz28A0yubS8h11M6LiB1TMdrNkhGdAyyMiC9FREcqFLtN+AZORBwXEevJ6rWCl9UXKnmXbVk+DBcm/PbK+64GTI6I4RFxYKqs/mvgFdyufkPgp8DWkjaTdLKkKZL+lI1TZlUie1B9FEl/iIjLschgOW6J80DRJg8Bm2Ppd6EN/Cvhrrbnpu31Csz9pp++PzYeBryFC+bNx1rx/bASo9t52EjOeFKakur8/SvOm3kGix/mR8QncajraVzb8wngn9IhDgCuSNUJ5gPHJG9k1SVi5Otw4AUQU/FiX4EdcGE/sHxxF7xAVMKApTB+Pdh/ATyMPaRvAL+R9GZT557J9CI5DypTnhSOosZKEsC40nBUvyZi0gy4+FAYuoTyT4n34CSqZ4peWwZL/wYXDJX+T7Ommcm0mhziy5THVcm/iI1NNRTqnmXjtCJVtywvZSAMHeqWFJlMnyWH+DKVIV2Nexfmumf1s0LL8sIf4ez070ZUvMiWW5Zn+jTZg8pUjo3NOOA2bIBK14IWp9dvw2G9bJy6puKW5eWO04jJZDLtSvagMtXhcN0Rue5ZXRRalnecgVUk+2PV3jw6W5Yvx2q+d9M2S/ATZepXkluWZ/o8WSSRyfQ2RQV4e2pZPhvYp2TXcVitQm5ZnukHZAOVybSC3LI8kylLXoPKZFrDpdgLqoUlaf9Mpk+TDVQm0wqydD+TKUsWSWQyrSJL9zOZHslrUJlMq8ktyzOZLskGKpNpF7J0P5NZgWygMplMJtOWZJFEJpPJZNqSbKAymUwm05ZkA5XJZDKZtiQbqEwmk8m0JdlAZTKZTKYtyQYqk8lkMm1JNlCZTCaTaUuygcpkMplMW5INVCaTyWTakmygMplMJtOWZAOVyWQymbYkG6hMJpPJtCXZQGUymUymLckGKpPJZDIcRqZcAAAAMklEQVRtSTZQmUwmk2lLsoHKZDKZTFuSDVQmk8lk2pJsoDKZTCbTlmQDlclkMpm25P8DwQR4OJKs6XAAAAAASUVORK5CYII=\n", + "text/plain": [ + "<Figure size 432x288 with 1 Axes>" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "sm = sm.get_largest_subgraph()\n", + "\n", + "_, _, _ = plot_structure(sm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After deciding on how the final structure model should look, we can instantiate a `BayesianNetwork`." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "from causalnex.network import BayesianNetwork\n", + "\n", + "bn = BayesianNetwork(sm)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are now ready to move on to learning the conditional probability distribution of different features in the `BayesianNetwork`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Fitting the Conditional Distribution of the Bayesian Network" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Preparing the Discretised Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Bayesian Networks in CausalNex support only discrete distributions. Any continuous features, or features with a large number of categories, should be discretised prior to fitting the Bayesian Network. Models containing variables with many possible values will typically be badly fit, and exhibit poor performance.\n", + "\n", + "For example, consider P(G2 | G1), where G1 and G2 have possible values 0 to 20. The discrete conditional probability distribution is therefore specified using 21x21 (441) possible combinations - most of which we will be unlikely to observe.\n", + "\n", + "CausalNex provides a few helper methods to make discretisation easier. Let's start by reducing the number of categories in some of the categorical features by combining similar values. We will make numeric features categorical by discretisation, and then give the buckets meaningful labels." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cardinality of Categorical Features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To reduce the cardinality of categorical features we can define a map `{old_value: new_value}`, and use this to update the feature. For example, in the `studytime` feature, we make the studytime which is more than 2 (2 means 2 to 5 hours here, see https://archive.ics.uci.edu/ml/datasets/Student+Performance) into `long-studytime`, and the rest into `short-studytime`." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "discretised_data = data.copy()\n", + "\n", + "data_vals = {col: data[col].unique() for col in data.columns}\n", + "\n", + "failures_map = {v: 'no-failure' if v == [0]\n", + " else 'have-failure' for v in data_vals['failures']}\n", + "\n", + "studytime_map = {v: 'short-studytime' if v in [1,2]\n", + " else 'long-studytime' for v in data_vals['studytime']}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once we have defined our maps `{old_value: new_value}` we can update each feature, applying the mapping transformation." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "discretised_data[\"failures\"] = discretised_data[\"failures\"].map(failures_map)\n", + "discretised_data[\"studytime\"] = discretised_data[\"studytime\"].map(studytime_map)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Discretising Numeric Features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make numeric features categorical, they must first be discretised. CausalNex provides a helper class `causalnex.discretiser.Discretiser`, which supports several discretisation methods. For our data the `fixed` method will be applied, providing static values that define the bucket boundaries. For example, `absences` will be discretised into the buckets < 1, 1 to 9, and >=10. Each bucket will be labelled as an integer from zero." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "from causalnex.discretiser import Discretiser\n", + "\n", + "discretised_data[\"absences\"] = Discretiser(method=\"fixed\", \n", + " numeric_split_points=[1, 10]).transform(discretised_data[\"absences\"].values)\n", + "\n", + "discretised_data[\"G1\"] = Discretiser(method=\"fixed\", \n", + " numeric_split_points=[10]).transform(discretised_data[\"G1\"].values)\n", + "\n", + "discretised_data[\"G2\"] = Discretiser(method=\"fixed\", \n", + " numeric_split_points=[10]).transform(discretised_data[\"G2\"].values)\n", + "\n", + "discretised_data[\"G3\"] = Discretiser(method=\"fixed\", \n", + " numeric_split_points=[10]).transform(discretised_data[\"G3\"].values)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create Labels for Numeric Features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To make the discretised categories more readable, we can map the category labels onto something more meaningful in the same way that we mapped category feature values." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "absences_map = {0: \"No-absence\", 1: \"Low-absence\", 2: \"High-absence\"}\n", + "\n", + "G1_map = {0: \"Fail\", 1: \"Pass\"}\n", + "G2_map = {0: \"Fail\", 1: \"Pass\"}\n", + "G3_map = {0: \"Fail\", 1: \"Pass\"}\n", + "\n", + "discretised_data[\"absences\"] = discretised_data[\"absences\"].map(absences_map)\n", + "discretised_data[\"G1\"] = discretised_data[\"G1\"].map(G1_map)\n", + "discretised_data[\"G2\"] = discretised_data[\"G2\"].map(G2_map)\n", + "discretised_data[\"G3\"] = discretised_data[\"G3\"].map(G3_map)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Train / Test Split" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Like many other machine learning models, we will use a train and test split to help us validate our findings." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "# Split 90% train and 10% test\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "train, test = train_test_split(discretised_data, train_size=0.9, test_size=0.1, random_state=7)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model Probability" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With the learnt structure model from earlier and the discretised data, we can now fit the probability distrbution of the Bayesian Network. The first step in this is specifying all of the states that each node can take. This can be done either from data, or providing a dictionary of node values. We use the full dataset here to avoid cases where states in our test set do not exist in the training set. For real-world applications, these states may need to be provided using the dictionary method." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "bn = bn.fit_node_states(discretised_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Fit Conditional Probability Distributions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `fit_cpds` method of `BayesianNetwork` accepts a dataset to learn the conditional probablilty distributions (CPDs) of each node, along with a method of how to do this fit." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/ben_horsburgh/opt/anaconda3/envs/causal-test/lib/python3.7/site-packages/pandas/core/generic.py:5069: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.\n", + " object.__getattribute__(self, name)\n", + "/Users/ben_horsburgh/opt/anaconda3/envs/causal-test/lib/python3.7/site-packages/pandas/core/generic.py:5070: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.\n", + " return object.__setattr__(self, name, value)\n" + ] + } + ], + "source": [ + "bn = bn.fit_cpds(train, method=\"BayesianEstimator\", bayes_prior=\"K2\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once we have the the CPDs, we can inspect them through the `cpds` property, which is a dictionary of node->cpd." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead tr th {\n", + " text-align: left;\n", + " }\n", + "\n", + " .dataframe thead tr:last-of-type th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr>\n", + " <th>failures</th>\n", + " <th colspan=\"8\" halign=\"left\">have-failure</th>\n", + " <th colspan=\"8\" halign=\"left\">no-failure</th>\n", + " </tr>\n", + " <tr>\n", + " <th>higher</th>\n", + " <th colspan=\"4\" halign=\"left\">no</th>\n", + " <th colspan=\"4\" halign=\"left\">yes</th>\n", + " <th colspan=\"4\" halign=\"left\">no</th>\n", + " <th colspan=\"4\" halign=\"left\">yes</th>\n", + " </tr>\n", + " <tr>\n", + " <th>schoolsup</th>\n", + " <th colspan=\"2\" halign=\"left\">no</th>\n", + " <th colspan=\"2\" halign=\"left\">yes</th>\n", + " <th colspan=\"2\" halign=\"left\">no</th>\n", + " <th colspan=\"2\" halign=\"left\">yes</th>\n", + " <th colspan=\"2\" halign=\"left\">no</th>\n", + " <th colspan=\"2\" halign=\"left\">yes</th>\n", + " <th colspan=\"2\" halign=\"left\">no</th>\n", + " <th colspan=\"2\" halign=\"left\">yes</th>\n", + " </tr>\n", + " <tr>\n", + " <th>studytime</th>\n", + " <th>long-studytime</th>\n", + " <th>short-studytime</th>\n", + " <th>long-studytime</th>\n", + " <th>short-studytime</th>\n", + " <th>long-studytime</th>\n", + " <th>short-studytime</th>\n", + " <th>long-studytime</th>\n", + " <th>short-studytime</th>\n", + " <th>long-studytime</th>\n", + " <th>short-studytime</th>\n", + " <th>long-studytime</th>\n", + " <th>short-studytime</th>\n", + " <th>long-studytime</th>\n", + " <th>short-studytime</th>\n", + " <th>long-studytime</th>\n", + " <th>short-studytime</th>\n", + " </tr>\n", + " <tr>\n", + " <th>G1</th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " <th></th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>Fail</th>\n", + " <td>0.75</td>\n", + " <td>0.806452</td>\n", + " <td>0.5</td>\n", + " <td>0.75</td>\n", + " <td>0.5</td>\n", + " <td>0.612245</td>\n", + " <td>0.5</td>\n", + " <td>0.75</td>\n", + " <td>0.5</td>\n", + " <td>0.612903</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.032967</td>\n", + " <td>0.15016</td>\n", + " <td>0.111111</td>\n", + " <td>0.255814</td>\n", + " </tr>\n", + " <tr>\n", + " <th>Pass</th>\n", + " <td>0.25</td>\n", + " <td>0.193548</td>\n", + " <td>0.5</td>\n", + " <td>0.25</td>\n", + " <td>0.5</td>\n", + " <td>0.387755</td>\n", + " <td>0.5</td>\n", + " <td>0.25</td>\n", + " <td>0.5</td>\n", + " <td>0.387097</td>\n", + " <td>0.5</td>\n", + " <td>0.5</td>\n", + " <td>0.967033</td>\n", + " <td>0.84984</td>\n", + " <td>0.888889</td>\n", + " <td>0.744186</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + "failures have-failure \\\n", + "higher no \n", + "schoolsup no yes \n", + "studytime long-studytime short-studytime long-studytime short-studytime \n", + "G1 \n", + "Fail 0.75 0.806452 0.5 0.75 \n", + "Pass 0.25 0.193548 0.5 0.25 \n", + "\n", + "failures \\\n", + "higher yes \n", + "schoolsup no yes \n", + "studytime long-studytime short-studytime long-studytime short-studytime \n", + "G1 \n", + "Fail 0.5 0.612245 0.5 0.75 \n", + "Pass 0.5 0.387755 0.5 0.25 \n", + "\n", + "failures no-failure \\\n", + "higher no \n", + "schoolsup no yes \n", + "studytime long-studytime short-studytime long-studytime short-studytime \n", + "G1 \n", + "Fail 0.5 0.612903 0.5 0.5 \n", + "Pass 0.5 0.387097 0.5 0.5 \n", + "\n", + "failures \n", + "higher yes \n", + "schoolsup no yes \n", + "studytime long-studytime short-studytime long-studytime short-studytime \n", + "G1 \n", + "Fail 0.032967 0.15016 0.111111 0.255814 \n", + "Pass 0.967033 0.84984 0.888889 0.744186 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bn.cpds[\"G1\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The CPD dictionaries are multi-indexed, and so the `loc` function can be a useful way to interact with them:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Predict the State given the Input Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `predict` method of `BayesianNetwork` allows us to make predictions based on the data using the learnt Bayesian Network. For example, we want to predict if a student fails or passes their exam based on the input data. Imagine we have an incoming student data that looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "address U\n", + "famsize GT3\n", + "Pstatus T\n", + "Medu 3\n", + "Fedu 2\n", + "traveltime 1\n", + "studytime short-studytime\n", + "failures have-failure\n", + "schoolsup no\n", + "famsup yes\n", + "paid yes\n", + "activities yes\n", + "nursery yes\n", + "higher yes\n", + "internet yes\n", + "romantic no\n", + "famrel 5\n", + "freetime 5\n", + "goout 5\n", + "Dalc 2\n", + "Walc 4\n", + "health 5\n", + "absences Low-absence\n", + "G2 Fail\n", + "G3 Fail\n", + "Name: 18, dtype: object" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "discretised_data.loc[18, discretised_data.columns != 'G1']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Based on these data, we want to predict if this student fails their exam. Intuitively, we would expect this student to fail because they spend a shorter amount of time on their study and have failed in the past. Let's see how our Bayesian Network performs on this:" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "predictions = bn.predict(discretised_data, \"G1\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The prediction is 'Fail'\n" + ] + } + ], + "source": [ + "print('The prediction is \\'{prediction}\\''.format(prediction=predictions.loc[18, 'G1_prediction']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The prediction by the Bayesian Network turns out to be a `Fail`. Let's compare this to the ground truth:" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The ground truth is 'Fail'\n" + ] + } + ], + "source": [ + "print('The ground truth is \\'{truth}\\''.format(truth=discretised_data.loc[18, 'G1']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which turns out to be the same." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model Quality" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To evaluate the quality of the model that has been learned, CausalNex supports two main approaches: Classification Report and Reciever Operating Characteristics (ROC) / Area Under the ROC Curve (AUC). In this section each will be discussed." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Classification Report" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To obtain a classification report using a BN, we need to provide a test set, and the node we are trying to classify. The report will predict the target node for all rows in the test set, and evaluate how well those predictions are made." + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>precision</th>\n", + " <th>recall</th>\n", + " <th>f1-score</th>\n", + " <th>support</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>G1_Fail</th>\n", + " <td>0.777778</td>\n", + " <td>0.583333</td>\n", + " <td>0.666667</td>\n", + " <td>12</td>\n", + " </tr>\n", + " <tr>\n", + " <th>G1_Pass</th>\n", + " <td>0.910714</td>\n", + " <td>0.962264</td>\n", + " <td>0.935780</td>\n", + " <td>53</td>\n", + " </tr>\n", + " <tr>\n", + " <th>macro avg</th>\n", + " <td>0.844246</td>\n", + " <td>0.772799</td>\n", + " <td>0.801223</td>\n", + " <td>65</td>\n", + " </tr>\n", + " <tr>\n", + " <th>micro avg</th>\n", + " <td>0.892308</td>\n", + " <td>0.892308</td>\n", + " <td>0.892308</td>\n", + " <td>65</td>\n", + " </tr>\n", + " <tr>\n", + " <th>weighted avg</th>\n", + " <td>0.886172</td>\n", + " <td>0.892308</td>\n", + " <td>0.886097</td>\n", + " <td>65</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " precision recall f1-score support\n", + "G1_Fail 0.777778 0.583333 0.666667 12\n", + "G1_Pass 0.910714 0.962264 0.935780 53\n", + "macro avg 0.844246 0.772799 0.801223 65\n", + "micro avg 0.892308 0.892308 0.892308 65\n", + "weighted avg 0.886172 0.892308 0.886097 65" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from causalnex.evaluation import classification_report\n", + "classification_report(bn, test, \"G1\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This report shows that the model we have defined is able to classify whether a student passes their exam reasonably well.\n", + "\n", + "For the predictions where the student fails, the precision is good, but recall is bad. This implies that we can rely on predictions for this class when they are made, but we are likely to miss some of the predictions we should have made. Perhaps these missing predictions are as a result of something missing in our structure - this could be an interesting area to explore." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### ROC / AUC" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Reciever Operating Characteristics (ROC), and the Area Under the ROC Curve (AUC) can be obtained using the `roc_auc` method within the CausalNex metrics module. Again, a test set and target node must be provided. The ROC curve is computed by micro-averaging predictions made across all states (classes) of the target node." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.9181065088757396\n" + ] + } + ], + "source": [ + "from causalnex.evaluation import roc_auc\n", + "roc, auc = roc_auc(bn, test, \"G1\")\n", + "print(auc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The AUC value for our model is high, giving us confidence in the performance." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Querying Marginals" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After iterating over our model structure, CPDs, and validating our model quality, we can query our model under defferent observation to gain insights." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Baseline Marginals" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To query the model for baseline marginals that reflect the population as a whole, a `query` method can be used. First let's update our model using the complete dataset, since the one we currently have was only built from training data." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Replacing existing CPD for address\n", + "WARNING:root:Replacing existing CPD for absences\n", + "WARNING:root:Replacing existing CPD for Pstatus\n", + "WARNING:root:Replacing existing CPD for famrel\n", + "WARNING:root:Replacing existing CPD for studytime\n", + "WARNING:root:Replacing existing CPD for G1\n", + "WARNING:root:Replacing existing CPD for failures\n", + "WARNING:root:Replacing existing CPD for schoolsup\n", + "WARNING:root:Replacing existing CPD for paid\n", + "WARNING:root:Replacing existing CPD for higher\n", + "WARNING:root:Replacing existing CPD for internet\n", + "WARNING:root:Replacing existing CPD for G2\n", + "WARNING:root:Replacing existing CPD for G3\n" + ] + } + ], + "source": [ + "bn = bn.fit_cpds(discretised_data, method=\"BayesianEstimator\", bayes_prior=\"K2\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can safely ignore these warnings, which let us know we are replacing the previously existing CPDs. \n", + "\n", + "For inference, we must create a new InferenceEngine from our BayesianNetwork, which lets us query the model. The query method will compute the marginal likelihood of all states for all nodes." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Fail': 0.25260687281677224, 'Pass': 0.7473931271832277}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from causalnex.inference import InferenceEngine\n", + "\n", + "ie = InferenceEngine(bn)\n", + "marginals = ie.query()\n", + "marginals[\"G1\"] " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output observed tells us that `P(G1=Fail) = 0.25`, and the `P(G1=Pass) = 0.75`. As a quick sanity check, we can compute what proportion of our dataset are `Fail`, which should be approximately the same." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('Fail', 157), ('Pass', 492)]" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy as np\n", + "labels, counts = np.unique(discretised_data[\"G1\"], return_counts=True)\n", + "list(zip(labels, counts))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The proportion of the students who fail is `157 / (157+492) = 0.242` - which is close to our computed marginal likelihood." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Marginals after Observations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can also query the marginal likelihood of states in our network given some observations. These observations can be made anywhere in the network, and their impact will be propagated through to the node of interest.\n", + "\n", + "Let's look at the difference in the likelihood of `G1` based on `studytime`." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Marginal G1 | Short Studtyime {'Fail': 0.2776556433482524, 'Pass': 0.7223443566517477}\n", + "Marginal G1 | Long Studytime {'Fail': 0.15504850337837614, 'Pass': 0.8449514966216239}\n" + ] + } + ], + "source": [ + "marginals_short = ie.query({\"studytime\": \"short-studytime\"})\n", + "marginals_long = ie.query({\"studytime\": \"long-studytime\"})\n", + "print(\"Marginal G1 | Short Studtyime\", marginals_short[\"G1\"])\n", + "print(\"Marginal G1 | Long Studytime\", marginals_long[\"G1\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Based on our data we can see that students who study longer are more likely to pass their exam." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Do Calculus" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "CausalNex also supports simple Do-Calculus, allowing as to specify interventions. In this section we will take a look at the supported methods, and what they mean." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Updating a Node Distribution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can apply an intervention to any node in our data, updating its distribution using a `do` operator. This can be thought of as asking our model \"What if\" something were different. For example, we could ask what would happen if 100% of students wanted to go on to do higher education." + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "distribution before do {'no': 0.10752688172043011, 'yes': 0.8924731182795698}\n", + "distribution after do {'no': 0.0, 'yes': 0.9999999999999998}\n" + ] + } + ], + "source": [ + "print(\"distribution before do\", ie.query()[\"higher\"])\n", + "ie.do_intervention(\"higher\", \n", + " {'yes': 1.0, \n", + " 'no': 0.0})\n", + "print(\"distribution after do\", ie.query()[\"higher\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Resetting a Node Distribution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can reset any interventions that we make by using the `reset_intervention` method, and providing the node that we want to reset." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "ie.reset_do(\"higher\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Effect of Do on Marginals" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can again use `query` to examine the effect that an intervention has on our marginal likelihoods. In this case, we can look at how the likelihood of achieving a pass changes if 100% of students wanted to do higher education." + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "marginal G1 {'Fail': 0.25260687281677224, 'Pass': 0.7473931271832277}\n", + "updated marginal G1 {'Fail': 0.20682952942551894, 'Pass': 0.7931704705744809}\n" + ] + } + ], + "source": [ + "print(\"marginal G1\", ie.query()[\"G1\"])\n", + "ie.do_intervention(\"higher\", \n", + " {'yes': 1.0, \n", + " 'no': 0.0})\n", + "print(\"updated marginal G1\", ie.query()[\"G1\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this case, we can see that if 100% of students wanted to do higher education (as opposed to 90% in our data population), then we estimate that pass rate would increase from 74.7% to 79.3%." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:causal-test] *", + "language": "python", + "name": "conda-env-causal-test-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/source/04_user_guide/04_user_guide.md b/docs/source/04_user_guide/04_user_guide.md new file mode 100644 index 0000000..347434e --- /dev/null +++ b/docs/source/04_user_guide/04_user_guide.md @@ -0,0 +1,304 @@ +# Causal Inference with Bayesian Networks. Main Concepts and Methods + +## 1. Causality + +### 1.1 Why is causality important? + +Experts and practitioners in various domains are commonly interested in discovering causal relationships to answer questions like + +> "What drives economical prosperity?", "What fraction of patients can a given drug save?", +"How much would a power failure cost to a given manufacturing plant?". + +The ability to identify truly causal relationships is fundamental to developing impactful interventions in medicine, policy, business, and other domains. + +Often, in the absence of randomised control trials, there is a need for causal inference purely from observational data. +However, in this case the commonly known fact that + +> correlation does not imply causation + +comes to life. Therefore, it is crucial to distinguish between events that _cause_ specific outcomes and those that merely _correlate_. +One possible explanation for correlation between variables where neither causes the other is the presence of _confounding_ variables +that influence both the target and a driver of that target. Unobserved confounding variables are severe +threats when doing causal inference on observational data. +The research community has made significant contributions to develop methods and techniques for this type of analysis. +[Potential outcomes framework (Rubin causal model)](https://5harad.com/mse331/papers/rubin_causal_inference.pdf), +[propensity score matching](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3144483/) and +[structural causal models](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2836213/) are, arguably, the most popular frameworks for observational causal inference. + +Here, we focus on the structural causal models and one particular type, Bayesian Networks. + +Interested users can find more details in the references below. +- [Causal inference using potential outcomes: Design, +modeling, decisions. Journal of the American Statistical Association](https://5harad.com/mse331/papers/rubin_causal_inference.pdf) by D. Rubin; +- [Lecture notes on potential outcomes approach](http://statweb.stanford.edu/~rag/stat209/jorogosa06.pdf), Dept of Psychiatry & Behavioral Sciences, Stanford University by Booil Jo; +- [Probabilistic graphical models: principles and techniques](https://mitpress.mit.edu/books/probabilistic-graphical-models) by D. Koller and N. Friedman. + +### 1.2 Structural Causal Models (SCMs) + +*Structural causal models* represent causal dependencies using graphical models that provide an intuitive visualisation by +representing variables as nodes and relationships between variables as edges in a graph. + +SCMs serve as a comprehensive framework unifying graphical models, structural equations, and counterfactual +and interventional logic. + +Graphical models serve as a language for structuring and visualising knowledge about the world and can incorporate both data-driven and human inputs. + +Counterfactuals enable the articulation of something there is a desire to know, and structural equations serve to tie the two together. + +SCMs had a transformative impact on multiple data-intensive disciplines (e.g. epidemiology, economics, etc.), enabling the codification of the existing knowledge in diagrammatic and algebraic forms and consequently leveraging data to estimate the answers to interventional and counterfacutal +questions. + +Bayesian Networks are one of the most widely used SCMs and are at the core of this library. + +More on SCMs: [Causality: Models, Reasoning, and Inference](http://bayes.cs.ucla.edu/BOOK-2K/) by J. Pearl. + + +## 2. Bayesian Networks (BNs) + +### 2.1 Directed Acyclic Graph (DAG) +A *graph* is a collection of *nodes* and *edges*, where the *nodes* are some objects, and *edges* between them represent some connection between these objects. +A *directed graph*, is a graph in which each edge is orientated from one node to another node. +In a directed graph, an edge goes from a *parent* node to a *child* node. +A *path* in a directed graph is a sequence of edges such that the ending node of each edge is the starting node of the next edge in the sequence. +A *cycle* is a path in which the starting node of its first edge equals the ending node of its last edge. +A *directed acyclic graph* is a directed graph that has no cycles. + +<figure> + <img src="graph.png" width="210"/> + <figcaption>Figure 1: A simple directed acyclic graph.</figcaption> +</figure> + + +<figure> + <img src="graph_definitions.png" width="350"/> + <figcaption>Figure 2: A more complex graph with a cycle and an isolated node. + This graph can be turned into a DAG by removing one of the edges forming a cycle: (F, G), (E, F) or (G, E).</figcaption> +</figure> + +### 2.2 What Bayesian Networks are and are not +**What are Bayesian Networks?** + +*Bayesian Networks* are probabilistic graphical models that represent the dependency structure of a set of variables and their joint distribution efficiently in a factorised way. + +Bayesian Network consists of a DAG, a causal graph where nodes represents random variables and edges represent the the relationship between them, and a conditional probability distribution (CPDs) associated with +each of the random variables. + +If a random variable has parents in the BN then the CPD represents \\(P(\text{variable|parents}) \\) i.e. the +probability of that variable given its parents. In the case, when +the random variable has no parents it simply represents \\(P(\text{variable}) \\) i.e. the probability of that variable. + +Even though we are interested in the joint distribution of the variables in the graph, Bayes' rule requires to only specify the conditional distributions of each variable given its parents. + +> The links between variables in BNs encode dependency not necessarily causality. In this package we are mostly interested in the case where BNs are causal. Hence, the edge between nodes should be seen as *cause -> effect* relationship. + +Let's consider an example of a simple Bayesian network shown in figure below. It shows how the actions of customer relationship managers (emails sent and meetings held) affect the bank's income. + +<figure> + <img src="BN.png" width="700"/> + <figcaption>Figure 3: A Bayesian Network describing a banking case study. Tables attributed to the nodes show the CPDs of the corresponding variables given their parents (if present).</figcaption> +</figure> + +New sales and the number of meetings with a customer directly affect the bank's income. However, these +two drivers are not independent but the number of meetings also influences +whether a new sale takes place. In addition, system prompts indirectly influence +the bank's income through the generation of new sales. This example +shows that BNs are able to capture complex relationships between variables and represent dependencies between +drivers and include drivers that do not affect the target directly. + + +**Steps for working with a Bayesian Network** + +BN models are built in a multi-step process before they can be used for analysis. + +1. **Structure Learning**. The structure of a network describing the relationships between variables can be learned from data, or built from expert knowledge. +2. **Structure Review**. Each relationship should be validated, so that it can be asserted to be causal. This may involve flipping / removing / adding learned edges, or confirming expert knowledge from trusted literature or empirical beliefs. +3. **Likelihood Estimation**. The conditional probability distribution of each variable given its parents can be learned from data. +4. **Prediction & Inference**. The given structure and likelihoods can be used to make predictions, or perform observational and counterfactual inference. +CausalNex supports structure learning from continuous data, and expert opinion. CausalNex supports likelihood estimation and prediction/inference from discrete data. A `Discretiser` class is provided to help discretising continuous data in a meaningful way. + + +> Since BNs themselves are not inherently causal models, the structure learning algorithms on their own merely learn that there are dependencies between variables. A useful approach to the problem is to fi rst group the features into themes and constrain the search space to respect how themes of variables relate. If there is further domain knowledge available, it can be used as additional constraints before learning a graph algorithmically. + +**What can we use Bayesian Networks for?** + +The probabilities of variables in Bayesian Networks update as observations are added to the model. +This is useful for inference or counterfactuals, and for predictive analytics. +Metrics can help us understand the strength of relationships between variables. + +- The sensitivity of nodes to changes in observations of other events can be used to assess what changes could lead to what effects; +- The active trail of a target node identifies which other variables have any effect on the target. + +### 2.3 Advantages and Drawbacks of Bayesian Networks + +**Advantages** + +- Bayesian Networks offer a graphical representation that is reasonably interpretable and easily explainable; +- Relationships captured between variables in a Bayesian Network are more complex yet hopefully more informative than a conventional model; +- Models can reflect both statistically significant information (learned from the data) and domain expertise simultaneously; +- Multiple metrics can used to measure the significance of relationships and help identify the effect of specific actions; +- Offer a mechanism of suggesting counterfactual actions and combine actions without aggressive independence assumptions. + +**Drawbacks** + +- Granularity of modelling may have to be lower. However, this may either not be necessary, or can be run in tangent to other techniques that provide accuracy +but are less interpretable; +- Computational complexity is higher. However, this can be offset with careful feature selection and a less granular discretisation policy, but at the expense of predictive power; +- This is (unfortunately) not a way of fully automating Causal Inference. + +## 3. `BayesianNetwork` + +The `BayesianNetwork` class is the central class for the causal inference analysis in the package. +It is built on top of the `StructureModel`, which is an extension of `networkx.DiGraph` + +`StructureModel` represents a causal graph, a DAG of the respective BN and holds directed edges, describing +a _cause -> effect_ relationship. In order to define the `BayesianNetwork`, users should provide a relevant `StructureModel`. + +> Cycles are permitted within a `StructureModel`. However, only **acyclic connected** `StructureModel` are allowed in the construction of `BayesianNetwork`; isolated nodes are not allowed. + +### 3.1 Defining the DAG with `StructureModel` + +Our package enables a _hybrid way_ to learn structure of the model. + +For instance, users can define a causal model **fully manually**, e.g., using the domain expertise: + +```python + from causalnex.structure import StructureModel + # Encoding the causal graph suggested by an expert + # d + # ↙ ↓ ↘ + # a ← b → c + # ↑ ↗ + # e + sm_manual = StructureModel() + sm_manual.add_edges_from( + [ + ("b", "a", origin="expert"), + ("b", "c", origin="expert"), + ("d", "a", origin="expert"), + ("d", "c", origin="expert"), + ("d", "b", origin="expert"), + ("e", "c", origin="expert"), + ("e", "b", origin="expert"), + ] + ) +``` +Or, users can learn the network structure **automatically** from the data using the [`NOTEARS`](https://papers.nips.cc/paper/8157-dags-with-no-tears-continuous-optimization-for-structure-learning.pdf) algorithm. Moreover, if there is domain knowledge available, +it can be used as **additional constraints** before learning a graph algorithmically. + +> Recently published [NOTEARS](https://papers.nips.cc/paper/8157-dags-with-no-tears-continuous-optimization-for-structure-learning.pdf) algorithm for learning DAGs from data based on a continuous optimisation problem +allowed to overcome the challenges of combinatorial optimisation giving a new impulse to the usage of BNs in the machine learning applications. + +```python + from causalnex.structure.notears import from_pandas + from causalnex.network import BayesianNetwork + + # Unconstrained learning of the structure from data + sm = from_pandas(data) + # Imposing edges that are not allowed in the causal model + sm_with_tabu_edges = from_pandas(data, tabu_edges=[("e", "a")]) + # Imposing parent nodes that are not allowed in the causal model + sm_with_tabu_parents = from_pandas(data, tabu_parent_nodes=["a", "c"]) + # Imposing child nodes that are not allowed in the causal model + sm_with_tabu_parents = from_pandas(data, tabu_child_nodes=["d", "e"]) +``` + +Finally, the output of the algorithm should be **inspected** and **adjusted**, if required, +by a domain expert. This is a targeted effort to encode important domain knowledge in models, and avoid spurious relationships. + +```python + # Removing the learned edge from the model + sm.remove_edge("a", "c") + # Changing the direction of the learned edge + sm.remove_edge("c", "d") + sm.add_edge("d", "c", origin="learned") + # Adding the edge that was not learned by the algorithm + sm.add_edge("a", "e", origin="expert") +``` + +> When defining the structure model, we recommend to use the **entire** dataset **without** discretisation of continuous variables. + +### 3.2 Likelihood Estimation and Predictions with `BayesianNetwork` + +Once the graph has been determined, the `BayesianNetwork` can be initialised and the conditional probability distributions of the variables can be learned from the data. + +Maximum likelihood or Bayesian parameter estimation can be used for CPDs learning. +> When learning CPDs of the BN, +> - The dicscretised data should be used (either high or low granularity of features and target variables can be used); +> - Overfitting and underfitting of CPDs can be avoided with an appropriate train/test split of the data. + +```python + from causalnex.network import BayesianNetwork + from causalnex.discretiser import Discretiser + + # Inititalise BN with defined structure model + bn = BayesianNetwork(sm) + # First, learn all the possible states of the nodes using the whole dataset + bn.fit_node_states(data_discrete) + # Fit CPDs using the training dataset with the discretised continuous variable "c" + train_data_discrete = train_data.copy() + train_data_discrete["c"] = Discretiser(method="uniform").transform(discretised_data["c"].values) + bn.fit_cpds(train_data_discrete, method="BayesianEstimator", bayes_prior="K2") +``` + +Once the CPDs are learned, they can be used to predict the state of a node as well as probability of each possible state of a node, based on some input data (e.g., previously unseen test data) and learned CPDs: + +```python + predictions = bn.predict(test_data_discrete, "c") + predicted_probabilities = bn.predict_probability(test_data_discrete, "c") +``` +> When all parents of a given node exist within input data, the method inspects the CPDs directly and avoids traversing the full network. When some parents do not exist within input data, the most likely state for every node that is not contained within data is computed, and the predictions are made accordingly. + +## 4. Querying model and making interventions with `InferenceEngine` + +After iterating over the model structure, CPDs, and validating the model quality, we can +undertake inference on a BN to examine expected behaviour and gain insights. + +`InferenceEngine` class provides methods to query marginals based on observations and to make interventions (a.k.a. DO-calculus) on a Bayesian Network. + +### 4.1 Querying marginals with `InferenceEngine.query` + +Inference and observation of evidence are done on the fly, following a deterministic [Junction Tree Algorithm (JTA)](https://ermongroup.github.io/cs228-notes/inference/jt/). + +To query the model for baseline marginals that reflect the population as a whole, a `query` method can be used. + +> We recommend to update the model using the complete dataset for this type of queries. + +```python + from causalnex.inference import InferenceEngine + + # Updating the model on the whole dataset + bn.fit_cpds(data_discrete, method="BayesianEstimator", bayes_prior="K2") + ie = InferenceEngine(bn) + # Querying all the marginal probabilities of the model's distribution + marginals = ie.query({}) +``` + +Users can also query the marginals of states in a BN given some _observations_. +These observations can be made anywhere in the network; the marginal distributions of nodes (including the target variable) will be updated and their impact will be propagated through to the node of interest: + +```python + # Querying the marginal probabilities of the model's distribution + # after an observed state of the node "b" + marginals_after_observations = ie.query({"b": True}) +``` + +> - For complex networks, the JTA may take an hour to update the probabilities throughout the network; +> - This process can not be parallelised, but multiple queries can be run in parallel; + +### 4.2 Making interventions (Do-calculus) with `InferenceEngine.do_intervention` + +Finally, users can use the insights from the inference and observation of evidence to encode taking _actions_ and observe the effect of these actions on the target variable. + +Our package supports simple Do-Calculus, allowing as to Make an intervention on the Bayesian Network. + +Users can apply an intervention to any node in the data, updating its distribution using a _do_ operator, +examining the effect of that intervention by querying marginals and resetting any interventions: + +```python + # Doing an intervention to the node "d" + ie.do_intervention("d", True) + # Querying all the updated marginal probabilities of the model's distribution + marginals_after_interventions = ie.query({}) + # Re-introducing the original conditional dependencies + ie.reset_do("d") +``` diff --git a/docs/source/04_user_guide/images/BN.png b/docs/source/04_user_guide/images/BN.png new file mode 100644 index 0000000..d0659d3 Binary files /dev/null and b/docs/source/04_user_guide/images/BN.png differ diff --git a/docs/source/04_user_guide/images/graph.png b/docs/source/04_user_guide/images/graph.png new file mode 100644 index 0000000..b37389c Binary files /dev/null and b/docs/source/04_user_guide/images/graph.png differ diff --git a/docs/source/04_user_guide/images/graph_definitions.png b/docs/source/04_user_guide/images/graph_definitions.png new file mode 100644 index 0000000..3ba7230 Binary files /dev/null and b/docs/source/04_user_guide/images/graph_definitions.png differ diff --git a/docs/source/05_resources/05_faq.md b/docs/source/05_resources/05_faq.md new file mode 100644 index 0000000..2242776 --- /dev/null +++ b/docs/source/05_resources/05_faq.md @@ -0,0 +1,103 @@ +# Frequently asked questions + +> *Note:* This documentation is based on `CausalNex 0.4.0`, if you spot anything that is incorrect then please create an [issue](https://github.com/quantumblacklabs/causalnex/issues) or pull request. + +## What is CausalNex? + +[CausalNex](https://github.com/quantumblacklabs/causalnex) is a python library that allows data scientists and domain experts to co-develop models which go beyond correlation to consider causal relationships. It was originally designed by [Paul Beaumont](https://www.linkedin.com/in/pbeaumont/) and [Ben Horsburgh](https://www.linkedin.com/in/benhorsburgh/) to solve challenges they faced in inferencing causality in their project work. + +This work was later turned into a product thanks to the following contributors: [Ivan Danov](https://github.com/idanov), [Dmitrii Deriabin](https://github.com/DmitryDeryabin), [Yetunde Dada](https://github.com/yetudada), [Wesley Leong](https://www.linkedin.com/in/wesleyleong/), [Steve Ler](https://www.linkedin.com/in/song-lim-steve-ler-380366106/), [Viktoriia Oliinyk](https://www.linkedin.com/in/victoria-oleynik/), [Roxana Pamfil](https://www.linkedin.com/in/roxana-pamfil-1192053b/), [Fabian Peter](https://www.linkedin.com/in/fabian-peters-6291ab105/), [Nisara Sriwattanaworachai](https://www.linkedin.com/in/nisara-sriwattanaworachai-795b357/) and [Nikolaos Tsaousis](https://www.linkedin.com/in/ntsaousis/). + +## What are the benefits of using CausalNex? + +It is important to consider the primary benefits of CausalNex in the context of an end-to-end causality and counterfactual analysis. + +As we see it, CausalNex: + +- **Generates transparency and trust in models** it creates by allowing users to collaborate with domain experts during the modelling process. +- Uses an **optimised structure learning algorithm**, [NOTEARS](https://papers.nips.cc/paper/8157-dags-with-no-tears-continuous-optimization-for-structure-learning.pdf) where the runtime to learn structure is no longer exponential but scales cubically with number of nodes. +- **Add known relationships or remove spurious correlations** so that your model can better consider causal relationships in data +- **Visualise networks using common tools** built upon [NetworkX](https://networkx.github.io/), allowing users to understand relationships in their data more intuitively, and work with experts to encode their knowledge +- **Streamlines the use of Bayesian Networks** for an end-to-end counterfactual analysis, which in the past was a complicated process involving the use of at least three separate open source libraries, each with its own interface. + +## When should you consider using CausalNex? + +CausalNex is created specifically for data scientists who would like an efficient and intuitive process to identify causal relationships and the right intervention through data and collaboration with domain experts. + +## Why NOTEARS algorithm over other structure learning methods? + +Historically, structure learning has been a very **hard** problem. We are interested in looking for the optimal directed acyclic graph (DAGs) that describes the conditional dependencies between variables. However, the search space for this is **combinatorial** and scales **super-exponentially** with the number of nodes. [NOTEARS](https://papers.nips.cc/paper/8157-dags-with-no-tears-continuous-optimization-for-structure-learning.pdf) algorithm cleverly introduces a new optimisation heuristic and approach to solving this problem, where the runtime for this is no longer exponential but scales **cubically** with the number of nodes. + +## What is the recommended type of dataset to be used in NOTEARS? + +[NOTEARS](https://papers.nips.cc/paper/8157-dags-with-no-tears-continuous-optimization-for-structure-learning.pdf) works by detecting if a small increase in the value of the node will result in an increase in another node. If there is, NOTEARS will be able to capture this and assert that this is a causal relationship. Therefore, we highly recommend that the dataset to be used is **continuous**. + +**Categorical variables** like blood type **won’t be able to work** in this case. Nonetheless, after learning the structure using NOTEARS, one can still manually add the relationships for these features to the structure based on their domain knowledge. + +## What is the recommended number of samples for satisfactory performance? + +According to the benchmarking done on **synthetic dataset** in-house, it is highly recommended that **at least 1000 samples** is used to get a satisfactory performance. We also discovered that any further increase than 1000 samples **does not help improve the accuracy** regardless of number of nodes, and it takes a **longer time** to run. + +## Why can my StructureModel be cyclic, but not my BayesianNetwork? + +StructureModel is used when discovering the causal structure of a dataset. Part of this process is adding, removing, and flipping edges, until the appropriate structure is completed. As edges are modified, cycles can be temporarily introduced into the structure, which would raise an Exception within a BayesianNetwork, which is a specialised **directed acyclic graph**. + +Once the structure is finalised, and is acyclic, then it can be used to create a [BayesianNetwork](https://causalnex.readthedocs.io/en/latest/04_user_guide/04_user_guide.html) + + +## Why a separate data pre-processing process for probability fitting than structure learning? / Why discretise data in probability fitting? + +We treat Bayesian Network probability fitting and Structure Learning as two separate problems. The data for Structure Learning should be continuous for the causal relationships to be learnt. **Once we already knew the causal relationship between all the nodes**, we can start doing probability fitting. At the moment, we are **only supporting discrete Bayesian Network model**, and this requires the continuous features to be discretised. + +## Why call fit_node_states before fit_cpds? + +Before fitting, the model first has to know how many states each node has to carry out the computations. Alternatively, one can also call **fit_node_states_and_cpds**. However, there is a chance that this might not work if one were to do train/test splitting as the model might not see all the possible states. + +For example, rare blood type like AB-negative might not appear in the training data but in the test data. Therefore, we strongly encourage users to do **fit_node_states using all data** and **fit_cpds using training data** to test the model quality, so that the model knows all the possible states that each node can have. + +## What is Do-intervention and when to use it? + +[Do-intervention](https://causalnex.readthedocs.io/en/latest/04_user_guide/04_user_guide.html) is symbolically described as p(y|do(x)). It asks the question of what is the probability distribution of Y if we were to **set** the value of X to x **arbitrarily**. + +For example, we have 50% of males and 50% of females in the world, but we might be interested to learn about the probability distribution of happiness index if we had 80% of females and 20% males in the world. + +Do-intervention is very useful in **counterfactual analysis**, where we are interested to know if the outcomes would have been different if we had taken a different action/intervention. + +## How can I make inference faster? + +At the moment, the algorithm calculates the probability of **every node** in a Bayesian Network. If users are interested in making inference of the target node faster, user can remove nodes that are independent from the target node, and also children of the target node. For example, if we have C<-A->B->D and we want to learn P(B|A), we can remove C and D to make the inference faster. + +## How does CausalNex compare to other projects, e.g. CausalML, DoWhy? + +The following points describe how we are unique comparing to the others: +1) We are one of the very few causal packages that use **Bayesian Networks** to model the problems. Most of the causal packages use statistical matching technique like **propensity score matching** to approach these problems. +2) One of the main hurdle to applying Bayesian Network is to find the optimal graph structure. In CausalNex, We **simplify** this process by providing the ability for the users to learn the graph structure through: i) **encoding domain expertise** by manually adding the edges, and ii) **leveraging the data** using the state-of-the-art [structure learning algorithm](https://papers.nips.cc/paper/8157-dags-with-no-tears-continuous-optimization-for-structure-learning.pdf). +3) We provide the ability for the users to do **counterfactual analysis** using Bayesian Network by introducing **Do-Calculus**, which is not commonly found in Bayesian Network packages. + +## What version of Python does CausalNex use? + +CausalNex is built for Python 3.5, 3.6 and 3.7. + +## How do I upgrade CausalNex? + +We use [SemVer](http://semver.org/) for versioning. The best way to upgrade safely is to check our [release notes](RELEASE.md) for any notable breaking changes. + +Once CausalNex is installed, you can check your version as follows: + +``` +pip show causalnex +``` + +To later upgrade CausalNex to a different version, simply run: + +``` +pip install causalnex -U +``` + +## How can I find out more CausalNex? + +CausalNex is on GitHub, and our preferred community channel for feedback is through [GitHub issues](https://github.com/quantumblacklabs/causalnex/issues). You can find news about updates and new features introduced by heading over to [RELEASE.md](https://github.com/quantumblacklabs/causalnex/blob/develop/RELEASE.md). + +## Where can I learn more about Bayesian Networks? + +You can read our [documentation](https://causalnex.readthedocs.io/en/latest/04_user_guide/04_user_guide.htm) to know more about the concepts and other useful references with regards to using Bayesian Networks for Causal Inference. diff --git a/docs/source/api_docs/causalnex.rst b/docs/source/api_docs/causalnex.rst new file mode 100644 index 0000000..b98ba91 --- /dev/null +++ b/docs/source/api_docs/causalnex.rst @@ -0,0 +1,21 @@ +causalnex +======== + +.. rubric:: Description + +.. automodule:: causalnex + + + + .. rubric:: Modules + + .. autosummary:: + :toctree: + :template: autosummary/module.rst + + causalnex.structure + causalnex.plots + causalnex.discretiser + causalnex.network + causalnex.evaluation + causalnex.inference diff --git a/docs/source/api_docs/index.rst b/docs/source/api_docs/index.rst new file mode 100644 index 0000000..70b133c --- /dev/null +++ b/docs/source/api_docs/index.rst @@ -0,0 +1,99 @@ +.. causalnex documentation master file, created by + sphinx-quickstart on Mon Dec 18 11:31:24 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +.. image:: causalnex_banner.png + :alt: CausalNex logo + :class: causalnex-logo + +Welcome to CausalNex's API docs and tutorials! +============================================= + +.. image:: https://circleci.com/gh/quantumblacklabs/causalnex/tree/master.svg?style=shield + :target: https://circleci.com/gh/quantumblacklabs/causalnex/tree/master + :alt: CircleCI build status + +.. image:: https://img.shields.io/badge/coverage-100%25-brightgreen.svg + :target: https://github.com/quantumblacklabs/causalnex + :alt: Test coverage + +.. image:: https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7-blue.svg + :target: https://pypi.org/project/causalnex/ + :alt: Python version 3.5, 3.6, 3.7 + +.. image:: https://badge.fury.io/py/causalnex.svg + :target: https://pypi.org/project/causalnex/ + :alt: PyPI package version + +.. image:: https://readthedocs.org/projects/causalnex/badge/?version=latest + :target: https://causalnex.readthedocs.io/ + :alt: Docs build status + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/ambv/black + :alt: Code style is Black + +.. image:: https://img.shields.io/badge/license-Apache%202.0-blue.svg + :target: https://opensource.org/licenses/Apache-2.0 + :alt: License is Apache 2.0 + +.. image:: https://pepy.tech/badge/causalnex + :target: https://pepy.tech/project/causalnex + :alt: Downloads + +.. toctree:: + :maxdepth: 2 + :caption: Introduction + + 01_introduction/01_introduction + +.. toctree:: + :maxdepth: 2 + :caption: Getting Started + + 02_getting_started/01_prerequisites + 02_getting_started/02_install + +.. toctree:: + :maxdepth: 2 + :caption: Tutorial + + 03_tutorial/03_tutorial.md + + +.. toctree:: + :maxdepth: 2 + :caption: User guide + + 04_user_guide/04_user_guide + 04_user_guide/04_reference + +.. toctree:: + :maxdepth: 2 + :caption: Resources + + 05_resources/05_faq + + +API Docs +======== + +.. toctree:: + :maxdepth: 0 + :caption: API Docs + :hidden: + + causalnex + +.. autosummary:: + :template: autosummary/module.rst + + causalnex + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/causalnex_banner.png b/docs/source/causalnex_banner.png new file mode 100644 index 0000000..06a5584 Binary files /dev/null and b/docs/source/causalnex_banner.png differ diff --git a/docs/source/css/causalnex.css b/docs/source/css/causalnex.css new file mode 100644 index 0000000..2c33866 --- /dev/null +++ b/docs/source/css/causalnex.css @@ -0,0 +1,22 @@ +table { + font-family: "Trebuchet MS", Arial, Helvetica, sans-serif; + display: block; + overflow: scroll; +} + +td, th { + border: 1px solid #ddd; + padding: 8px; +} + +tr:nth-child(even){background-color: #f2f2f2;} + +tr:hover {background-color: #ddd;} + +th { + padding-top: 12px; + padding-bottom: 12px; + text-align: left; + background-color: #34BB54; + color: white; +} diff --git a/docs/source/css/copybutton.css b/docs/source/css/copybutton.css new file mode 100644 index 0000000..6c01d71 --- /dev/null +++ b/docs/source/css/copybutton.css @@ -0,0 +1,60 @@ +/* Copied from: https://github.com/raw/choldgraf/sphinx-copybutton/master/sphinx_copybutton/_static/copybutton.css */ + + /* Copy buttons */ +a.copybtn { + position: absolute; + top: 2px; + right: 2px; + width: 1.7em; + height: 1.7em; + padding: .3em; + opacity: .6; + transition: opacity 0.5s; +} + + div.highlight { + position: relative; + background: #f5f5f5; +} + + .highlight:hover .copybtn { + opacity: 1; +} + + /** + * A minimal CSS-only tooltip copied from: + * https://codepen.io/mildrenben/pen/rVBrpK + * + * To use, write HTML like the following: + * + * <p class="o-tooltip--left" data-tooltip="Hey">Short</p> + */ + .o-tooltip--left { + position: relative; + } + + .o-tooltip--left:after { + opacity: 0; + visibility: hidden; + position: absolute; + content: attr(data-tooltip); + padding: 2px; + top: 0; + left: 0; + background: grey; + font-size: 1rem; + color: white; + white-space: nowrap; + z-index: 2; + border-radius: 2px; + transform: translateX(-102%) translateY(0); + transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1); +} + + .o-tooltip--left:hover:after { + display: block; + opacity: 1; + visibility: visible; + transform: translateX(-100%) translateY(0); + transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1); +} diff --git a/docs/source/css/qb1-sphinx-rtd.css b/docs/source/css/qb1-sphinx-rtd.css new file mode 100644 index 0000000..8dfcbe3 --- /dev/null +++ b/docs/source/css/qb1-sphinx-rtd.css @@ -0,0 +1,405 @@ +@import url("https://fonts.googleapis.com/css?family=Titillium+Web:300,400,600"); + +html, body.wy-body-for-nav { + margin: 0; + padding: 0; + -webkit-font-smoothing: antialiased; + font-family: 'Titillium Web', sans-serif; + font-weight: 400; + line-height: 2rem; +} + +html { + font-size: 62.5%; +} + +body.wy-body-for-nav { + font-size: 1.6rem; + background: rgb(250, 250, 250) !important; + color: black; +} + +.wy-side-nav-search { + text-align: left; +} + +.wy-side-nav-search input[type=text] { + display: block; + box-sizing: border-box; + width: 100%; + padding: 6px 12px; + color: #666; + background-color: #fff; + font-family: inherit; + font-size: 1.6rem; + border: 1px #ccc solid; + border-radius: 2px; + transition: all ease 0.15s; + box-shadow: none; +} + +.wy-side-nav-search input[type=text]:focus { + border-color: #888; + color: #333; +} + +.wy-body-for-nav .wy-nav-side { + position: fixed; + top: 0; + bottom: 0; + left: 0; + padding-bottom: 2em; + width: 300px; + overflow-x: hidden; + overflow-y: auto; + min-height: 100%; + background: white; + z-index: 200; + box-shadow: 0 2px 5px 0 rgba(0, 0, 0, .05); +} + +.wy-body-for-nav .wy-side-scroll { + width: 100%; + position: relative; + overflow-x: initial; + overflow-y: initial; + height: initial; +} + +.wy-body-for-nav .wy-side-nav-search { + width: 100%; + background: none; + margin: 0; + padding: 0 20px; +} +.wy-body-for-nav .wy-side-nav-search a { + display: inline-flex; + flex-direction: row-reverse; + align-items: center; + font-size: 4rem; + margin: 0; + padding: 0; + height: 7rem; + line-height: 7rem; + color: black; +} +.wy-body-for-nav .wy-side-nav-search a:before { + display: none; +} +.wy-body-for-nav .wy-side-nav-search a img.logo { + width: 4.5rem; + height: auto; + margin: 0 0.3rem 0 -0.6rem; + padding: 0; +} +.wy-body-for-nav .wy-side-nav-search>div.version { + color: #555; + display: inline-block; + margin-left: 0.5em; + font-size: 1.4rem; +} + +.wy-body-for-nav .wy-menu-vertical { + width: 300px; + margin: 0; + padding: 0 20px; +} + +.wy-body-for-nav .wy-menu-vertical p.caption { + margin: 1.5em 0 0.2em; + padding: 0; + color: #666; + font-weight: bold; + font-size: 2rem; +} + +.wy-body-for-nav .wy-menu-vertical li.on a, .wy-body-for-nav .wy-menu-vertical li.current>a { + border-top: none; + border-bottom: none; +} + +.wy-body-for-nav .wy-menu-vertical li.current { + background: none; +} + +.wy-body-for-nav .wy-menu-vertical li { + margin: 0; + padding: 0 0 0 20px; +} + +.wy-body-for-nav .wy-menu-vertical li a { + display: block; + margin: 0; + padding: 1.2rem 0 !important; + font-size: 1.6rem; + line-height: 1; + color: #222; + background: none !important; +} + +.rst-content.style-external-links a.reference.external:after { + color: inherit; + opacity: 0.8; +} + +.wy-body-for-nav .wy-menu-vertical a:hover { + color: #000; +} + +.wy-body-for-nav li span.toctree-expand { + margin-left: -20px; +} + +.wy-body-for-nav .toctree-expand:before { + font-size: 15px; + margin-right: 5px; +} + +.wy-body-for-nav .wy-nav-content-wrap { + margin-left: 300px; + background: #fafafa; +} + +.wy-body-for-nav .wy-nav-content { + padding: 0; + max-width: initial; +} + +.wy-body-for-nav .wy-breadcrumbs { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + width: 100%; + height: 7rem; + padding-left: 3.2rem; + background: white; + background-image: initial; + background-position-x: initial; + background-position-y: initial; + background-size: initial; + background-repeat-x: initial; + background-repeat-y: initial; + background-attachment: initial; + background-origin: initial; + background-clip: initial; + background-color: white; + box-shadow: 0 2px 5px 0 rgba(0, 0, 0, .05); +} + +.wy-body-for-nav .wy-breadcrumbs + hr { + display: none; +} +.wy-body-for-nav .wy-breadcrumbs li { + font-size: 1.8rem; +} + +.wy-body-for-nav .wy-breadcrumbs li:not(:first-child) { + margin-left: 6px; +} + +.wy-body-for-nav .wy-nav-content .document { + padding: 3.2rem 3.2rem 2rem; + max-width: 100rem; +} + +.causalnex-logo { + margin-bottom: 3rem; +} + +.wy-body-for-nav footer { + padding: 32px; +} + +.wy-body-for-nav .wy-body-for-nav { + background: #fafafa !important; + background-image: none; + background-size: initial; +} + +.wy-body-for-nav .wy-menu-vertical li.current a:hover { + background: none; +} + +.wy-menu-vertical a span.toctree-expand { + color: #555 !important; +} + +.wy-menu-vertical a:hover span.toctree-expand { + color: #000 !important; +} + +.wy-body-for-nav .wy-menu-vertical li.current a { + border-right: none; +} + +.wy-body-for-nav .toctree-l2 { + padding-left: 15px !important; +} + +.wy-body-for-nav .toctree-l3 { + padding-left: 15px !important; +} + +.wy-body-for-nav .toctree-l4 { + padding-left: 15px !important; + } + +.wy-body-for-nav .toctree-l5 { + padding-left: 15px !important; +} + + +.wy-body-for-nav .toctree-l2.current > a, .wy-body-for-nav .toctree-l3.current > a,.wy-body-for-nav .toctree-l4.current > a { + font-weight: normal !important; +} +.wy-body-for-nav .toctree-l4 a { + word-break: break-word; +} + +.wy-body-for-nav b, .wy-body-for-nav strong { + font-weight: normal; +} + +.wy-plain-list-disc li, +.rst-content .section ul li, +.rst-content .toctree-wrapper ul li, +article ul li { + margin-top: 0.35em; + margin-bottom: 0.35em; +} + +h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend { + font-family: 'Titillium Web', sans-serif; + margin: 1em 0 0.3em; + line-height: 1.2em; +} + +.wy-body-for-nav .document > div > .section > *:not(:empty):first-of-type { + margin-top: 0; +} + +.wy-body-for-nav h1 { + font-size: 3.9rem; + letter-spacing: -0.3px; +} + +.wy-body-for-nav h2 { + font-size: 3.25rem; +} + +.wy-body-for-nav h3 { + font-size: 2.6rem; +} + +.wy-body-for-nav h4 { + font-size: 1.95rem; +} + +.wy-body-for-nav h5 { + font-size: 1.625rem; +} + +.wy-body-for-nav h6 { + font-size: 1.4625rem; +} + +.wy-body-for-nav .headerlink { + display: none !important; +} + +.wy-body-for-nav p { + font-size: 100%; + margin: 0 0 15px 0; + line-height: 1.5; +} + +.wy-body-for-nav blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid #ddd; + color: #6a6a6a; +} + +.wy-body-for-nav .rst-content a, .wy-body-for-nav footer a { + font-family: inherit; + font-size: inherit; + color: #006ea7; + text-decoration: none; +} + +.wy-body-for-nav .rst-content a:visited, .wy-body-for-nav footer a:visited { + color: #446b7f; +} + +.wy-body-for-nav .rst-content a:hover, .wy-body-for-nav footer a:hover { + text-decoration: underline; +} + +.wy-body-for-nav .rst-content .btn:hover, .wy-body-for-nav footer .btn:hover { + text-decoration: none; +} + +.wy-body-for-nav .wy-nav-top { + padding: 5px 20px; + background: white; + color: black; + box-shadow: 0 2px 5px 0 rgba(0, 0, 0, 0.05); +} + +.wy-body-for-nav .wy-nav-top i { + transform: translateY(6px); +} + +.wy-body-for-nav .wy-nav-top a { + font-size: 2.8rem; + color: black !important; +} + + +@media screen and (max-width: 768px) { + .wy-body-for-nav .wy-nav-side { + transform: translate(-300px, 0); + transition: all ease 0.3s; + } + .wy-body-for-nav .wy-nav-side.shift { + width: 85%; + transform: translate(0, 0); + } + .wy-body-for-nav .wy-nav-content-wrap { + margin-left: 0; + transform: translate(0, 0); + transition: all ease 0.3s; + } + .wy-body-for-nav .wy-nav-content-wrap.shift { + position: relative; + left: 0; + transform: translate(85%, 0); + } + .wy-body-for-nav .wy-breadcrumbs { + display: none; + } + .wy-body-for-nav .wy-nav-content .document { + padding: 20px; + } +} + +@media screen and (min-width: 1600px) { + .wy-body-for-nav .wy-nav-side { + width: 350px; + } + .wy-body-for-nav .wy-nav-content-wrap { + margin-left: 350px; + } + html { + font-size: 70%; + } +} + +/* Fix Read The Docs side-effects */ +.wy-body-for-nav .rst-versions { + font-size: 16px; + line-height: 1; +} diff --git a/docs/source/css/theme-overrides.css b/docs/source/css/theme-overrides.css new file mode 100644 index 0000000..c928bd0 --- /dev/null +++ b/docs/source/css/theme-overrides.css @@ -0,0 +1,11 @@ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td { + white-space: normal; + } + + .wy-table-responsive { + overflow: visible; + } +} diff --git a/docs/source/examples/.gitkeep b/docs/source/examples/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/legal_header.txt b/legal_header.txt new file mode 100644 index 0000000..5da8261 --- /dev/null +++ b/legal_header.txt @@ -0,0 +1,27 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0a0069c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +matplotlib>=3.0.3, <4.0 +networkx==2.2 +numpy>=1.14.2, <2.0 +pandas==0.24.0 +pgmpy==0.1.6 +prettytable==0.7.2 +scikit-learn==0.20.2 +scipy>=1.2.0, <1.3 +wrapt>=1.11.0, <1.12 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..10fbf27 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,7 @@ +[tool:pytest] +addopts=--cov-report term-missing + --cov causalnex + --cov ebaybbn + --cov tests + --no-cov-on-fail + -ra diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8e1462d --- /dev/null +++ b/setup.py @@ -0,0 +1,77 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from os import path + +from setuptools import find_packages, setup + +name = "causalnex" +here = path.abspath(path.dirname(__file__)) + +# get package version +with open(path.join(here, name, "__init__.py"), encoding="utf-8") as f: + result = re.search(r'__version__ = ["\']([^"\']+)', f.read()) + if not result: + raise ValueError("Can't find the version in causalnex/__init__.py") + version = result.group(1) + +# get the dependencies and installs +with open("requirements.txt", "r", encoding="utf-8") as f: + requires = [x.strip() for x in f if x.strip()] + +# get test dependencies and installs +with open("test_requirements.txt", "r", encoding="utf-8") as f: + test_requires = [x.strip() for x in f if x.strip() and not x.startswith("-r")] + +# Get the long description from the README file +with open(path.join(here, "README.md"), encoding="utf-8") as f: + readme = f.read() + +setup( + name=name, + version=version, + description="Toolkit for causal reasoning (Bayesian Networks / Inference)", + long_description=readme, + long_description_content_type="text/markdown", + url="https://github.com/quantumblacklabs/causalnex", + python_requires=">=3.5, <3.8", + author="QuantumBlack Labs", + author_email="causalnex@quantumblack.com", + packages=find_packages(exclude=["docs*", "tests*", "tools*"]), + include_package_data=True, + tests_require=test_requires, + install_requires=requires, + keywords="Causal Reasoning, Bayesian Network, Inference, Structure Learning, Do-Calculus", + classifiers=[ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + ], +) diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 0000000..756b28a --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,9 @@ +-r requirements.txt +flake8>=3.5,<4.0 +isort>=4.3.16, <5.0 +mock>=2.0.0,<3.0 +pre-commit>=1.17.0, <2.0.0 +pylint>=2.3.1, <3.0 +pytest-cov>=2.5, <3.0 +pytest-mock>=1.7.1,<2.0 +pytest>=4.3.0,<5.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..6c320f5 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +""" +causalnex +Toolkit for causal reasoning (Bayesian Networks / Inference) +""" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0ca3604 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,489 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +import numpy as np +import pandas as pd +import pytest +from pgmpy.models import BayesianModel + +from causalnex.network import BayesianNetwork +from causalnex.structure import StructureModel +from causalnex.structure.notears import from_pandas + + +@pytest.fixture +def train_model() -> StructureModel: + """ + This Bayesian Model structure will be used in all tests, and all fixtures will adhere to this structure. + + Cause-only nodes: [d, e] + Effect-only nodes: [a, c] + Cause / Effect nodes: [b] + + d + ↙ ↓ ↘ + a ← b → c + ↑ ↗ + e + """ + model = StructureModel() + model.add_edges_from( + [ + ("b", "a"), + ("b", "c"), + ("d", "a"), + ("d", "c"), + ("d", "b"), + ("e", "c"), + ("e", "b"), + ] + ) + return model + + +@pytest.fixture +def train_model_idx(train_model) -> BayesianModel: + """ + This Bayesian model is identical to the train_model() fixture, with the exception that node names + are integers from zero to 1, mapped by: + + {"a": 0, "b": 1, "c": 2, "d": 3, "e": 4} + """ + model = BayesianModel() + idx_map = {"a": 0, "b": 1, "c": 2, "d": 3, "e": 4} + model.add_edges_from([(idx_map[u], idx_map[v]) for u, v in train_model.edges]) + return model + + +@pytest.fixture +def train_data() -> pd.DataFrame: + """ + Training data for testing Bayesian Networks. There are 98 samples, with 5 columns: + + - a: {"a", "b", "c", "d"} + - b: {"x", "y", "z"} + - c: 0.0 - 100.0 + - d: Boolean + - e: Boolean + + This data was generated by constructing the Bayesian Model train_model(), and then sampling + from this structure. Since e and d are both independent of all other nodes, these were sampled first for + each row (form their respective pre-defined distributions). This then allows the sampling of all further + variables based on their conditional dependencies. + + The approximate distributions used to sample from can be viewed by inspecting train_data_cpds(). + + """ + + data_arr = [ + ["a", "x", 73.78658346945414, False, False], + ["d", "x", 12.765853213346603, False, False], + ["c", "y", 22.43657132589221, False, False], + ["a", "x", 4.267744937038964, False, False], + ["b", "x", 62.87087344904927, False, False], + ["c", "x", 31.55295196889971, False, False], + ["a", "x", 37.403388911083965, False, False], + ["b", "x", 63.171968604247155, False, False], + ["d", "x", 11.140539452118263, False, False], + ["d", "x", 0.1555338799942385, True, False], + ["c", "x", 9.269926225399187, False, True], + ["b", "z", 75.38846241765208, True, True], + ["c", "z", 33.10212378889936, False, True], + ["b", "z", 57.04657630213301, True, True], + ["b", "x", 72.03855905511072, True, False], + ["c", "x", 5.106018765399956, False, False], + ["c", "z", 5.802617702038839, False, True], + ["c", "x", 17.22538330530506, False, False], + ["a", "y", 87.05395007052729, False, False], + ["d", "y", 19.09989481093348, False, False], + ["c", "x", 4.313272835124353, True, False], + ["b", "x", 13.660704178900938, True, True], + ["b", "x", 7.693287813764131, False, False], + ["c", "y", 32.791770073523246, False, False], + ["c", "y", 12.039098492465282, False, False], + ["a", "x", 51.97718339128754, False, False], + ["d", "x", 8.393970656769238, False, False], + ["a", "x", 0.3610815726384886, False, False], + ["a", "y", 35.31788713900731, True, False], + ["b", "x", 35.84702992379284, False, True], + ["c", "y", 32.872350426703356, True, False], + ["a", "x", 21.218746335586868, False, True], + ["b", "y", 71.5495653029006, True, False], + ["c", "x", 15.393846082097575, False, False], + ["d", "y", 4.514559208625406, False, False], + ["d", "x", 0.704928173400301, False, False], + ["c", "y", 34.10829794112354, True, False], + ["d", "x", 6.84602512195673, False, False], + ["b", "y", 25.43743439885204, False, False], + ["d", "x", 7.544831467091971, False, False], + ["d", "x", 13.923699372025073, False, False], + ["b", "x", 21.493005760070915, False, False], + ["a", "x", 41.353977640369436, False, False], + ["c", "z", 10.015835005248583, True, True], + ["c", "z", 29.40115954319444, False, True], + ["c", "x", 17.305145945035388, False, False], + ["b", "x", 57.3687951851441, False, False], + ["a", "x", 59.31395756039643, False, False], + ["d", "x", 19.557939187075984, False, False], + ["d", "y", 15.739556224725082, False, False], + ["c", "x", 6.850626809845993, True, False], + ["c", "x", 7.774579861173826, False, False], + ["c", "x", 20.807136344297092, True, False], + ["b", "y", 29.406207780312343, False, False], + ["a", "x", 34.38851648220974, False, False], + ["d", "x", 1.0951104244381218, True, False], + ["c", "x", 37.27483338042188, False, False], + ["b", "x", 15.745994603442064, False, False], + ["c", "x", 17.78180189764816, False, True], + ["a", "x", 17.067548428231493, True, False], + ["c", "x", 26.857320012899727, False, False], + ["a", "x", 41.0038510689549, False, True], + ["d", "x", 0.2299684913699096, False, True], + ["a", "x", 57.35885570158893, True, False], + ["d", "x", 12.40118443712448, False, False], + ["c", "x", 22.624550487374112, False, False], + ["a", "x", 93.08587619178269, False, False], + ["b", "y", 18.33030505634329, False, False], + ["a", "z", 64.29945681859853, False, True], + ["b", "x", 73.66024742961967, False, False], + ["b", "x", 16.717397443478287, False, True], + ["c", "y", 4.642615342125205, False, True], + ["c", "x", 9.431345661106931, False, False], + ["c", "y", 31.76238774237109, False, False], + ["c", "y", 3.6961806894707965, False, False], + ["d", "y", 2.298895066631253, True, False], + ["d", "y", 13.222298172220462, False, False], + ["c", "x", 28.301638775451153, False, False], + ["d", "x", 7.702270580869413, True, False], + ["a", "y", 41.38492280508702, True, False], + ["d", "x", 13.047815503255656, True, False], + ["c", "x", 22.14641490202623, False, False], + ["b", "z", 43.13007970158368, False, True], + ["b", "x", 60.09518672623882, True, False], + ["a", "x", 79.6370082234198, False, False], + ["d", "x", 16.60880504367762, False, False], + ["a", "z", 22.88783470451029, False, True], + ["a", "x", 33.66416643964188, False, False], + ["b", "y", 69.91787304290465, True, True], + ["c", "x", 31.941092922567663, True, False], + ["d", "x", 16.739638908154518, False, False], + ["a", "z", 11.129589373273108, False, True], + ["d", "y", 4.96943558614434, True, False], + ["d", "y", 6.585354730457387, False, False], + ["d", "x", 9.859942318446954, False, False], + ["b", "z", 18.541485302271496, False, True], + ["a", "x", 87.53473074574995, True, False], + ["a", "z", 59.61068083691302, False, True], + ] + + data = pd.DataFrame(data_arr, columns=["a", "b", "c", "d", "e"]) + return data + + +@pytest.fixture +def train_data_discrete(train_data) -> pd.DataFrame: + """ + train_data in discretised form. This maps "c" into 5 buckets: + - 0: x < 20 + - 1: 20 <= x < 40 + - 2: 40 <= x < 60 + - 3: 60 <= x < 80 + - 4: 80 <= x + """ + df = train_data.copy(deep=True) # type: pd.DataFrame + df["c"] = df["c"].apply( + lambda c: 0 if c < 20 else 1 if c < 40 else 2 if c < 60 else 3 if c < 80 else 4 + ) + return df + + +@pytest.fixture +def train_data_idx(train_data) -> pd.DataFrame: + """ + train_data in integer index form. This maps each column into values from 0..n + """ + + df = train_data.copy(deep=True) # type: pd.DataFrame + + df["a"] = df["a"].map({"a": 0, "b": 1, "c": 2, "d": 3}) + df["b"] = df["b"].map({"x": 0, "y": 1, "z": 2}) + df["c"] = df["c"].apply( + lambda c: 0 if c < 20 else 1 if c < 40 else 2 if c < 60 else 3 if c < 80 else 4 + ) + df["d"] = df["d"].map({True: 1, False: 0}) + df["e"] = df["e"].map({True: 1, False: 0}) + return df + + +@pytest.fixture +def train_data_idx_cpds(train_data_idx) -> Dict[str, np.ndarray]: + """Conditional probability distributions of train_data in the train_model""" + + return create_cpds(train_data_idx) + + +@pytest.fixture +def train_data_discrete_cpds(train_data_discrete) -> Dict[str, np.ndarray]: + """Conditional probability distributions of train_data in the train_model""" + + return create_cpds(train_data_discrete) + + +@pytest.fixture +def train_data_discrete_cpds_k2(train_data_discrete) -> Dict[str, np.ndarray]: + """Conditional probability distributions of train_data in the train_model""" + + return create_cpds(train_data_discrete, pc=1) + + +def create_cpds(data, pc=0): + + df = data.copy(deep=True) # type: pd.DataFrame + + df_vals = {col: list(df[col].unique()) for col in df.columns} + for _, vals in df_vals.items(): + vals.sort() + + cpd_a = np.array( + [ + [ + (len(df[(df["a"] == a) & (df["b"] == b) & (df["d"] == d)]) + pc) + / (len(df[(df["b"] == b) & (df["d"] == d)]) + (pc * len(df_vals["a"]))) + for b in df_vals["b"] + for d in df_vals["d"] + ] + for a in df_vals["a"] + ] + ) + + cpd_b = np.array( + [ + [ + (len(df[(df["b"] == b) & (df["d"] == d) & (df["e"] == e)]) + pc) + / (len(df[(df["d"] == d) & (df["e"] == e)]) + (pc * len(df_vals["b"]))) + for d in df_vals["d"] + for e in df_vals["e"] + ] + for b in df_vals["b"] + ] + ) + + cpd_c = np.array( + [ + [ + ( + ( + len( + df[ + (df["c"] == c) + & (df["b"] == b) + & (df["d"] == d) + & (df["e"] == e) + ] + ) + + pc + ) + / ( + len(df[(df["b"] == b) & (df["d"] == d) & (df["e"] == e)]) + + (pc * len(df_vals["c"])) + ) + ) + if not df[(df["b"] == b) & (df["d"] == d) & (df["e"] == e)].empty + else (1 / len(df_vals["c"])) + for b in df_vals["b"] + for d in df_vals["d"] + for e in df_vals["e"] + ] + for c in df_vals["c"] + ] + ) + + cpd_d = np.array( + [ + [(len(df[df["d"] == d]) + pc) / (len(df) + (pc * len(df_vals["d"])))] + for d in df_vals["d"] + ] + ) + + cpd_e = np.array( + [ + [(len(df[df["e"] == e]) + pc) / (len(df) + (pc * len(df_vals["e"])))] + for e in df_vals["e"] + ] + ) + + return {"a": cpd_a, "b": cpd_b, "c": cpd_c, "d": cpd_d, "e": cpd_e} + + +@pytest.fixture +def train_data_idx_marginals(train_data_idx_cpds): + + return create_marginals( + train_data_idx_cpds, + { + "a": list(range(4)), + "b": list(range(3)), + "c": list(range(5)), + "d": list(range(2)), + "e": list(range(2)), + }, + ) + + +@pytest.fixture +def train_data_discrete_marginals(train_data_discrete_cpds): + + return create_marginals( + train_data_discrete_cpds, + { + "a": ["a", "b", "c", "d"], + "b": ["x", "y", "z"], + "c": [0, 1, 2, 3, 4], + "d": [False, True], + "e": [False, True], + }, + ) + + +def create_marginals(cpds, data_vals): + cpd_d = cpds["d"] + p_d = {i: cpd_d[i, 0] for i in range(len(cpd_d))} + + cpd_e = cpds["e"] + p_e = {i: cpd_e[i, 0] for i in range(len(cpd_e))} + + cpd_b = cpds["b"] + c_b = np.array( + [ + [p_d[d] * p_e[e] for d in range(len(cpd_d)) for e in range(len(cpd_e))] + for _ in range(len(cpd_b)) + ] + ) + p_b = dict(enumerate((c_b * cpd_b).sum(axis=1))) + + cpd_a = cpds["a"] + c_a = np.array( + [ + [p_b[b] * p_d[d] for b in range(len(cpd_b)) for d in range(len(cpd_d))] + for _ in range(len(cpd_a)) + ] + ) + p_a = dict(enumerate((c_a * cpd_a).sum(axis=1))) + + cpd_c = cpds["c"] + c_c = np.array( + [ + [ + p_b[b] * p_d[d] * p_e[e] + for b in range(len(cpd_b)) + for d in range(len(cpd_d)) + for e in range(len(cpd_e)) + ] + for _ in range(len(cpd_c)) + ] + ) + p_c = dict(enumerate((c_c * cpd_c).sum(axis=1))) + + marginals = { + "a": {data_vals["a"][k]: v for k, v in p_a.items()}, + "b": {data_vals["b"][k]: v for k, v in p_b.items()}, + "c": {data_vals["c"][k]: v for k, v in p_c.items()}, + "d": {data_vals["d"][k]: v for k, v in p_d.items()}, + "e": {data_vals["e"][k]: v for k, v in p_e.items()}, + } + + return marginals + + +@pytest.fixture +def test_data_c() -> pd.DataFrame: + """Test data created so that C should be perfectly predicted based on train_data_cpds. + + Given the two independent variables are set randomly (d, e), all other variables are set to be + from the category with maximum likelihood in train_data_cpds""" + + data_arr = [ + ["a", "x", 1, False, False], + ["b", "x", 2, False, True], + ["c", "x", 3, True, False], + ["d", "x", 4, True, True], + ["d", "y", 1, False, False], + ["c", "y", 2, False, True], + ["b", "y", 23, True, False], + ["a", "y", 64, True, True], + ["c", "z", 1, False, False], + ["a", "z", 2, False, True], + ["d", "z", 3, True, False], + ["b", "z", 0, True, True], + ] + + data = pd.DataFrame(data_arr, columns=["a", "b", "c", "d", "e"]) + return data + + +@pytest.fixture +def test_data_c_discrete(test_data_c) -> pd.DataFrame: + """Test data C that has been discretised (see train_data_discrete)""" + df = test_data_c.copy(deep=True) # type: pd.DataFrame + df["c"] = df["c"].apply( + lambda c: 0 if c < 20 else 1 if c < 40 else 2 if c < 60 else 3 if c < 80 else 4 + ) + return df + + +@pytest.fixture +def test_data_c_likelihood(train_data_discrete_cpds) -> pd.DataFrame: + """Marginal likelihoods for train_data in train_model""" + + # Known bug in pylint with generated Dict: https://github.com/PyCQA/pylint/issues/1498 + data_arr = [ + [ + (train_data_discrete_cpds["c"])[ # pylint: disable=unsubscriptable-object + y, x + ] + for y in range( + len( + # pylint: disable=unsubscriptable-object + train_data_discrete_cpds["c"] + ) + ) + ] + for x in range(len(train_data_discrete_cpds["c"][0])) + ] + + likelihood = pd.DataFrame(data_arr, columns=["c_0", "c_1", "c_2", "c_3", "c_4"]) + return likelihood + + +@pytest.fixture +def bn(train_data_idx, train_data_discrete) -> BayesianNetwork: + return BayesianNetwork( + from_pandas(train_data_idx, w_threshold=0.3) + ).fit_node_states_and_cpds(train_data_discrete) diff --git a/tests/contrib/__init__.py b/tests/contrib/__init__.py new file mode 100644 index 0000000..5da8261 --- /dev/null +++ b/tests/contrib/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/ebaybbn/__init__.py b/tests/ebaybbn/__init__.py new file mode 100644 index 0000000..5da8261 --- /dev/null +++ b/tests/ebaybbn/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/ebaybbn/conftest.py b/tests/ebaybbn/conftest.py new file mode 100644 index 0000000..b106bdc --- /dev/null +++ b/tests/ebaybbn/conftest.py @@ -0,0 +1,205 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# The methods found in this file are adapted from a repository under Apache 2.0: +# eBay's Pythonic Bayesian Belief Network Framework. +# @online{ +# author = {Neville Newey,Anzar Afaq}, +# title = {bayesian-belief-networks}, +# organisation = {eBay}, +# codebase = {https://github.com/eBay/bayesian-belief-networks}, +# } +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from causalnex.ebaybbn import ( + BBN, + Node, + build_bbn, + build_join_tree, + combinations, + make_moralized_copy, + make_undirected_copy, +) +from causalnex.ebaybbn.utils import make_key + + +@pytest.fixture +def sprinkler_graph(): + """The Sprinkler Example as a BBN + to be used in tests. + """ + cloudy = Node("Cloudy") + sprinkler = Node("Sprinkler") + rain = Node("Rain") + wet_grass = Node("WetGrass") + cloudy.children = [sprinkler, rain] + sprinkler.parents = [cloudy] + sprinkler.children = [wet_grass] + rain.parents = [cloudy] + rain.children = [wet_grass] + wet_grass.parents = [sprinkler, rain] + bbn = BBN(dict(cloudy=cloudy, sprinkler=sprinkler, rain=rain, wet_grass=wet_grass)) + return bbn + + +@pytest.fixture +def sprinkler_bbn(): + """Sprinkler BBN built with build_bbn.""" + + def f_rain(rain): + if rain is True: + return 0.2 + return 0.8 + + def f_sprinkler(rain, sprinkler): + sprinkler_dict = { + (False, True): 0.4, + (False, False): 0.6, + (True, True): 0.01, + (True, False): 0.99, + } + return sprinkler_dict[(rain, sprinkler)] + + def f_grass_wet(sprinkler, rain, grass_wet): + table = dict() + table["fft"] = 0.0 + table["fff"] = 1.0 + table["ftt"] = 0.8 + table["ftf"] = 0.2 + table["tft"] = 0.9 + table["tff"] = 0.1 + table["ttt"] = 0.99 + table["ttf"] = 0.01 + return table[make_key(sprinkler, rain, grass_wet)] + + return build_bbn(f_rain, f_sprinkler, f_grass_wet) + + +@pytest.fixture +def huang_darwiche_nodes(): + """The nodes for the Huang Darwich example""" + + def f_a(a): + if a: + return 1 / 2 + return 1 / 2 + + def f_b(a, b): + tt = dict(tt=0.5, ft=0.4, tf=0.5, ff=0.6) + return tt[make_key(a, b)] + + def f_c(a, c): + tt = dict(tt=0.7, ft=0.2, tf=0.3, ff=0.8) + return tt[make_key(a, c)] + + def f_d(b, d): + tt = dict(tt=0.9, ft=0.5, tf=0.1, ff=0.5) + return tt[make_key(b, d)] + + def f_e(c, e): + tt = dict(tt=0.3, ft=0.6, tf=0.7, ff=0.4) + return tt[make_key(c, e)] + + def f_f(d, e, f): + tt = dict( + ttt=0.01, + ttf=0.99, + tft=0.01, + tff=0.99, + ftt=0.01, + ftf=0.99, + fft=0.99, + fff=0.01, + ) + return tt[make_key(d, e, f)] + + def f_g(c, g): + tt = dict(tt=0.8, tf=0.2, ft=0.1, ff=0.9) + return tt[make_key(c, g)] + + def f_h(e, g, h): + tt = dict( + ttt=0.05, + ttf=0.95, + tft=0.95, + tff=0.05, + ftt=0.95, + ftf=0.05, + fft=0.95, + fff=0.05, + ) + return tt[make_key(e, g, h)] + + return [f_a, f_b, f_c, f_d, f_e, f_f, f_g, f_h] + + +@pytest.fixture +def huang_darwiche_dag(huang_darwiche_nodes): + + nodes = huang_darwiche_nodes + return build_bbn(nodes) + + +@pytest.fixture +def huang_darwiche_moralized(huang_darwiche_dag): + + dag = huang_darwiche_dag + gu = make_undirected_copy(dag) + gm = make_moralized_copy(gu, dag) + + return gm + + +@pytest.fixture +def huang_darwiche_jt(huang_darwiche_dag): + def priority_func_override(node): + introduced_arcs = 0 + cluster = [node] + node.neighbours + for node_a, node_b in combinations(cluster, 2): + if node_a not in node_b.neighbours: + assert node_b not in node_a.neighbours + introduced_arcs += 1 + introduced_arcs_dict = { + "f_h": [introduced_arcs, 0], + "f_g": [introduced_arcs, 1], + "f_c": [introduced_arcs, 2], + "f_b": [introduced_arcs, 3], + "f_d": [introduced_arcs, 4], + "f_e": [introduced_arcs, 5], + "others": [introduced_arcs, 10], + } + if node.name in introduced_arcs_dict: + return introduced_arcs_dict[node.name] + + return introduced_arcs_dict["others"] + + dag = huang_darwiche_dag + jt = build_join_tree(dag, priority_func_override) + return jt diff --git a/tests/ebaybbn/test_ebaybbn.py b/tests/ebaybbn/test_ebaybbn.py new file mode 100644 index 0000000..96e3937 --- /dev/null +++ b/tests/ebaybbn/test_ebaybbn.py @@ -0,0 +1,619 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# The methods found in this file are adapted from a repository under Apache 2.0: +# eBay's Pythonic Bayesian Belief Network Framework. +# @online{ +# author = {Neville Newey,Anzar Afaq}, +# title = {bayesian-belief-networks}, +# organisation = {eBay}, +# codebase = {https://github.com/eBay/bayesian-belief-networks}, +# } +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import division + +import copy +from collections import Counter + +import pytest + +from causalnex.ebaybbn import ( + BBNNode, + JoinTree, + JoinTreeCliqueNode, + SepSet, + build_bbn, + build_bbn_from_conditionals, + build_join_tree, + combinations, + make_moralized_copy, + make_node_func, + make_undirected_copy, + priority_func, + triangulate, +) +from causalnex.ebaybbn.exceptions import ( + VariableNotInGraphError, + VariableValueNotInDomainError, +) +from causalnex.ebaybbn.graph import Node, UndirectedNode +from causalnex.ebaybbn.utils import get_args, get_original_factors, make_key + + +def r3(x): + return round(x, 3) + + +def r5(x): + return round(x, 5) + + +class TestBBN: + def test_get_graphviz_source(self, sprinkler_graph): + gv_src = """digraph G { + graph [ dpi = 300 bgcolor="transparent" rankdir="LR"]; + Cloudy [ shape="ellipse" color="blue"]; + Rain [ shape="ellipse" color="blue"]; + Sprinkler [ shape="ellipse" color="blue"]; + WetGrass [ shape="ellipse" color="blue"]; + Cloudy -> Rain; + Cloudy -> Sprinkler; + Rain -> WetGrass; + Sprinkler -> WetGrass; +} +""" + assert sprinkler_graph.get_graphviz_source() == gv_src + + def test_get_original_factors(self, huang_darwiche_nodes): + + original_factors = get_original_factors(huang_darwiche_nodes) + assert original_factors["a"] == huang_darwiche_nodes[0] + assert original_factors["b"] == huang_darwiche_nodes[1] + assert original_factors["c"] == huang_darwiche_nodes[2] + assert original_factors["d"] == huang_darwiche_nodes[3] + assert original_factors["e"] == huang_darwiche_nodes[4] + assert original_factors["f"] == huang_darwiche_nodes[5] + assert original_factors["g"] == huang_darwiche_nodes[6] + assert original_factors["h"] == huang_darwiche_nodes[7] + + def test_build_graph(self, huang_darwiche_nodes): + bbn = build_bbn(huang_darwiche_nodes) + nodes = {node.name: node for node in bbn.nodes} + assert nodes["f_a"].parents == [] + assert nodes["f_b"].parents == [nodes["f_a"]] + assert nodes["f_c"].parents == [nodes["f_a"]] + assert nodes["f_d"].parents == [nodes["f_b"]] + assert nodes["f_e"].parents == [nodes["f_c"]] + assert nodes["f_f"].parents == [nodes["f_d"], nodes["f_e"]] + assert nodes["f_g"].parents == [nodes["f_c"]] + assert nodes["f_h"].parents == [nodes["f_e"], nodes["f_g"]] + + def test_make_undirecred_copy(self, huang_darwiche_dag): + ug = make_undirected_copy(huang_darwiche_dag) + nodes = {node.name: node for node in ug.nodes} + assert set(nodes["f_a"].neighbours) == set([nodes["f_b"], nodes["f_c"]]) + assert set(nodes["f_b"].neighbours) == set([nodes["f_a"], nodes["f_d"]]) + assert set(nodes["f_c"].neighbours) == set( + [nodes["f_a"], nodes["f_e"], nodes["f_g"]] + ) + assert set(nodes["f_d"].neighbours) == set([nodes["f_b"], nodes["f_f"]]) + assert set(nodes["f_e"].neighbours) == set( + [nodes["f_c"], nodes["f_f"], nodes["f_h"]] + ) + assert set(nodes["f_f"].neighbours) == set([nodes["f_d"], nodes["f_e"]]) + assert set(nodes["f_g"].neighbours) == set([nodes["f_c"], nodes["f_h"]]) + assert set(nodes["f_h"].neighbours) == set([nodes["f_e"], nodes["f_g"]]) + + def test_make_moralized_copy(self, huang_darwiche_dag): + gu = make_undirected_copy(huang_darwiche_dag) + gm = make_moralized_copy(gu, huang_darwiche_dag) + nodes = {node.name: node for node in gm.nodes} + assert set(nodes["f_a"].neighbours) == set([nodes["f_b"], nodes["f_c"]]) + assert set(nodes["f_b"].neighbours) == set([nodes["f_a"], nodes["f_d"]]) + assert set(nodes["f_c"].neighbours) == set( + [nodes["f_a"], nodes["f_e"], nodes["f_g"]] + ) + assert set(nodes["f_d"].neighbours) == set( + [nodes["f_b"], nodes["f_f"], nodes["f_e"]] + ) + assert set(nodes["f_e"].neighbours) == set( + [nodes["f_c"], nodes["f_f"], nodes["f_h"], nodes["f_d"], nodes["f_g"]] + ) + assert set(nodes["f_f"].neighbours) == set([nodes["f_d"], nodes["f_e"]]) + assert set(nodes["f_g"].neighbours) == set( + [nodes["f_c"], nodes["f_h"], nodes["f_e"]] + ) + assert set(nodes["f_h"].neighbours) == set([nodes["f_e"], nodes["f_g"]]) + + def test_triangulate(self, huang_darwiche_moralized): + + # Because of ties in the priority q we will + # override the priority function here to + # insert tie breakers to ensure the same + # elimination ordering as Darwich Huang. + def priority_func_override(node): + introduced_arcs = 0 + cluster = [node] + node.neighbours + for node_a, node_b in combinations(cluster, 2): + if node_a not in node_b.neighbours: + assert node_b not in node_a.neighbours + introduced_arcs += 1 + introduced_arcs_dict = { + "f_h": [introduced_arcs, 0], + "f_g": [introduced_arcs, 1], + "f_c": [introduced_arcs, 2], + "f_b": [introduced_arcs, 3], + "f_d": [introduced_arcs, 4], + "f_e": [introduced_arcs, 5], + "others": [introduced_arcs, 10], + } + if node.name in introduced_arcs_dict: + return introduced_arcs_dict[node.name] + + return introduced_arcs_dict["others"] + + cliques, elimination_ordering = triangulate( + huang_darwiche_moralized, priority_func_override + ) + nodes = {node.name: node for node in huang_darwiche_moralized.nodes} + assert len(cliques) == 6 + assert cliques[0].nodes == set([nodes["f_e"], nodes["f_g"], nodes["f_h"]]) + assert cliques[1].nodes == set([nodes["f_c"], nodes["f_e"], nodes["f_g"]]) + assert cliques[2].nodes == set([nodes["f_d"], nodes["f_e"], nodes["f_f"]]) + assert cliques[3].nodes == set([nodes["f_a"], nodes["f_c"], nodes["f_e"]]) + assert cliques[4].nodes == set([nodes["f_a"], nodes["f_b"], nodes["f_d"]]) + assert cliques[5].nodes == set([nodes["f_a"], nodes["f_d"], nodes["f_e"]]) + + assert elimination_ordering == [ + "f_h", + "f_g", + "f_f", + "f_c", + "f_b", + "f_d", + "f_e", + "f_a", + ] + # Now lets ensure the triangulated graph is + # the same as Darwiche Huang fig. 2 pg. 13 + nodes = {node.name: node for node in huang_darwiche_moralized.nodes} + assert set(nodes["f_a"].neighbours) == set( + [nodes["f_b"], nodes["f_c"], nodes["f_d"], nodes["f_e"]] + ) + assert set(nodes["f_b"].neighbours) == set([nodes["f_a"], nodes["f_d"]]) + assert set(nodes["f_c"].neighbours) == set( + [nodes["f_a"], nodes["f_e"], nodes["f_g"]] + ) + assert set(nodes["f_d"].neighbours) == set( + [nodes["f_b"], nodes["f_f"], nodes["f_e"], nodes["f_a"]] + ) + assert set(nodes["f_e"].neighbours) == set( + [ + nodes["f_c"], + nodes["f_f"], + nodes["f_h"], + nodes["f_d"], + nodes["f_g"], + nodes["f_a"], + ] + ) + assert set(nodes["f_f"].neighbours) == set([nodes["f_d"], nodes["f_e"]]) + assert set(nodes["f_g"].neighbours) == set( + [nodes["f_c"], nodes["f_h"], nodes["f_e"]] + ) + assert set(nodes["f_h"].neighbours) == set([nodes["f_e"], nodes["f_g"]]) + + def test_triangulate_no_tie_break(self, huang_darwiche_moralized): + # Now lets see what happens if + # we dont enforce the tie-breakers... + # It seems the triangulated graph is + # different adding edges from d to c + # and b to c + # Will be interesting to see whether + # inference will still be correct. + triangulate(huang_darwiche_moralized) + nodes = {node.name: node for node in huang_darwiche_moralized.nodes} + assert set(nodes["f_a"].neighbours) == set([nodes["f_b"], nodes["f_c"]]) + assert set(nodes["f_b"].neighbours) == set( + [nodes["f_a"], nodes["f_d"], nodes["f_c"]] + ) + assert set(nodes["f_c"].neighbours) == set( + [nodes["f_a"], nodes["f_e"], nodes["f_g"], nodes["f_b"], nodes["f_d"]] + ) + assert set(nodes["f_d"].neighbours) == set( + [nodes["f_b"], nodes["f_f"], nodes["f_e"], nodes["f_c"]] + ) + assert set(nodes["f_e"].neighbours) == set( + [nodes["f_c"], nodes["f_f"], nodes["f_h"], nodes["f_d"], nodes["f_g"]] + ) + assert set(nodes["f_f"].neighbours) == set([nodes["f_d"], nodes["f_e"]]) + assert set(nodes["f_g"].neighbours) == set( + [nodes["f_c"], nodes["f_h"], nodes["f_e"]] + ) + assert set(nodes["f_h"].neighbours) == set([nodes["f_e"], nodes["f_g"]]) + + def test_build_join_tree(self, huang_darwiche_dag): + def priority_func_override(node): + introduced_arcs = 0 + cluster = [node] + node.neighbours + for node_a, node_b in combinations(cluster, 2): + if node_a not in node_b.neighbours: + assert node_b not in node_a.neighbours + introduced_arcs += 1 + introduced_arcs_dict = { + "f_h": [introduced_arcs, 0], + "f_g": [introduced_arcs, 1], + "f_c": [introduced_arcs, 2], + "f_b": [introduced_arcs, 3], + "f_d": [introduced_arcs, 4], + "f_e": [introduced_arcs, 5], + "others": [introduced_arcs, 10], + } + if node.name in introduced_arcs_dict: + return introduced_arcs_dict[node.name] + + return introduced_arcs_dict["others"] + + jt = build_join_tree(huang_darwiche_dag, priority_func_override) + for node in jt.sepset_nodes: + assert {n.clique for n in node.neighbours} == {node.sepset.X, node.sepset.Y} + # clique nodes. + + def test_initialize_potentials(self, huang_darwiche_jt, huang_darwiche_dag): + # Seems like there can be multiple assignments so + # for this test we will set the assignments explicitely + cliques = {node.name: node for node in huang_darwiche_jt.nodes} + bbn_nodes = {node.name: node for node in huang_darwiche_dag.nodes} + assignments = { + cliques["Clique_ACE"]: [bbn_nodes["f_c"], bbn_nodes["f_e"]], + cliques["Clique_ABD"]: [ + bbn_nodes["f_a"], + bbn_nodes["f_b"], + bbn_nodes["f_d"], + ], + } + huang_darwiche_jt.initialize_potentials(assignments, huang_darwiche_dag) + for node in huang_darwiche_jt.sepset_nodes: + for v in node.potential_tt.values(): + assert v == 1 + + # Note that in H&D there are two places that show + # initial potentials, one is for ABD and AD + # and the second is for ACE and CE + # We should test both here but we must enforce + # the assignments above because alternate and + # equally correct Junction Trees will give + # different potentials. + def r(x): + return round(x, 3) + + tt = cliques["Clique_ACE"].potential_tt + assert r(tt[("a", True), ("c", True), ("e", True)]) == 0.21 + assert r(tt[("a", True), ("c", True), ("e", False)]) == 0.49 + assert r(tt[("a", True), ("c", False), ("e", True)]) == 0.18 + assert r(tt[("a", True), ("c", False), ("e", False)]) == 0.12 + assert r(tt[("a", False), ("c", True), ("e", True)]) == 0.06 + assert r(tt[("a", False), ("c", True), ("e", False)]) == 0.14 + assert r(tt[("a", False), ("c", False), ("e", True)]) == 0.48 + assert r(tt[("a", False), ("c", False), ("e", False)]) == 0.32 + + tt = cliques["Clique_ABD"].potential_tt + assert r(tt[("a", True), ("b", True), ("d", True)]) == 0.225 + assert r(tt[("a", True), ("b", True), ("d", False)]) == 0.025 + assert r(tt[("a", True), ("b", False), ("d", True)]) == 0.125 + assert r(tt[("a", True), ("b", False), ("d", False)]) == 0.125 + assert r(tt[("a", False), ("b", True), ("d", True)]) == 0.180 + assert r(tt[("a", False), ("b", True), ("d", False)]) == 0.020 + assert r(tt[("a", False), ("b", False), ("d", True)]) == 0.150 + assert r(tt[("a", False), ("b", False), ("d", False)]) == 0.150 + + def test_jtclique_node_variable_names(self, huang_darwiche_jt): + for node in huang_darwiche_jt.clique_nodes: + if "ADE" in node.name: + assert set(node.variable_names) == set(["a", "d", "e"]) + + def test_propagate(self, huang_darwiche_jt, huang_darwiche_dag): + jt_cliques = {node.name: node for node in huang_darwiche_jt.clique_nodes} + assignments = huang_darwiche_jt.assign_clusters(huang_darwiche_dag) + huang_darwiche_jt.initialize_potentials(assignments, huang_darwiche_dag) + + huang_darwiche_jt.propagate(starting_clique=jt_cliques["Clique_ACE"]) + tt = jt_cliques["Clique_DEF"].potential_tt + assert r5(tt[(("d", False), ("e", True), ("f", True))]) == 0.00150 + assert r5(tt[(("d", True), ("e", False), ("f", True))]) == 0.00365 + assert r5(tt[(("d", False), ("e", False), ("f", True))]) == 0.16800 + assert r5(tt[(("d", True), ("e", True), ("f", True))]) == 0.00315 + assert r5(tt[(("d", False), ("e", False), ("f", False))]) == 0.00170 + assert r5(tt[(("d", True), ("e", True), ("f", False))]) == 0.31155 + assert r5(tt[(("d", False), ("e", True), ("f", False))]) == 0.14880 + assert r5(tt[(("d", True), ("e", False), ("f", False))]) == 0.36165 + + def test_marginal(self, huang_darwiche_jt, huang_darwiche_dag): + # The remaining marginals here come + # from the module itself, however they + # have been corrobarted by running + # inference using the sampling inference + # engine and the same results are + # achieved. + """ + +------+-------+----------+ + | Node | Value | Marginal | + +------+-------+----------+ + | a | False | 0.500000 | + | a | True | 0.500000 | + | b | False | 0.550000 | + | b | True | 0.450000 | + | c | False | 0.550000 | + | c | True | 0.450000 | + | d | False | 0.320000 | + | d | True | 0.680000 | + | e | False | 0.535000 | + | e | True | 0.465000 | + | f | False | 0.823694 | + | f | True | 0.176306 | + | g | False | 0.585000 | + | g | True | 0.415000 | + | h | False | 0.176900 | + | h | True | 0.823100 | + +------+-------+----------+ + """ + bbn_nodes = {node.name: node for node in huang_darwiche_dag.nodes} + assignments = huang_darwiche_jt.assign_clusters(huang_darwiche_dag) + huang_darwiche_jt.initialize_potentials(assignments, huang_darwiche_dag) + huang_darwiche_jt.propagate() + + # These test values come directly from + # pg. 22 of H & D + p_A = huang_darwiche_jt.marginal(bbn_nodes["f_a"]) + assert r3(p_A[(("a", True),)]) == 0.5 + assert r3(p_A[(("a", False),)]) == 0.5 + + p_D = huang_darwiche_jt.marginal(bbn_nodes["f_d"]) + assert r3(p_D[(("d", True),)]) == 0.68 + assert r3(p_D[(("d", False),)]) == 0.32 + + p_B = huang_darwiche_jt.marginal(bbn_nodes["f_b"]) + assert r3(p_B[(("b", True),)]) == 0.45 + assert r3(p_B[(("b", False),)]) == 0.55 + + p_C = huang_darwiche_jt.marginal(bbn_nodes["f_c"]) + assert r3(p_C[(("c", True),)]) == 0.45 + assert r3(p_C[(("c", False),)]) == 0.55 + + p_E = huang_darwiche_jt.marginal(bbn_nodes["f_e"]) + assert r3(p_E[(("e", True),)]) == 0.465 + assert r3(p_E[(("e", False),)]) == 0.535 + + p_F = huang_darwiche_jt.marginal(bbn_nodes["f_f"]) + assert r3(p_F[(("f", True),)]) == 0.176 + assert r3(p_F[(("f", False),)]) == 0.824 + + p_G = huang_darwiche_jt.marginal(bbn_nodes["f_g"]) + assert r3(p_G[(("g", True),)]) == 0.415 + assert r3(p_G[(("g", False),)]) == 0.585 + + p_H = huang_darwiche_jt.marginal(bbn_nodes["f_h"]) + assert r3(p_H[(("h", True),)]) == 0.823 + assert r3(p_H[(("h", False),)]) == 0.177 + + +def test_make_node_func(): + UPDATE = { + "prize_door": [ + # For nodes that have no parents + # use the empty list to specify + # the conditioned upon variables + # ie conditioned on the empty set + [[], {"A": 1 / 3, "B": 1 / 3, "C": 1 / 3}] + ], + "guest_door": [[[], {"A": 1 / 3, "B": 1 / 3, "C": 1 / 3}]], + "monty_door": [ + [[["prize_door", "A"], ["guest_door", "A"]], {"A": 0, "B": 0.5, "C": 0.5}], + [[["prize_door", "A"], ["guest_door", "B"]], {"A": 0, "B": 0, "C": 1}], + [[["prize_door", "A"], ["guest_door", "C"]], {"A": 0, "B": 1, "C": 0}], + [[["prize_door", "B"], ["guest_door", "A"]], {"A": 0, "B": 0, "C": 1}], + [[["prize_door", "B"], ["guest_door", "B"]], {"A": 0.5, "B": 0, "C": 0.5}], + [[["prize_door", "B"], ["guest_door", "C"]], {"A": 1, "B": 0, "C": 0}], + [[["prize_door", "C"], ["guest_door", "A"]], {"A": 0, "B": 1, "C": 0}], + [[["prize_door", "C"], ["guest_door", "B"]], {"A": 1, "B": 0, "C": 0}], + [[["prize_door", "C"], ["guest_door", "C"]], {"A": 0.5, "B": 0.5, "C": 0}], + ], + } + + node_func = make_node_func("prize_door", UPDATE["prize_door"]) + assert get_args(node_func) == ["prize_door"] + assert node_func("A") == 1 / 3 + assert node_func("B") == 1 / 3 + assert node_func("C") == 1 / 3 + + node_func = make_node_func("guest_door", UPDATE["guest_door"]) + assert get_args(node_func) == ["guest_door"] + assert node_func("A") == 1 / 3 + assert node_func("B") == 1 / 3 + assert node_func("C") == 1 / 3 + + node_func = make_node_func("monty_door", UPDATE["monty_door"]) + assert get_args(node_func) == ["guest_door", "prize_door", "monty_door"] + assert node_func("A", "A", "A") == 0 + assert node_func("A", "A", "B") == 0.5 + assert node_func("A", "A", "C") == 0.5 + assert node_func("A", "B", "A") == 0 + assert node_func("A", "B", "B") == 0 + assert node_func("A", "B", "C") == 1 + assert node_func("A", "C", "A") == 0 + assert node_func("A", "C", "B") == 1 + assert node_func("A", "C", "C") == 0 + assert node_func("B", "A", "A") == 0 + assert node_func("B", "A", "B") == 0 + assert node_func("B", "A", "C") == 1 + assert node_func("B", "B", "A") == 0.5 + assert node_func("B", "B", "B") == 0 + assert node_func("B", "B", "C") == 0.5 + assert node_func("B", "C", "A") == 1 + assert node_func("B", "C", "B") == 0 + assert node_func("B", "C", "C") == 0 + assert node_func("C", "A", "A") == 0 + assert node_func("C", "A", "B") == 1 + assert node_func("C", "A", "C") == 0 + assert node_func("C", "B", "A") == 1 + assert node_func("C", "B", "B") == 0 + assert node_func("C", "B", "C") == 0 + assert node_func("C", "C", "A") == 0.5 + assert node_func("C", "C", "B") == 0.5 + assert node_func("C", "C", "C") == 0 + + +def close_enough(x, y, r=3): + return round(x, r) == round(y, r) + + +def test_build_bbn_from_conditionals(): + UPDATE = { + "prize_door": [ + # For nodes that have no parents + # use the empty list to specify + # the conditioned upon variables + # ie conditioned on the empty set + [[], {"A": 1 / 3, "B": 1 / 3, "C": 1 / 3}] + ], + "guest_door": [[[], {"A": 1 / 3, "B": 1 / 3, "C": 1 / 3}]], + "monty_door": [ + [[["prize_door", "A"], ["guest_door", "A"]], {"A": 0, "B": 0.5, "C": 0.5}], + [[["prize_door", "A"], ["guest_door", "B"]], {"A": 0, "B": 0, "C": 1}], + [[["prize_door", "A"], ["guest_door", "C"]], {"A": 0, "B": 1, "C": 0}], + [[["prize_door", "B"], ["guest_door", "A"]], {"A": 0, "B": 0, "C": 1}], + [[["prize_door", "B"], ["guest_door", "B"]], {"A": 0.5, "B": 0, "C": 0.5}], + [[["prize_door", "B"], ["guest_door", "C"]], {"A": 1, "B": 0, "C": 0}], + [[["prize_door", "C"], ["guest_door", "A"]], {"A": 0, "B": 1, "C": 0}], + [[["prize_door", "C"], ["guest_door", "B"]], {"A": 1, "B": 0, "C": 0}], + [[["prize_door", "C"], ["guest_door", "C"]], {"A": 0.5, "B": 0.5, "C": 0}], + ], + } + g = build_bbn_from_conditionals(UPDATE) + result = g.query() + assert close_enough(result[("guest_door", "A")], 0.333) + assert close_enough(result[("guest_door", "B")], 0.333) + assert close_enough(result[("guest_door", "C")], 0.333) + assert close_enough(result[("monty_door", "A")], 0.333) + assert close_enough(result[("monty_door", "B")], 0.333) + assert close_enough(result[("monty_door", "C")], 0.333) + assert close_enough(result[("prize_door", "A")], 0.333) + assert close_enough(result[("prize_door", "B")], 0.333) + assert close_enough(result[("prize_door", "C")], 0.333) + + result = g.query(guest_door="A", monty_door="B") + assert close_enough(result[("guest_door", "A")], 1) + assert close_enough(result[("guest_door", "B")], 0) + assert close_enough(result[("guest_door", "C")], 0) + assert close_enough(result[("monty_door", "A")], 0) + assert close_enough(result[("monty_door", "B")], 1) + assert close_enough(result[("monty_door", "C")], 0) + assert close_enough(result[("prize_door", "A")], 0.333) + assert close_enough(result[("prize_door", "B")], 0) + assert close_enough(result[("prize_door", "C")], 0.667) + + +def valid_sample(samples, query_result): + """For a group of samples from + a query result ensure that + the sample is approximately equivalent + to the query_result which is the + true distribution.""" + counts = Counter() + for sample in samples: + for var, val in sample.items(): + counts[(var, val)] += 1 + # Now lets normalize for each count... + differences = [] + for k, v in counts.items(): + counts[k] = v / len(samples) + difference = abs(counts.get(k, 0) - query_result[k]) + differences.append(difference) + return all([not round(difference, 2) > 0.01 for difference in differences]) + + +def test_draw_sample_sprinkler(sprinkler_bbn): + + query_result = sprinkler_bbn.query() + samples = sprinkler_bbn.draw_samples({}, 10000) + assert valid_sample(samples, query_result) + + +def test_repr(): + + assert repr(Node("test")) == "<Node test>" + assert repr(UndirectedNode("test")) == "<UndirectedNode test>" + assert ( + repr(BBNNode(get_original_factors)) + == "<BBNNode get_original_factors (['factors'])>" + ) + assert ( + repr(JoinTreeCliqueNode(UndirectedNode("test"))) + == "<JoinTreeCliqueNode: <UndirectedNode test>>" + ) + + +def test_exception(sprinkler_bbn): + with pytest.raises(VariableValueNotInDomainError): + sprinkler_bbn.query(rain="No") + with pytest.raises(VariableNotInGraphError): + sprinkler_bbn.query(sunny="True") + + +def test_make_key(): + class DummyTest: + def __init__(self, value): + + self.value = value + + def dummy_method(self, value): # Add this method to by pass linting + self.value = value + + test = DummyTest(8) + test.dummy_method(10) + with pytest.raises(ValueError, match=r"Unexpected type"): + make_key(test) + + +def test_insert_duplicate_clique(huang_darwiche_moralized): + + cliques, _ = triangulate(huang_darwiche_moralized, priority_func) + + forest = set() + for clique in cliques: + jt_node = JoinTreeCliqueNode(clique) + clique.node = jt_node + tree = JoinTree([jt_node]) + forest.add(tree) + + s = SepSet(cliques[0], cliques[0]) + assert s.insertable(forest) is False + s_copy = copy.deepcopy(s) + s.insert(forest) + assert len(s.X.node.neighbours) > len(s_copy.X.node.neighbours) diff --git a/tests/structure/__init__.py b/tests/structure/__init__.py new file mode 100644 index 0000000..5da8261 --- /dev/null +++ b/tests/structure/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/structure/test_notears.py b/tests/structure/test_notears.py new file mode 100644 index 0000000..baa2b8b --- /dev/null +++ b/tests/structure/test_notears.py @@ -0,0 +1,575 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import networkx as nx +import numpy as np +import pandas as pd +import pytest + +from causalnex.structure.notears import ( + from_numpy, + from_numpy_lasso, + from_pandas, + from_pandas_lasso, +) + + +class TestFromPandas: + """Test behaviour of the from_pandas method""" + + def test_all_columns_in_structure(self, train_data_idx): + """Every columns that is in the data should become a node in the learned structure""" + + g = from_pandas(train_data_idx) + assert len(g.nodes) == len(train_data_idx.columns) + + def test_isolated_nodes_exist(self, train_data_idx): + """Isolated nodes should still be in the learned structure""" + + g = from_pandas(train_data_idx, w_threshold=1.0) + assert len(g.nodes) == len(train_data_idx.columns) + + def test_expected_structure_learned(self, train_data_idx, train_model): + """Given a small data set that can be examined by hand, the structure should be deterministic""" + + g = from_pandas(train_data_idx, w_threshold=0.3) + assert set(g.edges) == set(train_model.edges) + + def test_empty_data_raises_error(self): + """ + Providing an empty data set should result in a Value Error explaining that data must not be empty. + This error is useful to catch and handle gracefully, because otherwise the user would experience + misleading division by zero, or unpacking errors. + """ + + with pytest.raises(ValueError): + from_pandas(pd.DataFrame(data=[], columns=["a"])) + + def test_non_numeric_data_raises_error(self): + """Only numeric data frames should be supported""" + + with pytest.raises(ValueError, match="All columns must have numeric data.*"): + from_pandas(pd.DataFrame(data=["x"], columns=["a"])) + + def test_single_iter_gets_converged_fail_warnings(self, train_data_idx): + """ + With a single iteration on this dataset, learn_structure fails to converge and should give warnings. + """ + + with pytest.warns( + UserWarning, match="Failed to converge. Consider increasing max_iter." + ): + from_pandas(train_data_idx, max_iter=1) + + def test_certain_relationships_get_near_certain_weight(self): + """If observations reliably show a==b and !a==!b then the relationship from a->b should be certain""" + + data = pd.DataFrame([[0, 1] for _ in range(10)], columns=["a", "b"]) + g = from_pandas(data) + assert all( + [ + 0.99 <= weight <= 1 + for u, v, weight in g.edges(data="weight") + if u == 0 and v == 1 + ] + ) + + def test_inverse_relationships_get_negative_weight(self): + """If observations indicate a==!b and b==!a then the weight of the relationship from a-> should be negative""" + + data = pd.DataFrame([[0, 1] for _ in range(10)], columns=["a", "b"]) + data.append(pd.DataFrame([[1, 0] for _ in range(10)], columns=["a", "b"])) + g = from_pandas(data) + assert all( + [weight < 0 for u, v, weight in g.edges(data="weight") if u == 0 and v == 1] + ) + + def test_no_cycles(self, train_data_idx): + """ + The learned structure should be acyclic + """ + + g = from_pandas(train_data_idx, w_threshold=0.3) + assert nx.algorithms.is_directed_acyclic_graph(g) + + def test_tabu_edges_on_non_existing_edges_do_nothing(self, train_data_idx): + """If tabu edges do not exist in the original unconstrained network then nothing changes""" + + g1 = from_pandas(train_data_idx, w_threshold=0.3) + g2 = from_pandas( + train_data_idx, w_threshold=0.3, tabu_edges=[("a", "d"), ("e", "a")] + ) + assert set(g1.edges) == set(g2.edges) + + def test_tabu_expected_edges(self, train_data_idx): + """Tabu edges should not exist in the network""" + + tabu_e = [("d", "a"), ("b", "c")] + g = from_pandas(train_data_idx, tabu_edges=tabu_e) + assert [e not in g.edges for e in tabu_e] + + def test_tabu_expected_parent_nodes(self, train_data_idx): + """Tabu parent nodes should not have any outgoing edges""" + + tabu_p = ["a", "d", "b"] + g = from_pandas(train_data_idx, tabu_parent_nodes=tabu_p) + assert [p not in [e[0] for e in g.edges] for p in tabu_p] + + def test_tabu_expected_child_nodes(self, train_data_idx): + """Tabu child nodes should not have any ingoing edges""" + + tabu_c = ["a", "d", "b"] + g = from_pandas(train_data_idx, tabu_child_nodes=tabu_c) + assert [c not in [e[1] for e in g.edges] for c in tabu_c] + + def test_multiple_tabu(self, train_data_idx): + """Any edge related to tabu edges/parent nodes/child nodes should not exist in the network""" + + tabu_e = [("d", "a"), ("b", "c")] + tabu_p = ["b"] + tabu_c = ["a", "d"] + g = from_pandas( + train_data_idx, + tabu_edges=tabu_e, + tabu_parent_nodes=tabu_p, + tabu_child_nodes=tabu_c, + ) + assert [e not in g.edges for e in tabu_e] + assert [p not in [e[0] for e in g.edges] for p in tabu_p] + assert [c not in [e[1] for e in g.edges] for c in tabu_c] + + +class TestFromPandasLasso: + """Test behaviour of the from_pandas_lasso method""" + + def test_all_columns_in_structure(self, train_data_idx): + """Every columns that is in the data should become a node in the learned structure""" + + g = from_pandas_lasso(train_data_idx, 0.1) + assert len(g.nodes) == len(train_data_idx.columns) + + def test_isolated_nodes_exist(self, train_data_idx): + """Isolated nodes should still be in the learned structure""" + + g = from_pandas_lasso(train_data_idx, 0.1, w_threshold=1.0) + assert len(g.nodes) == len(train_data_idx.columns) + + def test_expected_structure_learned(self, train_data_idx, train_model): + """Given an extremely small alpha and small data set that can be examined by hand, + the structure should be deterministic""" + + g = from_pandas_lasso(train_data_idx, 1e-8, w_threshold=0.3) + assert set(g.edges) == set(train_model.edges) + + def test_empty_data_raises_error(self): + """ + Providing an empty data set should result in a Value Error explaining that data must not be empty. + This error is useful to catch and handle gracefully, because otherwise the user would experience + misleading division by zero, or unpacking errors. + """ + + with pytest.raises(ValueError): + from_pandas_lasso(pd.DataFrame(data=[], columns=["a"]), 0.1) + + def test_non_numeric_data_raises_error(self): + """Only numeric data frames should be supported""" + + with pytest.raises(ValueError, match="All columns must have numeric data.*"): + from_pandas_lasso(pd.DataFrame(data=["x"], columns=["a"]), 0.1) + + def test_single_iter_gets_converged_fail_warnings(self, train_data_idx): + """ + With a single iteration on this dataset, learn_structure fails to converge and should give warnings. + """ + + with pytest.warns( + UserWarning, match="Failed to converge. Consider increasing max_iter." + ): + from_pandas_lasso(train_data_idx, 0.1, max_iter=1) + + def test_certain_relationships_get_near_certain_weight(self): + """If observations reliably show a==b and !a==!b then the relationship from a->b should be certain""" + + data = pd.DataFrame([[0, 1] for _ in range(10)], columns=["a", "b"]) + g = from_pandas_lasso(data, 0.1) + assert all( + [ + 0.99 <= weight <= 1 + for u, v, weight in g.edges(data="weight") + if u == 0 and v == 1 + ] + ) + + def test_inverse_relationships_get_negative_weight(self): + """If observations indicate a==!b and b==!a then the weight of the relationship from a-> should be negative""" + + data = pd.DataFrame([[0, 1] for _ in range(10)], columns=["a", "b"]) + data.append(pd.DataFrame([[1, 0] for _ in range(10)], columns=["a", "b"])) + g = from_pandas_lasso(data, 0.1) + assert all( + [weight < 0 for u, v, weight in g.edges(data="weight") if u == 0 and v == 1] + ) + + def test_no_cycles(self, train_data_idx): + """ + The learned structure should be acyclic + """ + + g = from_pandas_lasso(train_data_idx, 0.1, w_threshold=0.3) + assert nx.algorithms.is_directed_acyclic_graph(g) + + def test_tabu_expected_edges(self, train_data_idx): + """Tabu edges should not exist in the network""" + + tabu_e = [("d", "a"), ("b", "c")] + g = from_pandas_lasso(train_data_idx, 0.1, tabu_edges=tabu_e) + assert [e not in g.edges for e in tabu_e] + + def test_tabu_expected_parent_nodes(self, train_data_idx): + """Tabu parent nodes should not have any outgoing edges""" + + tabu_p = ["a", "d", "b"] + g = from_pandas_lasso(train_data_idx, 0.1, tabu_parent_nodes=tabu_p) + assert [p not in [e[0] for e in g.edges] for p in tabu_p] + + def test_tabu_expected_child_nodes(self, train_data_idx): + """Tabu child nodes should not have any ingoing edges""" + + tabu_c = ["a", "d", "b"] + g = from_pandas_lasso(train_data_idx, 0.1, tabu_child_nodes=tabu_c) + assert [c not in [e[1] for e in g.edges] for c in tabu_c] + + def test_multiple_tabu(self, train_data_idx): + """Any edge related to tabu edges/parent nodes/child nodes should not exist in the network""" + + tabu_e = [("d", "a"), ("b", "c")] + tabu_p = ["b"] + tabu_c = ["a", "d"] + g = from_pandas_lasso( + train_data_idx, + 0.1, + tabu_edges=tabu_e, + tabu_parent_nodes=tabu_p, + tabu_child_nodes=tabu_c, + ) + assert [e not in g.edges for e in tabu_e] + assert [p not in [e[0] for e in g.edges] for p in tabu_p] + assert [c not in [e[1] for e in g.edges] for c in tabu_c] + + def test_sparsity(self, train_data_idx): + """Structure learnt from larger lambda should be sparser than smaller lambda""" + + g1 = from_pandas_lasso(train_data_idx, 0.1, w_threshold=0.3) + g2 = from_pandas_lasso(train_data_idx, 1e-6, w_threshold=0.3) + assert len(g1.edges) > len(g2.edges) + + def test_sparsity_against_without_reg(self, train_data_idx): + """Structure learnt from regularisation should be sparser than the one without""" + + g1 = from_pandas_lasso(train_data_idx, 0.1, w_threshold=0.3) + g2 = from_pandas(train_data_idx, w_threshold=0.3) + assert len(g1.edges) > len(g2.edges) + + def test_f1_score(self, train_data_idx, train_model): + """Structure learnt from regularisation should have very high f1 score relative to the ground truth""" + g = from_pandas_lasso(train_data_idx, 0.1, w_threshold=0.3) + print(sorted(list(g.edges))) + print(train_model.edges) + + n_predictions_made = len(g.edges) + n_correct_predictions = len(set(g.edges).intersection(set(train_model.edges))) + n_relevant_predictions = len(train_model.edges) + + precision = n_correct_predictions / n_predictions_made + recall = n_correct_predictions / n_relevant_predictions + f1_score = 2 * (precision * recall) / (precision + recall) + + assert f1_score > 0.8 + + +class TestFromNumpy: + """Test behaviour of the from_numpy_lasso method""" + + def test_all_columns_in_structure(self, train_data_idx): + """Every columns that is in the data should become a node in the learned structure""" + + g = from_numpy(train_data_idx.values) + assert (len(g.nodes)) == len(train_data_idx.columns) + + def test_isolated_nodes_exist(self, train_data_idx): + """Isolated nodes should still be in the learned structure""" + + g = from_numpy(train_data_idx.values, w_threshold=1.0) + assert len(g.nodes) == len(train_data_idx.columns) + + def test_expected_structure_learned(self, train_data_idx, train_model_idx): + """Given a small data set that can be examined by hand, the structure should be deterministic""" + + g = from_numpy(train_data_idx.values, w_threshold=0.3) + assert set(g.edges) == set(train_model_idx.edges) + + def test_empty_data_raises_error(self): + """ + Providing an empty data set should result in a Value Error explaining that data must not be empty. + This error is useful to catch and handle gracefully, because otherwise the user would experience + misleading division by zero, or unpacking errors. + """ + + with pytest.raises(ValueError): + from_numpy(np.empty([0, 5])) + + def test_single_iter_gets_converged_fail_warnings(self, train_data_idx): + """ + With a single iteration on this dataset, learn_structure fails to converge and should give warnings. + """ + + with pytest.warns( + UserWarning, match="Failed to converge. Consider increasing max_iter." + ): + from_numpy(train_data_idx.values, max_iter=1) + + def test_certain_relationships_get_near_certain_weight(self): + """If observations reliably show a==b and !a==!b then the relationship from a->b should be certain""" + + data = pd.DataFrame([[0, 1] for _ in range(10)], columns=["a", "b"]) + g = from_numpy(data.values) + assert all( + [ + 0.99 <= weight <= 1 + for u, v, weight in g.edges(data="weight") + if u == 0 and v == 1 + ] + ) + + def test_inverse_relationships_get_negative_weight(self): + """If observations indicate a==!b and b==!a then the weight of the relationship from a-> should be negative""" + + data = pd.DataFrame([[0, 1] for _ in range(10)], columns=["a", "b"]) + data.append(pd.DataFrame([[1, 0] for _ in range(10)], columns=["a", "b"])) + g = from_numpy(data.values) + assert all( + [weight < 0 for u, v, weight in g.edges(data="weight") if u == 0 and v == 1] + ) + + def test_no_cycles(self, train_data_idx): + """ + The learned structure should be acyclic + """ + + g = from_numpy(train_data_idx.values, w_threshold=0.3) + assert nx.algorithms.is_directed_acyclic_graph(g) + + def test_tabu_edges_on_non_existing_edges_do_nothing(self, train_data_idx): + """If tabu edges do not exist in the original unconstrained network then nothing changes""" + + g1 = from_numpy(train_data_idx.values, w_threshold=0.3) + g2 = from_numpy( + train_data_idx.values, w_threshold=0.3, tabu_edges=[(0, 3), (4, 0), (1, 6)] + ) + assert set(g1.edges) == set(g2.edges) + + def test_tabu_expected_edges(self, train_data_idx): + """Tabu edges should not exist in the network""" + + tabu_e = [(3, 0), (1, 2)] + g = from_numpy(train_data_idx.values, tabu_edges=tabu_e) + assert [e not in g.edges for e in tabu_e] + + def test_tabu_expected_parent_nodes(self, train_data_idx): + """Tabu parent nodes should not have any outgoing edges""" + + tabu_p = [0, 3, 1] + g = from_numpy(train_data_idx.values, tabu_parent_nodes=tabu_p) + assert [p not in [e[0] for e in g.edges] for p in tabu_p] + + def test_tabu_expected_child_nodes(self, train_data_idx): + """Tabu child nodes should not have any ingoing edges""" + + tabu_c = [0, 3, 1] + g = from_numpy(train_data_idx.values, tabu_child_nodes=tabu_c) + assert [c not in [e[1] for e in g.edges] for c in tabu_c] + + def test_multiple_tabu(self, train_data_idx): + """Any edge related to tabu edges/parent nodes/child nodes should not exist in the network""" + + tabu_e = [(3, 0), (1, 2)] + tabu_p = [1] + tabu_c = [0, 3] + g = from_numpy( + train_data_idx.values, + tabu_edges=tabu_e, + tabu_parent_nodes=tabu_p, + tabu_child_nodes=tabu_c, + ) + assert [e not in g.edges for e in tabu_e] + assert [p not in [e[0] for e in g.edges] for p in tabu_p] + assert [c not in [e[1] for e in g.edges] for c in tabu_c] + + +class TestFromNumpyLasso: + """Test behaviour of the from_numpy_lasso method""" + + def test_all_columns_in_structure(self, train_data_idx): + """Every columns that is in the data should become a node in the learned structure""" + + g = from_numpy_lasso(train_data_idx.values, 0.1) + assert len(g.nodes) == len(train_data_idx.columns) + + def test_isolated_nodes_exist(self, train_data_idx): + """Isolated nodes should still be in the learned structure""" + + g = from_numpy_lasso(train_data_idx.values, 0.1, w_threshold=1.0) + assert len(g.nodes) == len(train_data_idx.columns) + + def test_expected_structure_learned(self, train_data_idx, train_model_idx): + """Given an extremely small lambda_lasso and small data set that can be examined by hand, + the structure should be deterministic""" + + g = from_numpy_lasso(train_data_idx.values, 1e-8, w_threshold=0.3) + assert set(g.edges) == set(train_model_idx.edges) + + def test_empty_data_raises_error(self): + """ + Providing an empty data set should result in a Value Error explaining that data must not be empty. + This error is useful to catch and handle gracefully, because otherwise the user would experience + misleading division by zero, or unpacking errors. + """ + + with pytest.raises(ValueError): + from_numpy_lasso(np.empty([0, 5]), 0.1) + + def test_single_iter_gets_converged_fail_warnings(self, train_data_idx): + """ + With a single iteration on this dataset, learn_structure fails to converge and should give warnings. + """ + + with pytest.warns( + UserWarning, match="Failed to converge. Consider increasing max_iter." + ): + from_numpy_lasso(train_data_idx.values, 0.1, max_iter=1) + + def test_certain_relationships_get_near_certain_weight(self): + """If observations reliably show a==b and !a==!b then the relationship from a->b should be certain""" + + data = pd.DataFrame([[0, 1] for _ in range(10)], columns=["a", "b"]) + g = from_numpy_lasso(data.values, 0.1) + assert all( + [ + 0.99 <= weight <= 1 + for u, v, weight in g.edges(data="weight") + if u == 0 and v == 1 + ] + ) + + def test_inverse_relationships_get_negative_weight(self): + """If observations indicate a==!b and b==!a then the weight of the relationship from a-> should be negative""" + + data = pd.DataFrame([[0, 1] for _ in range(10)], columns=["a", "b"]) + data.append(pd.DataFrame([[1, 0] for _ in range(10)], columns=["a", "b"])) + g = from_numpy_lasso(data.values, 0.1) + assert all( + [weight < 0 for u, v, weight in g.edges(data="weight") if u == 0 and v == 1] + ) + + def test_no_cycles(self, train_data_idx): + """ + The learned structure should be acyclic + """ + + g = from_numpy_lasso(train_data_idx.values, 0.1, w_threshold=0.3) + assert nx.algorithms.is_directed_acyclic_graph(g) + + def test_tabu_expected_edges(self, train_data_idx): + """Tabu edges should not exist in the network""" + + tabu_e = [("d", "a"), ("b", "c")] + g = from_numpy_lasso(train_data_idx.values, 0.1, tabu_edges=tabu_e) + assert [e not in g.edges for e in tabu_e] + + def test_tabu_expected_parent_nodes(self, train_data_idx): + """Tabu parent nodes should not have any outgoing edges""" + + tabu_p = ["a", "d", "b"] + g = from_numpy_lasso(train_data_idx.values, 0.1, tabu_parent_nodes=tabu_p) + assert [p not in [e[0] for e in g.edges] for p in tabu_p] + + def test_tabu_expected_child_nodes(self, train_data_idx): + """Tabu child nodes should not have any ingoing edges""" + + tabu_c = ["a", "d", "b"] + g = from_numpy_lasso(train_data_idx.values, 0.1, tabu_child_nodes=tabu_c) + assert [c not in [e[1] for e in g.edges] for c in tabu_c] + + def test_multiple_tabu(self, train_data_idx): + """Any edge related to tabu edges/parent nodes/child nodes should not exist in the network""" + + tabu_e = [("d", "a"), ("b", "c")] + tabu_p = ["b"] + tabu_c = ["a", "d"] + g = from_numpy_lasso( + train_data_idx.values, + 0.1, + tabu_edges=tabu_e, + tabu_parent_nodes=tabu_p, + tabu_child_nodes=tabu_c, + ) + assert [e not in g.edges for e in tabu_e] + assert [p not in [e[0] for e in g.edges] for p in tabu_p] + assert [c not in [e[1] for e in g.edges] for c in tabu_c] + + def test_sparsity(self, train_data_idx): + """Structure learnt from larger lambda should be sparser than smaller lambda""" + + g1 = from_numpy_lasso(train_data_idx.values, 0.1, w_threshold=0.3) + g2 = from_numpy_lasso(train_data_idx.values, 1e-6, w_threshold=0.3) + assert len(g1.edges) > len(g2.edges) + + def test_sparsity_against_without_reg(self, train_data_idx): + """Structure learnt from regularisation should be sparser than the one without""" + + g1 = from_numpy_lasso(train_data_idx.values, 0.1, w_threshold=0.3) + g2 = from_numpy(train_data_idx.values, w_threshold=0.3) + assert len(g1.edges) > len(g2.edges) + + def test_f1_score(self, train_data_idx, train_model_idx): + """Structure learnt from regularisation should have very high f1 score relative to the ground truth""" + g = from_numpy_lasso(train_data_idx.values, 0.1, w_threshold=0.3) + + print(g.edges) + print(train_model_idx.edges) + n_predictions_made = len(g.edges) + n_correct_predictions = len( + set(g.edges).intersection(set(train_model_idx.edges)) + ) + n_relevant_predictions = len(train_model_idx.edges) + + precision = n_correct_predictions / n_predictions_made + recall = n_correct_predictions / n_relevant_predictions + f1_score = 2 * (precision * recall) / (precision + recall) + + assert f1_score > 0.8 diff --git a/tests/structure/test_structuremodel.py b/tests/structure/test_structuremodel.py new file mode 100644 index 0000000..149bc8e --- /dev/null +++ b/tests/structure/test_structuremodel.py @@ -0,0 +1,479 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from causalnex.structure import StructureModel + + +class TestStructureModel: + def test_init_has_origin(self): + """Creating a StructureModel using constructor should give all edges unknown origin""" + + sm = StructureModel([(1, 2)]) + assert (1, 2) in sm.edges + assert (1, 2, "unknown") in sm.edges.data("origin") + + def test_init_with_origin(self): + """should be possible to specify origin during init""" + + sm = StructureModel([(1, 2)], origin="learned") + assert (1, 2, "learned") in sm.edges.data("origin") + + def test_edge_unknown_property(self): + """should return only edges whose origin is unknown""" + + sm = StructureModel() + sm.add_edge(1, 2, origin="unknown") + sm.add_edge(1, 3, origin="learned") + sm.add_edge(1, 4, origin="expert") + + assert sm.edges_with_origin("unknown") == [(1, 2)] + + def test_edge_learned_property(self): + """should return only edges whose origin is unknown""" + + sm = StructureModel() + sm.add_edge(1, 2, origin="unknown") + sm.add_edge(1, 3, origin="learned") + sm.add_edge(1, 4, origin="expert") + + assert sm.edges_with_origin("learned") == [(1, 3)] + + def test_edge_expert_property(self): + """should return only edges whose origin is unknown""" + + sm = StructureModel() + sm.add_edge(1, 2, origin="unknown") + sm.add_edge(1, 3, origin="learned") + sm.add_edge(1, 4, origin="expert") + + assert sm.edges_with_origin("expert") == [(1, 4)] + + def test_to_directed(self): + """should create a structure model""" + + sm = StructureModel() + edges = [(1, 2), (2, 1), (2, 3), (3, 4)] + sm.add_edges_from(edges) + + dag = sm.to_directed() + assert isinstance(dag, StructureModel) + assert all(edge in dag.edges for edge in edges) + + def test_to_undirected(self): + """should create an undirected Graph""" + + sm = StructureModel() + sm.add_edges_from([(1, 2), (2, 1), (2, 3), (3, 4)]) + + udg = sm.to_undirected() + assert all(edge in udg.edges for edge in [(2, 3), (3, 4)]) + assert (1, 2) in udg.edges or (2, 1) in udg.edges + assert len(udg.edges) == 3 + + +class TestStructureModelAddEdge: + def test_add_edge_default(self): + """edges added with default origin should be identified as unknown origin""" + + sm = StructureModel() + sm.add_edge(1, 2) + + assert (1, 2) in sm.edges + assert (1, 2, "unknown") in sm.edges.data("origin") + + def test_add_edge_unknown(self): + """edges added with unknown origin should be labelled as unknown origin""" + + sm = StructureModel() + sm.add_edge(1, 2, "unknown") + + assert (1, 2) in sm.edges + assert (1, 2, "unknown") in sm.edges.data("origin") + + def test_add_edge_learned(self): + """edges added with learned origin should be labelled as learned origin""" + + sm = StructureModel() + sm.add_edge(1, 2, "learned") + + assert (1, 2) in sm.edges + assert (1, 2, "learned") in sm.edges.data("origin") + + def test_add_edge_expert(self): + """edges added with expert origin should be labelled as expert origin""" + + sm = StructureModel() + sm.add_edge(1, 2, "expert") + + assert (1, 2) in sm.edges + assert (1, 2, "expert") in sm.edges.data("origin") + + def test_add_edge_other(self): + """edges added with other origin should throw an error""" + + sm = StructureModel() + + with pytest.raises(ValueError, match="^Unknown origin: must be one of.*$"): + sm.add_edge(1, 2, "other") + + def test_add_edge_custom_attr(self): + """it should be possible to add an edge with custom attributes""" + + sm = StructureModel() + sm.add_edge(1, 2, x="Y") + + assert (1, 2) in sm.edges + assert (1, 2, "Y") in sm.edges.data("x") + + def test_add_edge_multiple_times(self): + """adding an edge again should update the edges origin attr""" + + sm = StructureModel() + sm.add_edge(1, 2, origin="unknown") + assert (1, 2, "unknown") in sm.edges.data("origin") + sm.add_edge(1, 2, origin="learned") + assert (1, 2, "learned") in sm.edges.data("origin") + + def test_add_multiple_edges(self): + """it should be possible to add multiple edges with different origins""" + + sm = StructureModel() + sm.add_edge(1, 2, origin="unknown") + sm.add_edge(1, 3, origin="learned") + sm.add_edge(1, 4, origin="expert") + + assert (1, 2, "unknown") in sm.edges.data("origin") + assert (1, 3, "learned") in sm.edges.data("origin") + assert (1, 4, "expert") in sm.edges.data("origin") + + +class TestStructureModelAddEdgesFrom: + def test_add_edges_from_default(self): + """edges added with default origin should be identified as unknown origin""" + + sm = StructureModel() + edges = [(1, 2), (2, 3)] + sm.add_edges_from(edges) + + assert all(edge in sm.edges for edge in edges) + assert all((u, v, "unknown") in sm.edges.data("origin") for u, v in edges) + + def test_add_edges_from_unknown(self): + """edges added with unknown origin should be labelled as unknown origin""" + + sm = StructureModel() + edges = [(1, 2), (2, 3)] + sm.add_edges_from(edges, "unknown") + + assert all(edge in sm.edges for edge in edges) + assert all((u, v, "unknown") in sm.edges.data("origin") for u, v in edges) + + def test_add_edges_from_learned(self): + """edges added with learned origin should be labelled as learned origin""" + + sm = StructureModel() + edges = [(1, 2), (2, 3)] + sm.add_edges_from(edges, "learned") + + assert all(edge in sm.edges for edge in edges) + assert all((u, v, "learned") in sm.edges.data("origin") for u, v in edges) + + def test_add_edges_from_expert(self): + """edges added with expert origin should be labelled as expert origin""" + + sm = StructureModel() + edges = [(1, 2), (2, 3)] + sm.add_edges_from(edges, "expert") + + assert all(edge in sm.edges for edge in edges) + assert all((u, v, "expert") in sm.edges.data("origin") for u, v in edges) + + def test_add_edges_from_other(self): + """edges added with other origin should throw an error""" + + sm = StructureModel() + + with pytest.raises(ValueError, match="^Unknown origin: must be one of.*$"): + sm.add_edges_from([(1, 2)], "other") + + def test_add_edges_from_custom_attr(self): + """it should be possible to add edges with custom attributes""" + + sm = StructureModel() + edges = [(1, 2), (2, 3)] + sm.add_edges_from(edges, x="Y") + + assert all(edge in sm.edges for edge in edges) + assert all((u, v, "Y") in sm.edges.data("x") for u, v in edges) + + def test_add_edges_from_multiple_times(self): + """adding edges again should update the edges origin attr""" + + sm = StructureModel() + edges = [(1, 2), (2, 3)] + sm.add_edges_from(edges, "unknown") + assert all((u, v, "unknown") in sm.edges.data("origin") for u, v in edges) + sm.add_edges_from(edges, "learned") + assert all((u, v, "learned") in sm.edges.data("origin") for u, v in edges) + + def test_add_multiple_edges(self): + """it should be possible to add multiple edges with different origins""" + + sm = StructureModel() + sm.add_edges_from([(1, 2)], origin="unknown") + sm.add_edges_from([(1, 3)], origin="learned") + sm.add_edges_from([(1, 4)], origin="expert") + + assert (1, 2, "unknown") in sm.edges.data("origin") + assert (1, 3, "learned") in sm.edges.data("origin") + assert (1, 4, "expert") in sm.edges.data("origin") + + +class TestStructureModelAddWeightedEdgesFrom: + def test_add_weighted_edges_from_default(self): + """edges added with default origin should be identified as unknown origin""" + + sm = StructureModel() + edges = [(1, 2, 0.5), (2, 3, 0.5)] + sm.add_weighted_edges_from(edges) + + assert all((u, v, w) in sm.edges.data("weight") for u, v, w in edges) + assert all((u, v, "unknown") in sm.edges.data("origin") for u, v, w in edges) + + def test_add_weighted_edges_from_unknown(self): + """edges added with unknown origin should be labelled as unknown origin""" + + sm = StructureModel() + edges = [(1, 2, 0.5), (2, 3, 0.5)] + sm.add_weighted_edges_from(edges, origin="unknown") + + assert all((u, v, w) in sm.edges.data("weight") for u, v, w in edges) + assert all((u, v, "unknown") in sm.edges.data("origin") for u, v, w in edges) + + def test_add_weighted_edges_from_learned(self): + """edges added with learned origin should be labelled as learned origin""" + + sm = StructureModel() + edges = [(1, 2, 0.5), (2, 3, 0.5)] + sm.add_weighted_edges_from(edges, origin="learned") + + assert all((u, v, w) in sm.edges.data("weight") for u, v, w in edges) + assert all((u, v, "learned") in sm.edges.data("origin") for u, v, w in edges) + + def test_add_weighted_edges_from_expert(self): + """edges added with expert origin should be labelled as expert origin""" + + sm = StructureModel() + edges = [(1, 2, 0.5), (2, 3, 0.5)] + sm.add_weighted_edges_from(edges, origin="expert") + + assert all((u, v, w) in sm.edges.data("weight") for u, v, w in edges) + assert all((u, v, "expert") in sm.edges.data("origin") for u, v, w in edges) + + def test_add_weighted_edges_from_other(self): + """edges added with other origin should throw an error""" + + sm = StructureModel() + + with pytest.raises(ValueError, match="^Unknown origin: must be one of.*$"): + sm.add_weighted_edges_from([(1, 2, 0.5)], origin="other") + + def test_add_weighted_edges_from_custom_attr(self): + """it should be possible to add edges with custom attributes""" + + sm = StructureModel() + edges = [(1, 2, 0.5), (2, 3, 0.5)] + sm.add_weighted_edges_from(edges, x="Y") + + assert all((u, v, w) in sm.edges.data("weight") for u, v, w in edges) + assert all((u, v, "Y") in sm.edges.data("x") for u, v, _ in edges) + + def test_add_weighted_edges_from_multiple_times(self): + """adding edges again should update the edges origin attr""" + + sm = StructureModel() + edges = [(1, 2, 0.5), (2, 3, 0.5)] + sm.add_weighted_edges_from(edges, origin="unknown") + assert all((u, v, "unknown") in sm.edges.data("origin") for u, v, _ in edges) + sm.add_weighted_edges_from(edges, origin="learned") + assert all((u, v, "learned") in sm.edges.data("origin") for u, v, _ in edges) + + def test_add_multiple_weighted_edges(self): + """it should be possible to add multiple edges with different origins""" + + sm = StructureModel() + sm.add_weighted_edges_from([(1, 2, 0.5)], origin="unknown") + sm.add_weighted_edges_from([(1, 3, 0.5)], origin="learned") + sm.add_weighted_edges_from([(1, 4, 0.5)], origin="expert") + + assert (1, 2, "unknown") in sm.edges.data("origin") + assert (1, 3, "learned") in sm.edges.data("origin") + assert (1, 4, "expert") in sm.edges.data("origin") + + +class TestStructureModelRemoveEdgesBelowThreshold: + def test_remove_edges_below_threshold(self): + """Edges whose weight is less than a defined threshold should be removed""" + + sm = StructureModel() + strong_edges = [(1, 2, 1.0), (1, 3, 0.8), (1, 5, 2.0)] + weak_edges = [(1, 4, 0.4), (2, 3, 0.6), (3, 5, 0.5)] + sm.add_weighted_edges_from(strong_edges) + sm.add_weighted_edges_from(weak_edges) + + sm.remove_edges_below_threshold(0.7) + + assert set(sm.edges(data="weight")) == set(strong_edges) + + def test_negative_weights(self): + """Negative edges whose absolute value is greater than the defined threshold should not be removed""" + + sm = StructureModel() + strong_edges = [(1, 2, -3.0), (3, 1, 0.7), (1, 5, -2.0)] + weak_edges = [(1, 4, 0.4), (2, 3, -0.6), (3, 5, -0.5)] + sm.add_weighted_edges_from(strong_edges) + sm.add_weighted_edges_from(weak_edges) + + sm.remove_edges_below_threshold(0.7) + + assert set(sm.edges(data="weight")) == set(strong_edges) + + def test_equal_weights(self): + """Edges whose absolute value is equal to the defined threshold should not be removed""" + + sm = StructureModel() + strong_edges = [(1, 2, 1.0), (1, 5, 2.0)] + equal_edges = [(1, 3, 0.6), (2, 3, 0.6)] + weak_edges = [(1, 4, 0.4), (3, 5, 0.5)] + sm.add_weighted_edges_from(strong_edges) + sm.add_weighted_edges_from(equal_edges) + sm.add_weighted_edges_from(weak_edges) + + sm.remove_edges_below_threshold(0.6) + + assert set(sm.edges(data="weight")) == set.union( + set(strong_edges), set(equal_edges) + ) + + def test_graph_with_no_edges(self): + """Can still run even if the graph is without edges""" + + sm = StructureModel() + nodes = [1, 2, 3] + sm.add_nodes_from(nodes) + sm.remove_edges_below_threshold(0.6) + + assert set(sm.nodes) == set(nodes) + assert set(sm.edges) == set() + + +class TestStructureModelGetLargestSubgraph: + @pytest.mark.parametrize( + "test_input, expected", + [ + ([(0, 1), (1, 2), (1, 3), (4, 6)], [(0, 1), (1, 2), (1, 3)]), + ([(3, 4), (3, 5), (7, 6)], [(3, 4), (3, 5)]), + ], + ) + def test_get_largest_subgraph(self, test_input, expected): + """Should be able to return the largest subgraph""" + sm = StructureModel() + sm.add_edges_from(test_input) + largest_subgraph = sm.get_largest_subgraph() + + expected_graph = StructureModel() + expected_graph.add_edges_from(expected) + + assert set(largest_subgraph.nodes) == set(expected_graph.nodes) + assert set(largest_subgraph.edges) == set(expected_graph.edges) + + def test_more_than_one_largest(self): + """Return the first largest when there are more than one largest subgraph""" + + edges = [(0, 1), (1, 2), (3, 4), (3, 5)] + sm = StructureModel() + sm.add_edges_from(edges) + largest_subgraph = sm.get_largest_subgraph() + + expected_edges = [(0, 1), (1, 2)] + expected_graph = StructureModel() + expected_graph.add_edges_from(expected_edges) + + assert set(largest_subgraph.nodes) == set(expected_graph.nodes) + assert set(largest_subgraph.edges) == set(expected_graph.edges) + + def test_empty(self): + """Should return None if the structure model is empty""" + + sm = StructureModel() + assert sm.get_largest_subgraph() is None + + def test_isolates(self): + """Should return None if the structure model only contains isolates""" + nodes = [1, 3, 5, 2, 7] + + sm = StructureModel() + sm.add_nodes_from(nodes) + + assert sm.get_largest_subgraph() is None + + def test_isolates_nodes_and_edges(self): + """Should be able to return the largest subgraph""" + + edges = [(0, 1), (1, 2), (1, 3), (5, 6)] + isolated_nodes = [7, 8, 9] + sm = StructureModel() + sm.add_edges_from(edges) + sm.add_nodes_from(isolated_nodes) + largest_subgraph = sm.get_largest_subgraph() + + expected_edges = [(0, 1), (1, 2), (1, 3)] + expected_graph = StructureModel() + expected_graph.add_edges_from(expected_edges) + + assert set(largest_subgraph.nodes) == set(expected_graph.nodes) + assert set(largest_subgraph.edges) == set(expected_graph.edges) + + def test_different_origins_and_weights(self): + """The largest subgraph returned should still have the edge data preserved from the original graph""" + + sm = StructureModel() + sm.add_weighted_edges_from([(1, 2, 2.0)], origin="unknown") + sm.add_weighted_edges_from([(1, 3, 1.0)], origin="learned") + sm.add_weighted_edges_from([(5, 6, 0.7)], origin="expert") + + largest_subgraph = sm.get_largest_subgraph() + + assert set(largest_subgraph.edges.data("origin")) == set( + [(1, 2, "unknown"), (1, 3, "learned")] + ) + assert set(largest_subgraph.edges.data("weight")) == set( + [(1, 2, 2.0), (1, 3, 1.0)] + ) diff --git a/tests/test_bayesiannetwork.py b/tests/test_bayesiannetwork.py new file mode 100644 index 0000000..a54ae22 --- /dev/null +++ b/tests/test_bayesiannetwork.py @@ -0,0 +1,612 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import numpy as np +import pandas as pd +import pytest + +from causalnex.network import BayesianNetwork +from causalnex.structure import StructureModel +from causalnex.structure.notears import from_pandas + + +class TestFitNodeStates: + """Test behaviour of fit node states method""" + + @pytest.mark.parametrize( + "weighted_edges, data", + [ + ([("a", "b", 1)], pd.DataFrame([[1, 1]], columns=["a", "b"])), + ( + [("a", "b", 1)], + pd.DataFrame([[1, 1, 1, 1]], columns=["a", "b", "c", "d"]), + ), + # c and d are isolated nodes in the data + ], + ) + def test_all_nodes_included(self, weighted_edges, data): + """No errors if all the nodes can be found in the columns of training data""" + cg = StructureModel() + cg.add_weighted_edges_from(weighted_edges) + bn = BayesianNetwork(cg).fit_node_states(data) + assert all(node in data.columns for node in bn.node_states.keys()) + + def test_all_states_included(self): + """All states in a node should be included""" + cg = StructureModel() + cg.add_weighted_edges_from([("a", "b", 1)]) + bn = BayesianNetwork(cg).fit_node_states( + pd.DataFrame([[i, i] for i in range(10)], columns=["a", "b"]) + ) + assert all(v in bn.node_states["a"] for v in range(10)) + + def test_fit_with_null_states_raises_error(self): + """An error should be raised if fit is called with null data""" + cg = StructureModel() + cg.add_weighted_edges_from([("a", "b", 1)]) + with pytest.raises(ValueError, match="node '.*' contains None state"): + BayesianNetwork(cg).fit_node_states( + pd.DataFrame([[None, 1]], columns=["a", "b"]) + ) + + def test_fit_with_missing_feature_in_data(self): + """An error should be raised if fit is called with missing feature in data""" + cg = StructureModel() + + cg.add_weighted_edges_from([("a", "e", 1)]) + with pytest.raises( + KeyError, + match="The data does not cover all the features found in the Bayesian Network. " + "Please check the following features: {'e'}", + ): + BayesianNetwork(cg).fit_node_states( + pd.DataFrame([[1, 1, 1, 1]], columns=["a", "b", "c", "d"]) + ) + + +class TestFitCPDSErrors: + """Test errors for fit CPDs method""" + + def test_invalid_method(self, bn, train_data_discrete): + """a value error should be raised in an invalid method is provided""" + + with pytest.raises(ValueError, match=r"unrecognised method.*"): + bn.fit_cpds(train_data_discrete, method="INVALID") + + def test_invalid_prior(self, bn, train_data_discrete): + """a value error should be raised in an invalid prior is provided""" + + with pytest.raises(ValueError, match=r"unrecognised bayes_prior.*"): + bn.fit_cpds( + train_data_discrete, method="BayesianEstimator", bayes_prior="INVALID" + ) + + +class TestFitCPDsMaximumLikelihoodEstimator: + """Test behaviour of fit_cpds using MLE""" + + def test_cause_only_node(self, bn, train_data_discrete, train_data_discrete_cpds): + """Test that probabilities are fit correctly to nodes which are not caused by other nodes""" + + bn.fit_cpds(train_data_discrete) + cpds = bn.cpds + + assert ( + np.mean( + np.abs( + cpds["d"].values.reshape(2) + - train_data_discrete_cpds["d"].reshape(2) + ) + ) + < 1e-7 + ) + assert ( + np.mean( + np.abs( + cpds["e"].values.reshape(2) + - train_data_discrete_cpds["e"].reshape(2) + ) + ) + < 1e-7 + ) + + def test_dependent_node(self, bn, train_data_discrete, train_data_discrete_cpds): + """Test that probabilities are fit correctly to nodes that are caused by other nodes""" + + bn.fit_cpds(train_data_discrete) + cpds = bn.cpds + + assert ( + np.mean( + np.abs( + cpds["a"].values.reshape(24) + - train_data_discrete_cpds["a"].reshape(24) + ) + ) + < 1e-7 + ) + assert ( + np.mean( + np.abs( + cpds["b"].values.reshape(12) + - train_data_discrete_cpds["b"].reshape(12) + ) + ) + < 1e-7 + ) + assert ( + np.mean( + np.abs( + cpds["c"].values.reshape(60) + - train_data_discrete_cpds["c"].reshape(60) + ) + ) + < 1e-7 + ) + + +class TestFitBayesianEstimator: + """Test behaviour of fit_cpds using BE""" + + def test_cause_only_node_bdeu( + self, bn, train_data_discrete, train_data_discrete_cpds + ): + """Test that probabilities are fit correctly to nodes which are not caused by other nodes""" + + bn.fit_cpds( + train_data_discrete, + method="BayesianEstimator", + bayes_prior="BDeu", + equivalent_sample_size=5, + ) + cpds = bn.cpds + + assert ( + np.mean( + np.abs( + cpds["d"].values.reshape(2) + - train_data_discrete_cpds["d"].reshape(2) + ) + ) + < 0.02 + ) + assert ( + np.mean( + np.abs( + cpds["e"].values.reshape(2) + - train_data_discrete_cpds["e"].reshape(2) + ) + ) + < 0.02 + ) + + def test_cause_only_node_k2( + self, bn, train_data_discrete, train_data_discrete_cpds + ): + """Test that probabilities are fit correctly to nodes which are not caused by other nodes""" + + bn.fit_cpds(train_data_discrete, method="BayesianEstimator", bayes_prior="K2") + cpds = bn.cpds + + assert ( + np.mean( + np.abs( + cpds["d"].values.reshape(2) + - train_data_discrete_cpds["d"].reshape(2) + ) + ) + < 0.02 + ) + assert ( + np.mean( + np.abs( + cpds["e"].values.reshape(2) + - train_data_discrete_cpds["e"].reshape(2) + ) + ) + < 0.02 + ) + + def test_dependent_node_bdeu( + self, bn, train_data_discrete, train_data_discrete_cpds + ): + """Test that probabilities are fit correctly to nodes that are caused by other nodes""" + + bn.fit_cpds( + train_data_discrete, + method="BayesianEstimator", + bayes_prior="BDeu", + equivalent_sample_size=1, + ) + cpds = bn.cpds + + assert ( + np.mean( + np.abs( + cpds["a"].values.reshape(24) + - train_data_discrete_cpds["a"].reshape(24) + ) + ) + < 0.02 + ) + assert ( + np.mean( + np.abs( + cpds["b"].values.reshape(12) + - train_data_discrete_cpds["b"].reshape(12) + ) + ) + < 0.02 + ) + assert ( + np.mean( + np.abs( + cpds["c"].values.reshape(60) + - train_data_discrete_cpds["c"].reshape(60) + ) + ) + < 0.02 + ) + + def test_dependent_node_k2( + self, bn, train_data_discrete, train_data_discrete_cpds_k2 + ): + """Test that probabilities are fit correctly to nodes that are caused by other nodes""" + + bn.fit_cpds(train_data_discrete, method="BayesianEstimator", bayes_prior="K2") + cpds = bn.cpds + + assert ( + np.mean( + np.abs( + cpds["a"].values.reshape(24) + - train_data_discrete_cpds_k2["a"].reshape(24) + ) + ) + < 1e-7 + ) + assert ( + np.mean( + np.abs( + cpds["b"].values.reshape(12) + - train_data_discrete_cpds_k2["b"].reshape(12) + ) + ) + < 1e-7 + ) + assert ( + np.mean( + np.abs( + cpds["c"].values.reshape(60) + - train_data_discrete_cpds_k2["c"].reshape(60) + ) + ) + < 1e-7 + ) + + +class TestPredictMaximumLikelihoodEstimator: + """Test behaviour of predict using MLE""" + + def test_predictions_are_based_on_probabilities( + self, bn, train_data_discrete, test_data_c_discrete + ): + """Predictions made using the model should be based on the probabilities that are in the model""" + + bn.fit_cpds(train_data_discrete) + predictions = bn.predict(test_data_c_discrete, "c") + assert np.all( + predictions.values.reshape(len(predictions.values)) + == test_data_c_discrete["c"].values + ) + + def test_prediction_node_suffixed_as_prediction( + self, bn, train_data_discrete, test_data_c_discrete + ): + """The column that contains the values of the predicted node should be named node_prediction""" + + bn.fit_cpds(train_data_discrete) + predictions = bn.predict(test_data_c_discrete, "c") + assert "c_prediction" in predictions.columns + + def test_only_predicted_column_returned( + self, bn, train_data_discrete, test_data_c_discrete + ): + """The returned df should not contain any of the input data columns""" + + bn.fit_cpds(train_data_discrete) + predictions = bn.predict(test_data_c_discrete, "c") + assert len(predictions.columns) == 1 + + def test_predictions_are_not_appended_to_input_df( + self, bn, train_data_discrete, test_data_c_discrete + ): + """The predictions should not be appended to the input df""" + + expected_cols = test_data_c_discrete.columns + bn.fit_cpds(train_data_discrete) + bn.predict(test_data_c_discrete, "c") + assert np.array_equal(test_data_c_discrete.columns, expected_cols) + + def test_missing_parent(self, bn, train_data_discrete, test_data_c_discrete): + """Predictions made when parents are missing should still be reasonably accurate""" + + bn.fit_cpds(train_data_discrete) + predictions = bn.predict(test_data_c_discrete[["a", "b", "c", "d"]], "c") + + n = len(test_data_c_discrete) + + accuracy = ( + 1 + - np.count_nonzero( + predictions.values.reshape(len(predictions.values)) + - test_data_c_discrete["c"].values + ) + / n + ) + + assert accuracy > 0.9 + + def test_missing_non_parent(self, bn, train_data_discrete, test_data_c_discrete): + """It should be possible to make predictions with non-parent nodes missing""" + + bn.fit_cpds(train_data_discrete) + predictions = bn.predict(test_data_c_discrete[["b", "c", "d", "e"]], "c") + assert np.all( + predictions.values.reshape(len(predictions.values)) + == test_data_c_discrete["c"].values + ) + + +class TestPredictBayesianEstimator: + """Test behaviour of predict using BE""" + + def test_predictions_are_based_on_probabilities_dbeu( + self, bn, train_data_discrete, test_data_c_discrete + ): + """Predictions made using the model should be based on the probabilities that are in the model""" + + bn.fit_cpds( + train_data_discrete, + method="BayesianEstimator", + bayes_prior="BDeu", + equivalent_sample_size=5, + ) + predictions = bn.predict(test_data_c_discrete, "c") + assert np.all( + predictions.values.reshape(len(predictions.values)) + == test_data_c_discrete["c"].values + ) + + def test_predictions_are_based_on_probabilities_k2( + self, bn, train_data_discrete, test_data_c_discrete + ): + """Predictions made using the model should be based on the probabilities that are in the model""" + + bn.fit_cpds( + train_data_discrete, + method="BayesianEstimator", + bayes_prior="K2", + equivalent_sample_size=5, + ) + predictions = bn.predict(test_data_c_discrete, "c") + assert np.all( + predictions.values.reshape(len(predictions.values)) + == test_data_c_discrete["c"].values + ) + + +class TestPredictProbabilityMaximumLikelihoodEstimator: + """Test behaviour of predict_probability using MLE""" + + def test_expected_probabilities_are_predicted( + self, bn, train_data_discrete, test_data_c_discrete, test_data_c_likelihood + ): + """Probabilities should return exactly correct on a hand computable scenario""" + bn.fit_cpds(train_data_discrete) + probability = bn.predict_probability(test_data_c_discrete, "c") + + assert all( + np.isclose( + probability.values.flatten(), test_data_c_likelihood.values.flatten() + ) + ) + + def test_missing_parent( + self, bn, train_data_discrete, test_data_c_discrete, test_data_c_likelihood + ): + """Probabilities made when parents are missing should still be reasonably accurate""" + + bn.fit_cpds(train_data_discrete) + probability = bn.predict_probability( + test_data_c_discrete[["a", "b", "c", "d"]], "c" + ) + + n = len(probability.values.flatten()) + + accuracy = ( + np.count_nonzero( + [ + 1 if math.isclose(a, b, abs_tol=0.15) else 0 + for a, b in zip( + probability.values.flatten(), + test_data_c_likelihood.values.flatten(), + ) + ] + ) + / n + ) + + assert accuracy > 0.8 + + def test_missing_non_parent( + self, bn, train_data_discrete, test_data_c_discrete, test_data_c_likelihood + ): + """It should be possible to make predictions with non-parent nodes missing""" + + bn.fit_cpds(train_data_discrete) + probability = bn.predict_probability( + test_data_c_discrete[["b", "c", "d", "e"]], "c" + ) + assert all( + np.isclose( + probability.values.flatten(), test_data_c_likelihood.values.flatten() + ) + ) + + +class TestPredictProbabilityBayesianEstimator: + """Test behaviour of predict_probability using BayesianEstimator""" + + def test_expected_probabilities_are_predicted( + self, bn, train_data_discrete, test_data_c_discrete, test_data_c_likelihood + ): + """Probabilities should return exactly correct on a hand computable scenario""" + + bn.fit_cpds( + train_data_discrete, + method="BayesianEstimator", + bayes_prior="BDeu", + equivalent_sample_size=1, + ) + probability = bn.predict_probability(test_data_c_discrete, "c") + assert all( + np.isclose( + probability.values.flatten(), + test_data_c_likelihood.values.flatten(), + atol=0.1, + ) + ) + + +class TestFitNodesStatesAndCPDs: + """Test behaviour of helper function""" + + def test_behaves_same_as_seperate_calls(self, train_data_idx, train_data_discrete): + bn1 = BayesianNetwork(from_pandas(train_data_idx, w_threshold=0.3)) + bn2 = BayesianNetwork(from_pandas(train_data_idx, w_threshold=0.3)) + + bn1.fit_node_states(train_data_discrete).fit_cpds(train_data_discrete) + bn2.fit_node_states_and_cpds(train_data_discrete) + + assert bn1.edges == bn2.edges + assert bn1.node_states == bn2.node_states + + cpds1 = bn1.cpds + cpds2 = bn2.cpds + + assert cpds1.keys() == cpds2.keys() + + for k in cpds1: + assert cpds1[k].equals(cpds2[k]) + + +class TestCPDsProperty: + """Test behaviour of the CPDs property""" + + def test_row_index_of_state_values(self, bn): + """CPDs should have row index set to values of all possible states of the node""" + + assert bn.cpds["a"].index.tolist() == sorted(list(bn.node_states["a"])) + + def test_col_index_of_parent_state_combinations(self, bn): + """CPDs should have a column multi-index of parent state permutations""" + + assert bn.cpds["a"].columns.names == ["b", "d"] + + +class TestInit: + """Test behaviour when constructing a BayesianNetwork""" + + def test_cycles_in_structure(self): + """An error should be raised if cycles are present""" + + with pytest.raises( + ValueError, + match=r"The given structure is not acyclic\. " + r"Please review the following cycle\.*", + ): + BayesianNetwork(StructureModel([(0, 1), (1, 2), (2, 0)])) + + @pytest.mark.parametrize( + "test_input,n_components", + [([(0, 1), (1, 2), (3, 4), (4, 6)], 2), ([(0, 1), (1, 2), (3, 4), (5, 6)], 3)], + ) + def test_disconnected_components(self, test_input, n_components): + """An error should be raised if there is more than one graph component""" + + with pytest.raises( + ValueError, + match=r"The given structure has " + + str(n_components) + + r" separated graph components\. " + r"Please make sure it has only one\.", + ): + BayesianNetwork(StructureModel(test_input)) + + +class TestStructure: + """Test behaviour of the property structure""" + + def test_get_structure(self): + """The structure retrieved should be the same""" + + sm = StructureModel() + + sm.add_weighted_edges_from([(1, 2, 2.0)], origin="unknown") + sm.add_weighted_edges_from([(1, 3, 1.0)], origin="learned") + sm.add_weighted_edges_from([(3, 5, 0.7)], origin="expert") + + bn = BayesianNetwork(sm) + + sm_from_bn = bn.structure + + assert set(sm.edges.data("origin")) == set(sm_from_bn.edges.data("origin")) + assert set(sm.edges.data("weight")) == set(sm_from_bn.edges.data("weight")) + + assert set(sm.nodes) == set(sm_from_bn.nodes) + + def test_set_structure(self): + """An error should be raised if setting the structure""" + + sm = StructureModel() + sm.add_weighted_edges_from([(1, 2, 2.0)], origin="unknown") + sm.add_weighted_edges_from([(1, 3, 1.0)], origin="learned") + sm.add_weighted_edges_from([(3, 5, 0.7)], origin="expert") + + bn = BayesianNetwork(sm) + + new_sm = StructureModel() + sm.add_weighted_edges_from([(2, 5, 3.0)], origin="unknown") + sm.add_weighted_edges_from([(2, 3, 2.0)], origin="learned") + sm.add_weighted_edges_from([(3, 4, 1.7)], origin="expert") + + with pytest.raises(AttributeError, match=r"can't set attribute"): + bn.structure = new_sm diff --git a/tests/test_inference.py b/tests/test_inference.py new file mode 100644 index 0000000..99bd049 --- /dev/null +++ b/tests/test_inference.py @@ -0,0 +1,371 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import pytest + +from causalnex.inference import InferenceEngine +from causalnex.network import BayesianNetwork +from causalnex.structure import StructureModel +from causalnex.structure.notears import from_pandas + + +class TestInferenceEngineIdx: + def test_create_inference_from_bn(self, train_model, train_data_idx): + """It should be possible to create a new Inference object from an existing pgmpy model""" + + bn = BayesianNetwork(train_model).fit_node_states(train_data_idx) + bn.fit_cpds(train_data_idx) + InferenceEngine(bn) + + def test_create_inference_with_bad_variable_names_fails( + self, train_model, train_data_idx + ): + + model = StructureModel() + model.add_edges_from( + [ + (str(u).replace("a", "$a"), str(v).replace("a", "$a")) + for u, v in train_model.edges + ] + ) + + train_data_idx.rename(columns={"a": "$a"}, inplace=True) + + bn = BayesianNetwork(model).fit_node_states(train_data_idx) + bn.fit_cpds(train_data_idx) + + with pytest.raises(ValueError, match="Variable names must match.*"): + InferenceEngine(bn) + + def test_empty_query_returns_marginals( + self, train_model, train_data_idx, train_data_idx_marginals + ): + """An empty query should return all the marginal probabilities of the model's distribution""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + dist = ie.query({}) + + for node, states in dist.items(): + for state, p in states.items(): + assert math.isclose( + train_data_idx_marginals[node][state], p, abs_tol=0.05 + ) + + def test_observations_affect_marginals(self, train_model, train_data_idx): + """Observing the state of a node should affect the marginals of dependent nodes""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + + m1 = ie.query({}) + m2 = ie.query({"d": 1}) + + assert m2["d"][0] == 0 + assert m2["d"][1] == 1 + assert not math.isclose(m2["b"][1], m1["b"][1], abs_tol=0.01) + + def test_observations_does_not_affect_marginals_of_independent_nodes( + self, train_model, train_data_idx + ): + """Observing the state of a node should not affect the marginal probability of an independent node""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + + m1 = ie.query({}) + m2 = ie.query({"d": 1}) + + assert m2["d"][0] == 0 + assert m2["d"][1] == 1 + assert math.isclose(m2["e"][1], m1["e"][1], abs_tol=0.05) + + def test_do_sets_state_probability_to_one(self, train_model, train_data_idx): + """Do should update the probability of the given observation=state to 1""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + ie.do_intervention("d", 1) + assert math.isclose(ie.query()["d"][1], 1) + + def test_do_on_node_with_no_effects_not_allowed(self, train_model, train_data_idx): + """It should not be possible to create an isolated node in the network""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + + with pytest.raises( + ValueError, + match="Do calculus cannot be applied because it would result in an isolate", + ): + ie.do_intervention("a", 1) + + def test_do_sets_other_state_probabilitys_to_zero( + self, train_model, train_data_idx + ): + """Do should update the probability of every other state for the observation to zero""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + ie.do_intervention("d", 1) + assert ie.query()["d"][0] == 0 + + def test_do_accepts_all_state_probabilities(self, train_model, train_data_idx): + """Do should accept a map of state->p and update p accordingly""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + ie.do_intervention("d", {0: 0.7, 1: 0.3}) + assert math.isclose(ie.query()["d"][0], 0.7) + assert math.isclose(ie.query()["d"][1], 0.3) + + def test_do_expects_all_state_probabilities_sum_to_one( + self, train_model, train_data_idx + ): + """Do should accept only state probabilities where the full distribution is provided""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + + with pytest.raises( + ValueError, match="The cpd for the provided observation must sum to 1" + ): + ie.do_intervention("d", {0: 0.7, 1: 0.4}) + + def test_do_expects_all_states_have_a_probability( + self, train_model, train_data_idx + ): + """Do should accept only state probabilities where all states in the original cpds are present""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + + with pytest.raises( + ValueError, match="The cpd states do not match expected states*" + ): + ie.do_intervention("d", {1: 1}) + + def test_do_prevents_new_states_being_added(self, train_model, train_data_idx): + """Do should not allow the introduction of new states""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + + with pytest.raises( + ValueError, match="The cpd states do not match expected states*" + ): + ie.do_intervention("d", {0: 0.7, 1: 0.3, 2: 0.0}) + + def test_do_reflected_in_query(self, train_model, train_data_idx): + """Do should adjust marginals returned by query when given a different observation""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + + assert ie.query({"a": 1})["d"][1] != 1 + ie.do_intervention("d", 1) + assert ie.query({"a": 1})["d"][1] == 1 + + def test_reset_do_sets_probabilities_back_to_initial_state( + self, train_model, train_data_idx, train_data_idx_marginals + ): + """Resetting Do operator should re-introduce the original conditional dependencies""" + + bn = BayesianNetwork(train_model) + bn.fit_node_states(train_data_idx).fit_cpds(train_data_idx) + + ie = InferenceEngine(bn) + ie.do_intervention("d", {0: 0.7, 1: 0.3}) + ie.reset_do("d") + + assert math.isclose(ie.query()["d"][0], train_data_idx_marginals["d"][0]) + assert math.isclose(ie.query()["d"][1], train_data_idx_marginals["d"][1]) + + +class TestInferenceEngineDiscrete: + """Test behaviour of query and interventions""" + + def test_query_when_cpds_not_fit(self, train_data_idx, train_data_discrete): + """An error should be raised if query before CPDs are fit""" + + bn = BayesianNetwork( + from_pandas(train_data_idx, w_threshold=0.3) + ).fit_node_states(train_data_discrete) + + with pytest.raises( + ValueError, match=r"Bayesian Network does not contain any CPDs.*" + ): + InferenceEngine(bn) + + def test_empty_query_returns_marginals(self, bn, train_data_discrete_marginals): + """An empty query should return all the marginal probabilities of the model's distribution""" + + ie = InferenceEngine(bn) + dist = ie.query({}) + + for node, states in dist.items(): + for state, p in states.items(): + assert math.isclose( + train_data_discrete_marginals[node][state], p, abs_tol=0.05 + ) + + def test_observations_affect_marginals(self, bn): + """Observing the state of a node should affect the marginals of dependent nodes""" + + ie = InferenceEngine(bn) + + m1 = ie.query({}) + m2 = ie.query({"d": True}) + + assert m2["d"][False] == 0 + assert m2["d"][True] == 1 + assert not math.isclose(m2["b"]["x"], m1["b"]["x"], abs_tol=0.05) + + def test_observations_does_not_affect_marginals_of_independent_nodes(self, bn): + """Observing the state of a node should not affect the marginal probability of an independent node""" + + ie = InferenceEngine(bn) + + m1 = ie.query({}) + m2 = ie.query({"d": True}) + + assert m2["d"][False] == 0 + assert m2["d"][True] == 1 + assert math.isclose(m2["e"][True], m1["e"][True], abs_tol=0.05) + + def test_do_sets_state_probability_to_one(self, bn): + """Do should update the probability of the given observation=state to 1""" + + ie = InferenceEngine(bn) + ie.do_intervention("d", True) + assert math.isclose(ie.query()["d"][True], 1) + + def test_do_on_node_with_no_effects_not_allowed(self, bn): + """It should not be possible to create an isolated node in the network""" + + ie = InferenceEngine(bn) + + with pytest.raises( + ValueError, + match="Do calculus cannot be applied because it would result in an isolate", + ): + ie.do_intervention("a", "b") + + def test_do_sets_other_state_probabilitys_to_zero(self, bn): + """Do should update the probability of every other state for the observation to zero""" + + ie = InferenceEngine(bn) + ie.do_intervention("d", True) + assert ie.query()["d"][False] == 0 + + def test_do_accepts_all_state_probabilities(self, bn): + """Do should accept a map of state->p and update p accordingly""" + + ie = InferenceEngine(bn) + ie.do_intervention("d", {False: 0.7, True: 0.3}) + assert math.isclose(ie.query()["d"][False], 0.7) + assert math.isclose(ie.query()["d"][True], 0.3) + + def test_do_expects_all_state_probabilities_sum_to_one(self, bn): + """Do should accept only state probabilities where the full distribution is provided""" + + ie = InferenceEngine(bn) + + with pytest.raises( + ValueError, match="The cpd for the provided observation must sum to 1" + ): + ie.do_intervention("d", {False: 0.7, True: 0.4}) + + def test_do_expects_all_states_have_a_probability(self, bn): + """Do should accept only state probabilities where all states in the original cpds are present""" + + ie = InferenceEngine(bn) + + with pytest.raises( + ValueError, match="The cpd states do not match expected states*" + ): + ie.do_intervention("d", {False: 1}) + + def test_do_prevents_new_states_being_added(self, bn): + """Do should not allow the introduction of new states""" + + ie = InferenceEngine(bn) + + with pytest.raises( + ValueError, match="The cpd states do not match expected states*" + ): + ie.do_intervention("d", {False: 0.7, True: 0.3, "other": 0.0}) + + def test_do_reflected_in_query(self, bn): + """Do should adjust marginals returned by query when given a different observation""" + + ie = InferenceEngine(bn) + + assert ie.query({"a": "b"})["d"][True] != 1 + ie.do_intervention("d", True) + assert ie.query({"a": "b"})["d"][True] == 1 + + def test_reset_do_sets_probabilities_back_to_initial_state( + self, bn, train_data_discrete_marginals + ): + """Resetting Do operator should re-introduce the original conditional dependencies""" + + ie = InferenceEngine(bn) + ie.do_intervention("d", {False: 0.7, True: 0.3}) + ie.reset_do("d") + + assert math.isclose( + ie.query()["d"][False], train_data_discrete_marginals["d"][False] + ) + assert math.isclose( + ie.query()["d"][False], train_data_discrete_marginals["d"][False] + ) diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..9fee6b3 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,400 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import random + +import numpy as np +import pandas as pd + +from causalnex.evaluation import classification_report, roc_auc +from causalnex.network import BayesianNetwork +from causalnex.structure.notears import from_pandas +from causalnex.structure.structuremodel import StructureModel + + +class TestROCAUCStates: + """Test behaviour of the roc_auc_states metric""" + + def test_roc_of_incorrect_has_fpr_lt_tpr(self): + """The ROC of incorrect predictions should have FPR < TPR""" + + # regardless of a or b, c=1 is always more likely to varying amounts (to create multiple threshold + # points in roc curve) + train = pd.DataFrame( + [[a, b, 0] for a in range(3) for b in range(3) for _ in range(1)] + + [ + [a, b, 1] + for a in range(3) + for b in range(3) + for _ in range(a * 1000 + b * 1000 + 1000) + ], + columns=["a", "b", "c"], + ) + + cg = StructureModel() + cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)]) + + bn = BayesianNetwork(cg) + bn.fit_node_states(train) + bn.fit_cpds(train) + + assert np.allclose(bn.cpds["c"].loc[1].values, 1, atol=0.02) + + # in test, c=0 is always more likely (opposite of train) + test = pd.DataFrame( + [[a, b, 0] for a in range(3) for b in range(3) for _ in range(1000)] + + [[a, b, 1] for a in range(3) for b in range(3) for _ in range(1)], + columns=["a", "b", "c"], + ) + + roc, _ = roc_auc(bn, test, "c") + + assert len(roc) > 3 + assert all(fpr > tpr for fpr, tpr in roc if tpr not in [0.0, 1.0]) + + def test_auc_of_incorrect_close_to_zero(self): + """The AUC of incorrect predictions should be close to zero""" + + # regardless of a or b, c=1 is always more likely to varying amounts (to create multiple threshold + # points in roc curve) + train = pd.DataFrame( + [[a, b, 0] for a in range(3) for b in range(3) for _ in range(1)] + + [ + [a, b, 1] + for a in range(3) + for b in range(3) + for _ in range(a * 1000 + b * 1000 + 1000) + ], + columns=["a", "b", "c"], + ) + + cg = StructureModel() + cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)]) + + bn = BayesianNetwork(cg) + bn.fit_node_states(train) + bn.fit_cpds(train) + + assert np.allclose(bn.cpds["c"].loc[1].values, 1, atol=0.02) + + # in test, c=0 is always more likely (opposite of train) + test = pd.DataFrame( + [[a, b, 0] for a in range(3) for b in range(3) for _ in range(1000)] + + [[a, b, 1] for a in range(3) for b in range(3) for _ in range(1)], + columns=["a", "b", "c"], + ) + + _, auc = roc_auc(bn, test, "c") + + assert math.isclose(auc, 0, abs_tol=0.001) + + def test_roc_of_random_has_unit_gradient(self): + """The ROC curve for random predictions should be a line from (0,0) to (1,1)""" + + # regardless of a or b, c=1 is always more likely to varying amounts (to create multiple threshold + # points in roc curve) + train = pd.DataFrame( + [[a, b, 0] for a in range(3) for b in range(3) for _ in range(1)] + + [ + [a, b, 1] + for a in range(3) + for b in range(3) + for _ in range(a * 1000 + b * 1000 + 1000) + ], + columns=["a", "b", "c"], + ) + + cg = StructureModel() + cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)]) + + bn = BayesianNetwork(cg) + bn.fit_node_states(train) + bn.fit_cpds(train) + + assert np.allclose(bn.cpds["c"].loc[1].values, 1, atol=0.02) + + test = pd.DataFrame( + [ + [a, b, random.randint(0, 1)] + for a in range(3) + for b in range(3) + for _ in range(1000) + ], + columns=["a", "b", "c"], + ) + + roc, _ = roc_auc(bn, test, "c") + + assert len(roc) > 3 + assert all(math.isclose(a, b, abs_tol=0.03) for a, b in roc) + + def test_auc_of_random_is_half(self): + """The AUC of random predictions should be 0.5""" + + # regardless of a or b, c=1 is always more likely to varying amounts (to create multiple threshold + # points in roc curve) + train = pd.DataFrame( + [[a, b, 0] for _ in range(10) for a in range(3) for b in range(3)] + + [ + [a, b, 1] + for a in range(3) + for b in range(3) + for _ in range(a * 1000 + b * 1000 + 1000) + ], + columns=["a", "b", "c"], + ) + + cg = StructureModel() + cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)]) + + bn = BayesianNetwork(cg) + bn.fit_node_states(train) + bn.fit_cpds(train) + + assert np.allclose(bn.cpds["c"].loc[1].values, 1, atol=0.02) + + test = pd.DataFrame( + [ + [a, b, random.randint(0, 1)] + for a in range(3) + for b in range(3) + for _ in range(1000) + ], + columns=["a", "b", "c"], + ) + + _, auc = roc_auc(bn, test, "c") + + assert math.isclose(auc, 0.5, abs_tol=0.03) + + def test_roc_of_accurate_predictions(self): + """TPR should always be better than FPR for accurate predictions""" + + # equal class (c) weighting to guarantee high ROC expected + train = pd.DataFrame( + [[a, b, 0] for a in range(0, 2) for b in range(0, 2) for _ in range(10)] + + [ + [a, b, 1] + for a in range(0, 2) + for b in range(0, 2) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [ + [a, b, 0] + for a in range(2, 4) + for b in range(2, 4) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [[a, b, 1] for a in range(2, 4) for b in range(2, 4) for _ in range(10)], + columns=["a", "b", "c"], + ) + + cg = StructureModel() + cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)]) + + bn = BayesianNetwork(cg) + bn.fit_node_states(train) + bn.fit_cpds(train) + + roc, _ = roc_auc(bn, train, "c") + assert all(tpr > fpr for fpr, tpr in roc if tpr not in [0.0, 1.0]) + + def test_auc_of_accurate_predictions(self): + """AUC of accurate predictions should be 1""" + + # equal class (c) weighting to guarantee high ROC expected + train = pd.DataFrame( + [[a, b, 0] for a in range(0, 2) for b in range(0, 2) for _ in range(1)] + + [ + [a, b, 1] + for a in range(0, 2) + for b in range(0, 2) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [ + [a, b, 0] + for a in range(2, 4) + for b in range(2, 4) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [[a, b, 1] for a in range(2, 4) for b in range(2, 4) for _ in range(1)], + columns=["a", "b", "c"], + ) + + cg = StructureModel() + cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)]) + + bn = BayesianNetwork(cg) + bn.fit_node_states(train) + bn.fit_cpds(train) + + _, auc = roc_auc(bn, train, "c") + assert math.isclose(auc, 1, abs_tol=0.001) + + def test_auc_with_missing_state_in_test(self): + """AUC should still be calculated correctly with states missing in test set""" + + # equal class (c) weighting to guarantee high ROC expected + train = pd.DataFrame( + [[a, b, 0] for a in range(0, 2) for b in range(0, 2) for _ in range(1)] + + [ + [a, b, 1] + for a in range(0, 2) + for b in range(0, 2) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [ + [a, b, 0] + for a in range(2, 4) + for b in range(2, 4) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [[a, b, 1] for a in range(2, 4) for b in range(2, 4) for _ in range(1)], + columns=["a", "b", "c"], + ) + + test = train[train["c"] == 1] + assert len(test["c"].unique()) == 1 + + cg = StructureModel() + cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)]) + + bn = BayesianNetwork(cg) + bn.fit_node_states(train) + bn.fit_cpds(train) + + _, auc = roc_auc(bn, test, "c") + assert math.isclose(auc, 1, abs_tol=0.01) + + def test_auc_node_with_no_parents(self): + """Should be possible to compute auc for state with no parent nodes""" + + train = pd.DataFrame( + [[a, b, 0] for a in range(0, 2) for b in range(0, 2) for _ in range(1)] + + [ + [a, b, 1] + for a in range(0, 2) + for b in range(0, 2) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [ + [a, b, 0] + for a in range(2, 4) + for b in range(2, 4) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [[a, b, 1] for a in range(2, 4) for b in range(2, 4) for _ in range(1)], + columns=["a", "b", "c"], + ) + + cg = StructureModel() + cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)]) + + bn = BayesianNetwork(cg) + bn.fit_node_states(train) + bn.fit_cpds(train) + + _, auc = roc_auc(bn, train, "a") + assert math.isclose(auc, 0.5, abs_tol=0.01) + + def test_auc_for_nonnumeric_features(self): + """AUC of accurate predictions should be 1 even after remapping numbers to strings""" + + # equal class (c) weighting to guarantee high ROC expected + train = pd.DataFrame( + [[a, b, 0] for a in range(0, 2) for b in range(0, 2) for _ in range(1)] + + [ + [a, b, 1] + for a in range(0, 2) + for b in range(0, 2) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [ + [a, b, 0] + for a in range(2, 4) + for b in range(2, 4) + for _ in range(a * 10 + b * 10 + 1000) + ] + + [[a, b, 1] for a in range(2, 4) for b in range(2, 4) for _ in range(1)], + columns=["a", "b", "c"], + ) + + # remap values in column c + train["c"] = train["c"].map({0: "f", 1: "g"}) + + cg = StructureModel() + cg.add_weighted_edges_from([("a", "c", 1), ("b", "c", 1)]) + + bn = BayesianNetwork(cg) + bn.fit_node_states(train) + bn.fit_cpds(train) + + _, auc = roc_auc(bn, train, "c") + assert math.isclose(auc, 1, abs_tol=0.001) + + +class TestClassificationReport: + """Test behaviour of classification_report""" + + def test_contains_expected_columns(self, test_data_c_discrete, bn): + """Check that the report contains all of the required data""" + + report = classification_report(bn, test_data_c_discrete, "c") + + assert set(report.columns) == {"recall", "precision", "support", "f1-score"} + + def test_contains_all_class_data( + self, test_data_c_discrete, bn, test_data_c_likelihood + ): + """Check that the report contains data on each possible class""" + + report = classification_report(bn, test_data_c_discrete, "c") + + assert (label in report.index for label in test_data_c_likelihood.columns) + + def test_report_ignores_unrequired_columns_in_data( + self, train_data_idx, train_data_discrete, test_data_c_discrete + ): + """Classification report should ignore any columns that are no needed by predict""" + + bn = BayesianNetwork( + from_pandas(train_data_idx, w_threshold=0.3) + ).fit_node_states(train_data_discrete) + train_data_discrete["NEW_COL"] = [1] * len(train_data_discrete) + bn.fit_cpds(train_data_discrete) + classification_report(bn, test_data_c_discrete, "c") + + def test_report_on_node_with_no_parents_based_on_modal_state( + self, bn, train_data_discrete + ): + """Classification Report on a node with no parents should reflect that predictions are on modal state""" + + report = classification_report(bn, train_data_discrete, "d") + assert report.loc["d_False", "recall"] == 1 # always predicts most likely class + assert report.loc["d_True", "recall"] == 0 diff --git a/tests/test_plotting.py b/tests/test_plotting.py new file mode 100644 index 0000000..5e28884 --- /dev/null +++ b/tests/test_plotting.py @@ -0,0 +1,136 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +from string import ascii_lowercase + +import matplotlib as plt +import pytest +from matplotlib.colors import to_rgba + +from causalnex.plots import plot_structure +from causalnex.structure import StructureModel + + +class TestPlotStructure: + """Test behaviour of plot structure method""" + + @pytest.mark.parametrize( + "test_input,expected", [(None, ""), ("", ""), ("TEST", "TEST")] + ) + def test_title(self, test_input, expected): + """Title should be set correctly""" + sm = StructureModel([("a", "b")]) + _, ax, _ = plot_structure(sm, title=test_input) + assert ax.get_title() == expected + + def test_edges_exist(self): + """All edges should exist""" + + for num_nodes in range(2, 10): + nodes = [c for i, c in enumerate(ascii_lowercase) if i < num_nodes] + sm = StructureModel(list(zip(nodes[:-1], nodes[1:]))) + _, ax, _ = plot_structure(sm) + ax_edges = [ + patch + for patch in ax.patches + if isinstance(patch, plt.patches.FancyArrowPatch) + ] + assert len(ax_edges) == num_nodes - 1 + + @pytest.mark.parametrize( + "test_input,expected", + [("#123456", to_rgba("#123456")), ("blue", to_rgba("blue"))], + ) + def test_edge_color(self, test_input, expected): + """Edge color should be set if given""" + sm = StructureModel([("a", "b")]) + _, ax, _ = plot_structure(sm, edge_color=test_input) + ax_edges = [ + patch + for patch in ax.patches + if isinstance(patch, plt.patches.FancyArrowPatch) + ] + assert ax_edges[0].get_edgecolor() == expected + + def test_nodes_exist(self): + """All nodes should exist""" + + for num_nodes in range(2, 10): + nodes = [c for i, c in enumerate(ascii_lowercase) if i < num_nodes] + sm = StructureModel(list(zip(nodes[:-1], nodes[1:]))) + _, ax, _ = plot_structure(sm) + ax_nodes = ax.collections[0].get_offsets() + assert len(ax_nodes) == num_nodes + + @pytest.mark.parametrize( + "input_positions,expected_positions", + [({"a": [1, 1], "b": [2, 2]}, [[1.0, 1.0], [2.0, 2.0]])], + ) + def test_node_positions_respected(self, input_positions, expected_positions): + """Nodes should be at the positions provided""" + sm = StructureModel([("a", "b")]) + _, ax, _ = plot_structure(sm, node_positions=input_positions) + node_coords = [list(coord) for coord in ax.collections[0].get_offsets()] + assert all( + [ + node_x == exp_x and node_y == exp_y + for ((exp_x, exp_y), (node_x, node_y)) in zip( + expected_positions, sorted(node_coords) + ) + ] + ) + + @pytest.mark.parametrize( + "test_input,expected", + [("#123456", to_rgba("#123456")), ("blue", to_rgba("blue"))], + ) + def test_node_color(self, test_input, expected): + """Node color should be set if given""" + sm = StructureModel([("a", "b")]) + _, ax, _ = plot_structure(sm, node_color=test_input) + assert all( + all(face_color == expected) + for face_color in ax.collections[0].get_facecolors() + ) + + @pytest.mark.parametrize("test_input,expected", [(False, False), (True, True)]) + def test_show_labels(self, test_input, expected): + """Labels should be hidden when show_labels set to False""" + sm = StructureModel([("a", "b")]) + _, ax, _ = plot_structure(sm, show_labels=test_input) + + assert bool(ax.texts) == expected + + @pytest.mark.parametrize( + "test_input,expected", [("r", "r"), ("#123456", "#123456")] + ) + def test_label_colors(self, test_input, expected): + """Labels should have color provided to them""" + sm = StructureModel([("a", "b")]) + _, ax, _ = plot_structure(sm, show_labels=True, label_color=test_input) + assert all(text.get_color() == expected for text in ax.texts) diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py new file mode 100644 index 0000000..78f2d37 --- /dev/null +++ b/tests/test_preprocessing.py @@ -0,0 +1,454 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import numpy as np +import pytest + +from causalnex.discretiser import Discretiser + + +class TestUniform: + def test_fit_creates_exactly_uniform_splits_when_possible(self): + """splits should be exactly uniform if possible""" + + arr = np.array(range(20)) + np.random.shuffle(arr) + d = Discretiser(method="uniform", num_buckets=4) + d.fit(arr) + for n in range(2): + assert 4 < (d.numeric_split_points[n + 1] - d.numeric_split_points[n]) <= 5 + + def test_fit_creates_close_to_uniform_splits_when_uniform_not_possible(self): + """splits should be close to uniform if uniform is not possible""" + + arr = np.array(range(9)) + np.random.shuffle(arr) + d = Discretiser(method="uniform", num_buckets=4) + d.fit(arr) + + assert len(d.numeric_split_points) == 3 + for n in range(2): + assert 2 <= (d.numeric_split_points[n + 1] - d.numeric_split_points[n]) <= 3 + + def test_fit_does_not_attempt_to_deal_with_identical_split_points(self): + """if all data is identical, and num_buckets>1, then this is not possible. + In this case the standard behaviour of numpy is followed, and many identical + splits will be created. See transform for how these are applied""" + + arr = np.array([1 for _ in range(20)]) + d = Discretiser(method="uniform", num_buckets=4) + d.fit(arr) + assert np.array_equal( + np.array([d.numeric_split_points[0] for _ in range(3)]), + d.numeric_split_points, + ) + + def test_transform_uneven_split(self): + """Data that cannot be split evenly between buckets should be transformed + into near-even buckets""" + + arr = np.array([n + 1 for n in range(10)]) + np.random.shuffle(arr) + d = Discretiser(method="uniform", num_buckets=4) + d.fit(arr) + unique, counts = np.unique(d.transform(arr), return_counts=True) + # check all 4 buckets are used + assert np.array_equal([0, 1, 2, 3], unique) + # check largest difference in distribution is 1 item + assert (np.max(counts) - np.min(counts)) <= 1 + + def test_transform_larger_than_fit_range_goes_into_last_bucket(self): + """If a value larger than the input is transformed, then it + should go into the maximum bucket""" + + arr = np.array([n + 1 for n in range(10)]) + np.random.shuffle(arr) + d = Discretiser(method="uniform", num_buckets=4) + d.fit(arr) + assert np.array_equal([3], d.transform(np.array([101]))) + + def test_transform_smaller_than_fit_range_goes_into_first_bucket(self): + """If a value smaller than the input is transformed, then it + should go into the minimum bucket""" + + arr = np.array([n + 1 for n in range(10)]) + np.random.shuffle(arr) + d = Discretiser(method="uniform", num_buckets=4) + d.fit(arr) + assert np.array_equal([0], d.transform(np.array([-101]))) + + def test_fit_transform(self): + """fit transform should give the same result as calling fit and + transform separately""" + + arr = np.array([n + 1 for n in range(10)]) + np.random.shuffle(arr) + + d1 = Discretiser(method="uniform", num_buckets=4) + d1.fit(arr) + r1 = d1.transform(arr) + + d2 = Discretiser(method="uniform", num_buckets=4) + r2 = d2.fit_transform(arr) + + assert np.array_equal(r1, r2) + + +class TestQuantile: + def test_fit_uniform_data(self): + """Fitting uniform data should produce uniform splits""" + + arr = np.array(range(100001)) + np.random.shuffle(arr) + d = Discretiser(method="quantile", num_buckets=4) + d.fit(arr) + assert np.array_equal([25000, 50000, 75000], d.numeric_split_points) + + def test_fit_gauss_data(self): + """Fitting gauss data should produce standard percentiles splits""" + + arr = np.random.normal(loc=0, scale=1, size=100001) + np.random.shuffle(arr) + d = Discretiser(method="quantile", num_buckets=4) + d.fit(arr) + assert math.isclose(-0.675, d.numeric_split_points[0], abs_tol=0.025) + assert math.isclose(0, d.numeric_split_points[1], abs_tol=0.025) + assert math.isclose(0.675, d.numeric_split_points[2], abs_tol=0.025) + + def test_transform_gauss(self): + """Fitting gauss data should transform to predictable buckets""" + + arr = np.random.normal(loc=0, scale=1, size=1000000) + np.random.shuffle(arr) + d = Discretiser(method="quantile", num_buckets=4) + d.fit(arr) + unique, counts = np.unique(d.transform(arr), return_counts=True) + # check all 4 buckets are used + assert np.array_equal([0, 1, 2, 3], unique) + assert np.array_equal([250000 for n in range(4)], counts) + + def test_fit_transform(self): + """fit transform should give the same result as calling fit and + transform separately""" + + arr = np.array([n + 1 for n in range(10)]) + np.random.shuffle(arr) + + d1 = Discretiser(method="quantile", num_buckets=4) + d1.fit(arr) + r1 = d1.transform(arr) + + d2 = Discretiser(method="quantile", num_buckets=4) + r2 = d2.fit_transform(arr) + + assert np.array_equal(r1, r2) + + +class TestOutlier: + def test_outlier_percentile_lower_boundary(self): + """Discretiser should accept lower boundary down to zero""" + + Discretiser(method="outlier", outlier_percentile=0.0) + Discretiser(method="outlier", outlier_percentile=-0.0) + with pytest.raises(ValueError): + Discretiser(method="outlier", outlier_percentile=-0.1) + + def test_outlier_percentile_upper_boundary(self): + """Discretiser should accept upper boundary up to half""" + + Discretiser(method="outlier", outlier_percentile=0.49) + with pytest.raises(ValueError): + Discretiser(method="outlier", outlier_percentile=0.5) + + def test_outlier_lower_percentile(self): + """the split point for lower outliers should be at provided percentile""" + + arr = np.array(range(100001)) + np.random.shuffle(arr) + d = Discretiser(method="outlier", outlier_percentile=0.2) + d.fit(arr) + assert d.numeric_split_points[0] == 20000 + + def test_outlier_upper_percentile(self): + """the split point for upper outliers should be at range - provided percentile""" + + arr = np.array(range(100001)) + np.random.shuffle(arr) + d = Discretiser(method="outlier", outlier_percentile=0.2) + d.fit(arr) + assert d.numeric_split_points[1] == 80000 + + def test_transform_outlier(self): + """transforming outliers should put the expected amount of data in each bucket""" + + arr = np.array(range(100001)) + np.random.shuffle(arr) + d = Discretiser(method="outlier", outlier_percentile=0.2) + d.fit(arr) + unique, counts = np.unique(d.transform(arr), return_counts=True) + # check all 3 buckets are used + assert np.array_equal([0, 1, 2], unique) + # check largest difference in outliers is 1 + print(counts) + assert np.abs(counts[0] - counts[2]) <= 1 + + def test_fit_transform(self): + """fit transform should give the same result as calling fit and + transform separately""" + + arr = np.array([n + 1 for n in range(10)]) + np.random.shuffle(arr) + + d1 = Discretiser(method="outlier", outlier_percentile=0.2) + d1.fit(arr) + r1 = d1.transform(arr) + + d2 = Discretiser(method="outlier", outlier_percentile=0.2) + r2 = d2.fit_transform(arr) + + assert np.array_equal(r1, r2) + + +class TestFixed: + def test_fit_raises_error(self): + """since numeric split points are provided, fit will not do anything""" + + d = Discretiser(method="fixed", numeric_split_points=[1]) + with pytest.raises(RuntimeError): + d.fit(np.array([1])) + + def test_fit_transform_raises_error(self): + """since numeric split points are provided, fit will not do anything""" + + d = Discretiser(method="fixed", numeric_split_points=[1]) + with pytest.raises(RuntimeError): + d.fit_transform(np.array([1])) + + def test_transform_splits_using_defined_split_points(self): + """transforming should be done using the provided numeric split points""" + + d = Discretiser(method="fixed", numeric_split_points=[10, 20, 30]) + transformed = d.transform(np.array([9, 10, 11, 19, 20, 21, 29, 30, 31])) + assert np.array_equal(transformed, [0, 1, 1, 1, 2, 2, 2, 3, 3]) + + +class TestErrorHandling: + def test_invalid_method(self): + """a value error should be raised if an invalid method is given""" + + allowed_methods = ["uniform", "quantile", "outlier", "fixed", "percentiles"] + selected_method = "INVALID" + with pytest.raises( + ValueError, + match="{0} is not a recognised method. Use one of: {1}".format( + selected_method, " ".join(allowed_methods) + ), + ): + Discretiser(method=selected_method) + + def test_uniform_requires_num_buckets(self): + """a value error should be raised if method=uniform and num_buckets is not provided""" + + selected_method = "uniform" + with pytest.raises( + ValueError, + match="{0} method expects {1}".format(selected_method, "num_buckets"), + ): + Discretiser(method=selected_method) + + def test_quantile_requires_num_buckets(self): + """a value error should be raised if method=quantile and num_buckets is not provided""" + + selected_method = "quantile" + with pytest.raises( + ValueError, + match="{0} method expects {1}".format(selected_method, "num_buckets"), + ): + Discretiser(method=selected_method) + + def test_outlier_requires_outlier_percentile(self): + """a value error should be raised if method=outlier and outlier_percentile is not provided""" + + selected_method = "outlier" + with pytest.raises( + ValueError, + match="{0} method expects {1}".format( + selected_method, "outlier_percentile" + ), + ): + Discretiser(method=selected_method) + + def test_outlier_geq_zero(self): + """a value error should be raised if outlier is not >= 0""" + + Discretiser(method="outlier", outlier_percentile=0.0) + Discretiser(method="outlier", outlier_percentile=-0.0) + Discretiser(method="outlier", outlier_percentile=0.1) + with pytest.raises( + ValueError, + match="{0} must be between 0 and 0.5".format("outlier_percentile"), + ): + Discretiser(method="outlier", outlier_percentile=-0.0000001) + + def test_outlier_lt_half(self): + """a value error should be raised if outlier is not < 0.5""" + + Discretiser(method="outlier", outlier_percentile=0.49) + with pytest.raises( + ValueError, + match="{0} must be between 0 and 0.5".format("outlier_percentile"), + ): + Discretiser(method="outlier", outlier_percentile=0.5) + + def test_fixed_split_points(self): + """a value error should be raised if method=fixed and no numeric split points are provided""" + + selected_method = "fixed" + with pytest.raises( + ValueError, + match="{0} method expects {1}".format( + selected_method, "numeric_split_points" + ), + ): + Discretiser(method=selected_method) + + def test_fixed_split_points_monotonic(self): + """a value error should be raised if numeric split points are not monotonically increasing""" + + Discretiser(method="fixed", numeric_split_points=[-1, -0, 0, 1]) + with pytest.raises( + ValueError, + match="{0} must be monotonically increasing".format("numeric_split_points"), + ): + Discretiser(method="fixed", numeric_split_points=[1, -1]) + + def test_percentile_requires_percentile_split_points(self): + """a value error should be raised if method=percentiles and no percentile split points are provided""" + + selected_method = "percentiles" + with pytest.raises( + ValueError, + match="{0} method expects {1}".format( + selected_method, "percentile_split_points" + ), + ): + Discretiser(method=selected_method) + + def test_percentile_geq_zero(self): + """a value error should be raised if not all percentiles split points >= 0""" + + Discretiser(method="percentiles", percentile_split_points=[-0.0, 0.0, 0.0001]) + with pytest.raises( + ValueError, + match="{0} must be between 0 and 1".format("percentile_split_points"), + ): + Discretiser( + method="percentiles", percentile_split_points=[-0.0000001, 0.0001] + ) + + def test_percentile_leq_1(self): + """a value error should be raised if not all percentile split points <= 1""" + + Discretiser(method="percentiles", percentile_split_points=[0.0001, 1]) + with pytest.raises( + ValueError, + match="{0} must be between 0 and 1".format("percentile_split_points"), + ): + Discretiser( + method="percentiles", percentile_split_points=[0.0001, 1.0000001] + ) + + def test_percentile_split_points_monotonic(self): + """a value error should be raised if percentile split points are not monotonically increasing""" + + Discretiser(method="percentiles", percentile_split_points=[0, -0, 0.1, 1]) + with pytest.raises( + ValueError, + match="{0} must be monotonically increasing".format( + "percentile_split_points" + ), + ): + Discretiser(method="percentiles", percentile_split_points=[1, 0.1]) + + +class TestPercentile: + def test_fit_uniform_data(self): + """Fitting uniform data should produce expected percentile splits of uniform distribution""" + + arr = np.array(range(100001)) + np.random.shuffle(arr) + d = Discretiser(method="percentiles", percentile_split_points=[0.1, 0.4, 0.85]) + d.fit(arr) + assert np.array_equal([10000, 40000, 85000], d.numeric_split_points) + + def test_fit_gauss_data(self): + """Fitting gauss data should produce percentile splits of standard normal distribution""" + + arr = np.random.normal(loc=0, scale=1, size=100001) + np.random.shuffle(arr) + d = Discretiser(method="percentiles", percentile_split_points=[0.1, 0.4, 0.85]) + d.fit(arr) + assert math.isclose(-1.2815, d.numeric_split_points[0], abs_tol=0.025) + assert math.isclose(-0.253, d.numeric_split_points[1], abs_tol=0.025) + assert math.isclose(1.036, d.numeric_split_points[2], abs_tol=0.025) + + def test_transform_uniform(self): + """Fitting uniform data should transform to predictable buckets""" + + arr = np.array(range(100001)) + np.random.shuffle(arr) + d = Discretiser( + method="percentiles", percentile_split_points=[0.10, 0.40, 0.85] + ) + d.fit(arr) + unique, counts = np.unique(d.transform(arr), return_counts=True) + # check all 4 buckets are used + assert np.array_equal([0, 1, 2, 3], unique) + assert np.array_equal([10000, 30000, 45000, 15001], counts) + + def test_fit_transform(self): + """fit transform should give the same result as calling fit and + transform separately""" + + arr = np.array([n + 1 for n in range(10)]) + np.random.shuffle(arr) + + d1 = Discretiser( + method="percentiles", percentile_split_points=[0.10, 0.40, 0.85] + ) + d1.fit(arr) + r1 = d1.transform(arr) + + d2 = Discretiser( + method="percentiles", percentile_split_points=[0.10, 0.40, 0.85] + ) + r2 = d2.fit_transform(arr) + + assert np.array_equal(r1, r2) diff --git a/tools/github_release.sh b/tools/github_release.sh new file mode 100755 index 0000000..cd44e1b --- /dev/null +++ b/tools/github_release.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +GITHUB_USER=$1 +GITHUB_REPO=$2 +GITHUB_TOKEN=$3 +VERSION=$4 + +GITHUB_ENDPOINT="https://github.com/gitapi/repos/${GITHUB_USER}/${GITHUB_REPO}/releases" + +PAYLOAD=$(cat <<-END +{ + "tag_name": "${VERSION}", + "target_commitish": "master", + "name": "${VERSION}", + "body": "Release ${VERSION}", + "draft": false, + "prerelease": false +} +END +) + +STATUS=$(curl -o /dev/null -L -s -w "%{http_code}\n" -X POST -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Content-Type: application/json" ${GITHUB_ENDPOINT} -d "${PAYLOAD}") + +[ "${STATUS}" == "201" ] || [ "${STATUS}" == "422" ] diff --git a/tools/license_and_headers.py b/tools/license_and_headers.py new file mode 100644 index 0000000..3842454 --- /dev/null +++ b/tools/license_and_headers.py @@ -0,0 +1,133 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob + +PATHS_REQUIRING_HEADER = ["causalnex", "tests"] +LEGAL_HEADER_FILE = "legal_header.txt" +LICENSE_MD = "LICENSE.md" + +RED_COLOR = "\033[0;31m" +NO_COLOR = "\033[0m" + +LICENSE = """Copyright 2019-2020 QuantumBlack Visual Analytics Limited + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +(either separately or in combination, "QuantumBlack Trademarks") are +trademarks of QuantumBlack. The License does not grant you any right or +license to the QuantumBlack Trademarks. You may not use the QuantumBlack +Trademarks or any confusingly similar mark as a trademark for your product, +or use the QuantumBlack Trademarks in any other manner that might cause +confusion in the marketplace, including but not limited to in advertising, +on websites, or on software. + +See the License for the specific language governing permissions and +limitations under the License. +""" + + +def files_at_path(path: str): + return [fn for fn in glob.glob(path + '/**/*.py', recursive=True) + if not ('ebaybbn' in fn or 'structure/notears.py' in fn)] + + +def files_missing_substring(file_names, substring): + for file_name in file_names: + with open(file_name, "r", encoding="utf-8") as current_file: + content = current_file.read() + + if content.strip() and substring not in content: + yield file_name + + +def main(): + exit_code = 0 + + with open(LEGAL_HEADER_FILE) as header_f: + header = header_f.read() + + # find all .py files recursively + files = [ + new_file for path in PATHS_REQUIRING_HEADER for new_file in files_at_path(path) + ] + + # find all files which do not contain the header and are non-empty + files_with_missing_header = list(files_missing_substring(files, header)) + + # exit with an error and print all files without header in read, if any + if files_with_missing_header: + print( + RED_COLOR + + "The legal header is missing from the following files:\n- " + + "\n- ".join(files_with_missing_header) + + NO_COLOR + + "\nPlease add it by copy-pasting the below:\n\n" + + header + + "\n" + ) + exit_code = 1 + + # check the LICENSE.md exists and has the right contents + try: + files = list(files_missing_substring([LICENSE_MD], LICENSE)) + if files: + print( + RED_COLOR + + "Please make sure the LICENSE.md file " + + "at the root of the project " + + "has the right contents." + + NO_COLOR + ) + exit(1) + except IOError: + print( + RED_COLOR + "Please add the LICENSE.md file at the root of the project " + "with the appropriate contents." + NO_COLOR + ) + exit(1) + + # if it doesn't exist, send a notice + exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/tools/min_version.py b/tools/min_version.py new file mode 100644 index 0000000..3398d19 --- /dev/null +++ b/tools/min_version.py @@ -0,0 +1,49 @@ +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import platform +import shlex +import subprocess +import sys + +if __name__ == "__main__": + required_version = tuple(int(x) for x in sys.argv[1].strip().split(".")) + install_cmd = shlex.split(sys.argv[2]) + run_cmd = shlex.split(sys.argv[3]) + + current_version = tuple(map(int, platform.python_version_tuple()[:2])) + + if current_version < required_version: + print("Python version is too low, exiting") + sys.exit(0) + + try: + subprocess.run(run_cmd, check=True) + except FileNotFoundError: + subprocess.run(install_cmd, check=True) + subprocess.run(run_cmd, check=True) diff --git a/tools/python_version.sh b/tools/python_version.sh new file mode 100755 index 0000000..f4d929e --- /dev/null +++ b/tools/python_version.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +PACKAGE_DIR=$1 + +LINE=$(perl -ne "print if /^__version__\s+=\s+\"(\d+\.\d+(\.\d+|(rc\d+)*))\"$/" \ + ${PACKAGE_DIR}/__init__.py | (head -n1 && tail -n1)) + +if [ -z "${LINE}" ]; then + exit 1 +else + VERSION=$(echo ${LINE} | perl -p -e "s/__version__\s+=\s+\"(\d+\.\d+(\.\d+|(rc\d+)*))\"/\1/g") + echo ${VERSION} +fi diff --git a/tools/python_version_dev_bump.sh b/tools/python_version_dev_bump.sh new file mode 100755 index 0000000..c514335 --- /dev/null +++ b/tools/python_version_dev_bump.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Copyright 2019-2020 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +PACKAGE_DIR=$1 + +LCA=$(git merge-base origin/develop origin/master) +CNT_LCA=$(git rev-list --count ${LCA}..HEAD) + +LINE=$(perl -ne "print if /^__version__\s+=\s+\"(\d+\.\d+(\.\d+|(rc\d+)*))\"$/" \ + ${PACKAGE_DIR}/__init__.py | (head -n1 && tail -n1)) + +if [ ! -z "${LINE}" ] && [ ! -z "${CNT_LCA}" ]; then + perl -pi -e 's/(__version__.*(\.|rc))(\d+)(.+)/$1.($3 + 1)."'".dev${CNT_LCA}"'".$4/ge' ${PACKAGE_DIR}/__init__.py +else + exit 1 +fi