A_B_Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 282,
     "status": "ok",
     "timestamp": 1644584732645,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "gvTp-m9vzlnL"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import statsmodels.stats.api as sms\n",
    "import scipy.stats as stats\n",
    "from math import ceil\n",
    "from statsmodels.stats.proportion import proportions_ztest, proportion_confint\n",
    "\n",
    "import sup_defs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What do we have here?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 342,
     "status": "ok",
     "timestamp": 1644584733377,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "ga_URFY0zpUz",
    "outputId": "f244f4c6-8cff-4d3a-b9ec-de7ad3ef4b86"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>851104</td>\n",
       "      <td>2017-01-21 22:11:48.556739</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>804228</td>\n",
       "      <td>2017-01-12 08:01:45.159739</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>661590</td>\n",
       "      <td>2017-01-11 16:55:06.154213</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>853541</td>\n",
       "      <td>2017-01-08 18:28:03.143765</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>864975</td>\n",
       "      <td>2017-01-21 01:52:26.210827</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294473</th>\n",
       "      <td>751197</td>\n",
       "      <td>2017-01-03 22:28:38.630509</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294474</th>\n",
       "      <td>945152</td>\n",
       "      <td>2017-01-12 00:51:57.078372</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294475</th>\n",
       "      <td>734608</td>\n",
       "      <td>2017-01-22 11:45:03.439544</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294476</th>\n",
       "      <td>697314</td>\n",
       "      <td>2017-01-15 01:20:28.957438</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294477</th>\n",
       "      <td>715931</td>\n",
       "      <td>2017-01-16 12:40:24.467417</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>294478 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id                   timestamp      group landing_page  converted\n",
       "0        851104  2017-01-21 22:11:48.556739    control     old_page          0\n",
       "1        804228  2017-01-12 08:01:45.159739    control     old_page          0\n",
       "2        661590  2017-01-11 16:55:06.154213  treatment     new_page          0\n",
       "3        853541  2017-01-08 18:28:03.143765  treatment     new_page          0\n",
       "4        864975  2017-01-21 01:52:26.210827    control     old_page          1\n",
       "...         ...                         ...        ...          ...        ...\n",
       "294473   751197  2017-01-03 22:28:38.630509    control     old_page          0\n",
       "294474   945152  2017-01-12 00:51:57.078372    control     old_page          0\n",
       "294475   734608  2017-01-22 11:45:03.439544    control     old_page          0\n",
       "294476   697314  2017-01-15 01:20:28.957438    control     old_page          0\n",
       "294477   715931  2017-01-16 12:40:24.467417  treatment     new_page          0\n",
       "\n",
       "[294478 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data = pd.read_csv('ab_data.csv')\n",
    "ab_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No nulls found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1644584733378,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "5y7X8mH21Lt9",
    "outputId": "bec04b0c-c999-4348-fc25-4094dafe47bb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 294478 entries, 0 to 294477\n",
      "Data columns (total 5 columns):\n",
      " #   Column        Non-Null Count   Dtype \n",
      "---  ------        --------------   ----- \n",
      " 0   user_id       294478 non-null  int64 \n",
      " 1   timestamp     294478 non-null  object\n",
      " 2   group         294478 non-null  object\n",
      " 3   landing_page  294478 non-null  object\n",
      " 4   converted     294478 non-null  int64 \n",
      "dtypes: int64(2), object(3)\n",
      "memory usage: 11.2+ MB\n"
     ]
    }
   ],
   "source": [
    "ab_data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Groups differ in size, something is wrong here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1644584733378,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "WcfsDu6r18vU",
    "outputId": "ca9e194d-0b52-4b39-d49d-8291af8f61ea"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "treatment    147276\n",
       "control      147202\n",
       "Name: group, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data['group'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The 'group' and 'landing_page' columns should be the same size, but have a different number of observations. This is a job for a ~witcher~ analyst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1644584733378,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "oXosXf6w2Io3",
    "outputId": "b82484fc-86c1-4e58-a473-819d49e74762"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "old_page    147239\n",
       "new_page    147239\n",
       "Name: landing_page, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data['landing_page'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Four thousand messed up lines, that's a lot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 282,
     "status": "ok",
     "timestamp": 1644584733657,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "3wnEFgPy1Xni",
    "outputId": "b17104a9-8a9a-4df6-ccf5-39d3e9073835"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "      <th>check_page</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>767017</td>\n",
       "      <td>2017-01-12 22:58:14.991443</td>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>733976</td>\n",
       "      <td>2017-01-11 15:11:16.407599</td>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>857184</td>\n",
       "      <td>2017-01-20 07:34:59.832626</td>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327</th>\n",
       "      <td>686623</td>\n",
       "      <td>2017-01-09 14:26:40.734775</td>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>357</th>\n",
       "      <td>856078</td>\n",
       "      <td>2017-01-12 12:29:30.354835</td>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294014</th>\n",
       "      <td>813406</td>\n",
       "      <td>2017-01-09 06:25:33.223301</td>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294200</th>\n",
       "      <td>928506</td>\n",
       "      <td>2017-01-13 21:32:10.491309</td>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294252</th>\n",
       "      <td>892498</td>\n",
       "      <td>2017-01-22 01:11:10.463211</td>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294253</th>\n",
       "      <td>886135</td>\n",
       "      <td>2017-01-06 12:49:20.509403</td>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294331</th>\n",
       "      <td>689637</td>\n",
       "      <td>2017-01-13 11:34:28.339532</td>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3893 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id                   timestamp      group landing_page  \\\n",
       "22       767017  2017-01-12 22:58:14.991443    control     new_page   \n",
       "240      733976  2017-01-11 15:11:16.407599    control     new_page   \n",
       "308      857184  2017-01-20 07:34:59.832626  treatment     old_page   \n",
       "327      686623  2017-01-09 14:26:40.734775  treatment     old_page   \n",
       "357      856078  2017-01-12 12:29:30.354835  treatment     old_page   \n",
       "...         ...                         ...        ...          ...   \n",
       "294014   813406  2017-01-09 06:25:33.223301  treatment     old_page   \n",
       "294200   928506  2017-01-13 21:32:10.491309    control     new_page   \n",
       "294252   892498  2017-01-22 01:11:10.463211  treatment     old_page   \n",
       "294253   886135  2017-01-06 12:49:20.509403    control     new_page   \n",
       "294331   689637  2017-01-13 11:34:28.339532    control     new_page   \n",
       "\n",
       "        converted check_page  \n",
       "22              0  treatment  \n",
       "240             0  treatment  \n",
       "308             0    control  \n",
       "327             0    control  \n",
       "357             0    control  \n",
       "...           ...        ...  \n",
       "294014          0    control  \n",
       "294200          0  treatment  \n",
       "294252          0    control  \n",
       "294253          0  treatment  \n",
       "294331          0  treatment  \n",
       "\n",
       "[3893 rows x 6 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data['check_page'] = ab_data['landing_page'].replace({'old_page':'control', \n",
    "                                                         'new_page':'treatment'})\n",
    "ab_data[ab_data['group'] != ab_data['check_page']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Perhaps the problem is with the users. It needs to be checked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 21,
     "status": "ok",
     "timestamp": 1644584734323,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "T8tnpPbw3AIe",
    "outputId": "31fb60ea-a480-476d-b5f4-509e32e1e129"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "710039    2\n",
       "867226    2\n",
       "911040    2\n",
       "926754    2\n",
       "870710    2\n",
       "         ..\n",
       "917311    1\n",
       "930737    1\n",
       "630252    1\n",
       "913465    1\n",
       "920411    1\n",
       "Name: user_id, Length: 290584, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data = ab_data.sort_values(by='timestamp')\n",
    "ab_data['user_id'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Very strange, the same page was shown to one user, but he was assigned to different groups. Another user was shown different pages but was assigned to the same group. If during A-B testing both options are shown to one user, he should belong to different groups, and if there is one option, then the group should be the same."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 112
    },
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1644584734323,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "BdMzEbAq3Zlk",
    "outputId": "cced32b0-f604-4051-bdf0-091499c3b000"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "      <th>check_page</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>90794</th>\n",
       "      <td>805339</td>\n",
       "      <td>2017-01-12 08:45:15.557052</td>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234436</th>\n",
       "      <td>805339</td>\n",
       "      <td>2017-01-22 23:00:09.005288</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id                   timestamp      group landing_page  \\\n",
       "90794    805339  2017-01-12 08:45:15.557052  treatment     old_page   \n",
       "234436   805339  2017-01-22 23:00:09.005288    control     old_page   \n",
       "\n",
       "        converted check_page  \n",
       "90794           0    control  \n",
       "234436          0    control  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data[ab_data['user_id'] == 805339]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 112
    },
    "executionInfo": {
     "elapsed": 17,
     "status": "ok",
     "timestamp": 1644584734324,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "loFR0wx83sKW",
    "outputId": "adbb3765-42aa-4172-df81-ab0427820c82"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "      <th>check_page</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>245524</th>\n",
       "      <td>754884</td>\n",
       "      <td>2017-01-17 23:46:45.624036</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236842</th>\n",
       "      <td>754884</td>\n",
       "      <td>2017-01-18 20:10:48.538515</td>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id                   timestamp      group landing_page  \\\n",
       "245524   754884  2017-01-17 23:46:45.624036  treatment     new_page   \n",
       "236842   754884  2017-01-18 20:10:48.538515  treatment     old_page   \n",
       "\n",
       "        converted check_page  \n",
       "245524          0  treatment  \n",
       "236842          0    control  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data[ab_data['user_id'] == 754884]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 112
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1644584734324,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "lFVKyVtP35Wm",
    "outputId": "b242c83c-4d06-4456-a9c3-e38e93c6d76c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "      <th>check_page</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>262554</th>\n",
       "      <td>722274</td>\n",
       "      <td>2017-01-09 21:21:23.638444</td>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105487</th>\n",
       "      <td>722274</td>\n",
       "      <td>2017-01-19 01:46:53.093257</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id                   timestamp    group landing_page  converted  \\\n",
       "262554   722274  2017-01-09 21:21:23.638444  control     new_page          0   \n",
       "105487   722274  2017-01-19 01:46:53.093257  control     old_page          0   \n",
       "\n",
       "       check_page  \n",
       "262554  treatment  \n",
       "105487    control  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data[ab_data['user_id'] == 722274]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "executionInfo": {
     "elapsed": 15,
     "status": "ok",
     "timestamp": 1644584734325,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "z8XtaTgJ2aLD"
   },
   "source": [
    "If you remove all inconsistencies, all duplicate users disappear (except one)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 15,
     "status": "ok",
     "timestamp": 1644584734325,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "uC0qyifOCXYT",
    "outputId": "9bb19014-1210-4b01-c099-97cca72e65e1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "773192    2\n",
       "922696    1\n",
       "829063    1\n",
       "867327    1\n",
       "762358    1\n",
       "         ..\n",
       "655980    1\n",
       "861027    1\n",
       "887897    1\n",
       "924198    1\n",
       "920411    1\n",
       "Name: user_id, Length: 290584, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data_cl = ab_data[ab_data['group'] == ab_data['check_page']]\n",
    "ab_data_cl['user_id'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1644584734325,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "gnpd1qQ6FVcK"
   },
   "source": [
    "I think it's better to ignore this exception"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 386,
     "status": "ok",
     "timestamp": 1644584734704,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "k7Zcv4dc2c1T",
    "outputId": "73220f86-2718-432c-e1ac-27ead8202e83"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "new_page    145310\n",
       "old_page    145274\n",
       "Name: landing_page, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data_cl = ab_data_cl.drop_duplicates(subset='user_id', keep='first')\n",
    "ab_data_cl['landing_page'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17,
     "status": "ok",
     "timestamp": 1644584734705,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "idA5PeE22eM6",
    "outputId": "b44a0a07-5507-4919-e1e0-ef1084714833"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "treatment    145310\n",
       "control      145274\n",
       "Name: group, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data_cl['group'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Columns now match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 49
    },
    "executionInfo": {
     "elapsed": 13,
     "status": "ok",
     "timestamp": 1644584734705,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "gdlehD9cFgWg",
    "outputId": "56f4ed49-c570-444e-bed8-a9a67842e337"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "      <th>check_page</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [user_id, timestamp, group, landing_page, converted, check_page]\n",
       "Index: []"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_data_cl[ab_data_cl['group'] != ab_data_cl['check_page']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After processing the data, it is necessary to determine the number of observations sufficient to conduct the test. \n",
    "\n",
    "As always, we will test the null hypothesis H0, which states that there are no significant differences between the new and old designs. We will also test the alternative hypothesis H1, which states that there are still differencesare still differences\n",
    "\n",
    "I see no need to break the \"gold standard\" and chose alpha = 0.05, power = 0.8. The proportion_effectsize function calculates the effect size for a test comparing two proportions. The conversion rate is currently 12%, with the new page we expect this to increase by 2%. We need 4433 observations for each group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 242,
     "status": "ok",
     "timestamp": 1644586225535,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "O0wqj8X6KVoG",
    "outputId": "d0464884-9215-47c8-bb06-393f28f7e0c4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4433"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prop_effsize = sms.proportion_effectsize(0.12, 0.14)\n",
    "alpha = 0.05\n",
    "power = 0.8\n",
    "\n",
    "required_n = sms.NormalIndPower().solve_power(\n",
    "    prop_effsize,  \n",
    "    alpha=alpha,\n",
    "    power=power\n",
    "    )\n",
    "required_n = ceil(required_n)\n",
    "required_n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The required number of observations was placed in a separate dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1644584734706,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "Uj6aRrfGjihp",
    "outputId": "3166a1a5-e3dc-455a-a089-52fc5a31b95d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "      <th>check_page</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>781507</td>\n",
       "      <td>2017-01-02 13:42:15.234051</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>737319</td>\n",
       "      <td>2017-01-02 13:42:21.786186</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>799109</td>\n",
       "      <td>2017-01-02 13:42:45.219901</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>643921</td>\n",
       "      <td>2017-01-02 13:43:03.767329</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>1</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>677659</td>\n",
       "      <td>2017-01-02 13:43:32.744957</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8861</th>\n",
       "      <td>816556</td>\n",
       "      <td>2017-01-03 05:39:04.377045</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8862</th>\n",
       "      <td>706254</td>\n",
       "      <td>2017-01-03 05:39:46.805443</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8863</th>\n",
       "      <td>872254</td>\n",
       "      <td>2017-01-03 05:40:18.166385</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8864</th>\n",
       "      <td>832214</td>\n",
       "      <td>2017-01-03 05:40:27.946779</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8865</th>\n",
       "      <td>660194</td>\n",
       "      <td>2017-01-03 05:40:38.227138</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>treatment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8866 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id                   timestamp      group landing_page  converted  \\\n",
       "0      781507  2017-01-02 13:42:15.234051    control     old_page          0   \n",
       "1      737319  2017-01-02 13:42:21.786186    control     old_page          0   \n",
       "2      799109  2017-01-02 13:42:45.219901    control     old_page          0   \n",
       "3      643921  2017-01-02 13:43:03.767329    control     old_page          1   \n",
       "4      677659  2017-01-02 13:43:32.744957    control     old_page          0   \n",
       "...       ...                         ...        ...          ...        ...   \n",
       "8861   816556  2017-01-03 05:39:04.377045  treatment     new_page          0   \n",
       "8862   706254  2017-01-03 05:39:46.805443  treatment     new_page          0   \n",
       "8863   872254  2017-01-03 05:40:18.166385  treatment     new_page          0   \n",
       "8864   832214  2017-01-03 05:40:27.946779  treatment     new_page          0   \n",
       "8865   660194  2017-01-03 05:40:38.227138  treatment     new_page          0   \n",
       "\n",
       "     check_page  \n",
       "0       control  \n",
       "1       control  \n",
       "2       control  \n",
       "3       control  \n",
       "4       control  \n",
       "...         ...  \n",
       "8861  treatment  \n",
       "8862  treatment  \n",
       "8863  treatment  \n",
       "8864  treatment  \n",
       "8865  treatment  \n",
       "\n",
       "[8866 rows x 6 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ab_required = pd.concat(\n",
    "    [ab_data_cl[ab_data_cl['group'] == 'control'].head(required_n),\n",
    "     ab_data_cl[ab_data_cl['group'] == 'treatment'].head(required_n)], ignore_index=True)\n",
    "\n",
    "ab_required"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To evaluate the performance of a new page, we need to calculate the conversion, standard deviation, and standard error of the mean for each group. At this stage, the conclusion is that the new page is not more efficient than the old one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1644584734706,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "u6x9zdjLu0YA",
    "outputId": "5bea5abd-837b-4162-abba-43b609a80291"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_90442_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >conversion_rate</th>\n",
       "      <th class=\"col_heading level0 col1\" >std_deviation</th>\n",
       "      <th class=\"col_heading level0 col2\" >std_error</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >group</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_90442_level0_row0\" class=\"row_heading level0 row0\" >control</th>\n",
       "      <td id=\"T_90442_row0_col0\" class=\"data row0 col0\" >0.120</td>\n",
       "      <td id=\"T_90442_row0_col1\" class=\"data row0 col1\" >0.324</td>\n",
       "      <td id=\"T_90442_row0_col2\" class=\"data row0 col2\" >0.005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_90442_level0_row1\" class=\"row_heading level0 row1\" >treatment</th>\n",
       "      <td id=\"T_90442_row1_col0\" class=\"data row1 col0\" >0.117</td>\n",
       "      <td id=\"T_90442_row1_col1\" class=\"data row1 col1\" >0.321</td>\n",
       "      <td id=\"T_90442_row1_col2\" class=\"data row1 col2\" >0.005</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x228fd10f3a0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "std_dev = lambda x: np.std(x, ddof=0)\n",
    "std_err = lambda x: stats.sem(x, ddof=0)\n",
    "\n",
    "conv_rates = ab_required.groupby('group')['converted']\n",
    "conv_rates = conv_rates.agg([np.mean, std_dev, std_err])\n",
    "\n",
    "conv_rates.columns = ['conversion_rate', 'std_deviation', 'std_error']\n",
    "\n",
    "conv_rates.style.format('{:.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 517
    },
    "executionInfo": {
     "elapsed": 402,
     "status": "ok",
     "timestamp": 1644584735101,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "fguT3BWLtYhS",
    "outputId": "8b344a59-dbec-414d-c104-8037ddecfad1"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArwAAAH0CAYAAADfWf7fAAAgAElEQVR4Xu3db4xe130n9kNJkcexQ+9swm0kMyww3jTrdK12M2ALqMqbAK1Qheg0u10hUQwHKOKGzpKw9IbYGZD5R4JczBvLIBOPmqAvvIltaIskXBBcNEbfpQFcgVqss/6TjUvACmN7w80yov/RimQW58ne4cOHz8xz733uv3PuZ94kEs8953c+vzPDr67v3OfA3bt37wZfBAgQIECAAAECBDIVOCDwZtpZ2yJAgAABAgQIEJgICLwOAgECBAgQIECAQNYCAm/W7bU5AgQIECBAgAABgdcZIECAAAECBAgQyFpA4M26vTZHgAABAgQIECAg8DoDBAgQIECAAAECWQsIvFm31+YIECBAgAABAgQEXmeAAAECBAgQIEAgawGBN+v22hwBAgQIECBAgIDA6wwQIECAAAECBAhkLSDwZt1emyNAgAABAgQIEBB4nQECBAgQIECAAIGsBQTerNtrcwQIECBAgAABAgKvM0CAAAECBAgQIJC1gMCbdXttjgABAgQIECBAQOB1BggQIECAAAECBLIWEHizbq/NESBAgAABAgQICLzOAAECBAgQIECAQNYCAm/W7bU5AgQIECBAgAABgdcZIECAAAECBAgQyFpA4M26vTZHgAABAgQIECAg8DoDBAgQIECAAAECWQsIvFm31+YIECBAgAABAgQEXmeAAAECBAgQIEAgawGBN+v22hwBAgQIECBAgIDA6wwQIECAAAECBAhkLSDwZt1emyNAgAABAgQIEBB4nQECBAgQIECAAIGsBQTerNtrcwQIECBAgAABAgKvM0CAAAECBAgQIJC1gMCbdXttjgABAgQIECBAQOB1BggQIECAAAECBLIWEHizbq/NESBAgAABAgQICLzOAAECBAgQIECAQNYCAm/W7bU5AgQIECBAgAABgdcZIECAAAECBAgQyFpA4M26vTZHgAABAgQIECAw2MB7586dcP78+fDKK6/c16WjR4+Gra2tsLKyMpruXb58Oezs7IRz586F9fX1JPd98eLF8NnPfjZsb2+H1dXVJPegaAIECBAgQCBNgUEG3mvXroXTp0+H2XBbhOA/+ZM/CRcuXAhra2tpqlesWuCtCGY4AQIECBAgQGBKYHCB9/r162FzczM89dRT4eTJk3Ob9elPfzq85z3vGU3gzeHEusObQxftgQABAgQIpCkwuMAbg9Ef/uEfVrqDW4Tk27dvT7pw8ODBB64vxjz33HOTMfERgeJr+lGBeDf1E5/4xNz15/3ZrVu3wqlTp8KNGzfmzhf/ZRH2fu3Xfi187GMfmzymMV1j/POrV6/uXn/48OH7/qf/vWoq7vwWF8573KO49pd/+ZfDpz71qd1HRGbX2O/47lX/r/zKr4RPfvKTCx87md3fvB4Vd/WLOqrUl+a3nqoJECBAgACBrgQGFXiL8PjYY4+Vfk63CEqzoXX2mdfpUHz8+PGwsbExMZ4Nk0UNTzzxxH13mIvHKeI1xTPExdrT800H62KNIvDNC+Kzdz7nrTMv8O513Ve/+tUHwnK0mF573hqLAm8M5LP1x3lefPHF8Oyzz+7ebd9r7v3u8M77j5z9/sOjq28O6xAgQIAAAQJ5CAwy8M6Gzb2o9wqn03dVi1+SmhdE47hijmPHju2G4HnhrLg+3s2NvzhWBLtDhw498OjF7PV7hb396o+Pbfz4j//45JfzZsPfvJA/vZdpv72CY5wj2pR5Frrq4wjz5t5rjlnXotf72eTxrWcXBAgQIECAQFcCSQfevcLSvDu3iwLvdEjc6y7t9FsG9lt7NvDtFfam30QxfZd4tvmzoTX+85UrVx5440HZu8Nx/r1C87yDtyjwzj5aEeeYvRu81xx77aX4j5abN2+Wvtvf1TeNdQgQIECAAIG0BAYZeMs+0rBfaJv9syqBdzY4fvvb3548pzt9F3j2mdPZts8+o7vXK7nmvX5tNizOBt5FjwdMr7XfHd74JowyrzpbdIc6Pjs9fae4yh3eec/3TluO8TV0af0IUS0BAgQIEBi+wKACb3FXr+wvrbV1h7e4A1r8T/5//Md//MAvsu239mzbF90hnR5fzBtDb/E4xlDv8FZ5XKLOHd7hf/uokAABAgQIEEhBYHCBt8xryT7/+c9Pnm2NH2AQ77zOe+Z3NmBVucMbG1c8Qxpff/atb30rzD6rW+UZ0/3ukH7uc5+bvIJt+mtewJ1+c0RTz/Aue4d3r33Nu8O716MLVR6tSOEbSo0ECBAgQIDA8AQGF3iLu6tlP3hi3psS5n1QQ9XAG+uYfjZ13v/0X6z9zDPP3PeLazEMf+QjHwkvvPDCJJQveiRgOrCXfQ539s0GxXXz3tIw7zVrVYLmfsF2NjQX884+lrHfesVjDbPG8Zo/+qM/2vN9zMP7dlIRAQIECBAgMESBQQbeCFXlo4Wrvoe3eF3Y9J3ceXeJi3l/5Ed+ZM9fnJr3Ht447/Qvoe33SMNs7fHa2QDdxHt4Z9/G0ETgnf6Pk+Jwx9qffPLJuW+AmP4PiL0C8fQ3ybzXuA3xm0hNBAgQIECAwLAFBht4h82mOgIECBAgQIAAgVQEBN5UOqVOAgQIECBAgACBWgICby02FxEgQIAAAQIECKQiIPCm0il1EiBAgAABAgQI1BIQeGuxuYgAAQIECBAgQCAVAYE3lU6pkwABAgQIECBAoJaAwFuLzUUECBAgQIAAAQKpCAi8qXRKnQQIECBAgAABArUEBN5abC4iQIAAAQIECBBIRUDgTaVT6iRAgAABAgQIEKglIPDWYnMRAQIECBAgQIBAKgICbyqdUicBAgQIECBAgEAtAYG3FpuLCBAgQIAAAQIEUhEQeFPplDoJECBAgAABAgRqCQi8tdhcRIAAAQIECBAgkIqAwJtKp9RJgAABAgQIECBQS0DgrcXmIgIECBAgQIAAgVQEBN5UOqVOAgQIECBAgACBWgICby02FxEgQIAAAQIECKQiIPCm0il1EiBAgAABAgQI1BIQeGuxuYgAAQIECBAgQCAVAYE3lU6pkwABAgQIECBAoJaAwFuLzUUECBAgQIAAAQKpCAi8qXRKnQQIECBAgAABArUEBN5abC4iQIAAAQIECBBIRUDgTaVT6iRAgAABAgQIEKglIPDWYnMRAQIECBAgQIBAKgICbyqdUicBAgQIECBAgEAtAYG3FpuLCBAgQIAAAQIEUhEQeFPplDoJECBAgAABAgRqCQi8tdhcRIAAAQIECBAgkIqAwJtKp9RJgAABAgQIECBQS0DgrcXmIgIECBAgQIAAgVQEBN5UOqVOAgQIECBAgACBWgICby02FxEgQIAAAQIECKQiIPCm0il1EiBAgAABAgQI1BIQeGuxuYgAAQIECBAgQCAVAYE3lU6pkwABAgQIECBAoJaAwFuLzUUECBAgQIAAAQKpCAi8qXRKnQQIECBAgAABArUEBN5abC4iQIAAAQIECBBIRUDgTaVT6iRAgAABAgQIEKglIPDWYnMRAQIECBAgQIBAKgICbyqdUicBAgQIECBAgEAtAYG3FpuLCBAgQIAAAQIEUhEQeFPplDoJECBAgAABAgRqCQi8tdhcRIAAAQIECBAgkIqAwJtKp9RJgAABAgQIECBQS0DgrcXmIgIECBAgQIAAgVQEBN5UOqVOAgQIECBAgACBWgICby02FxEgQIAAAQIECKQiIPCm0il1EiBAgAABAgQI1BIQeGuxuYgAAQIECBAgQCAVAYE3lU6pkwABAgQIECBAoJaAwFuLzUUECBAgQIAAAQKpCAi8qXRKnQQIECBAgAABArUEBN5abC4iQIAAAQIECBBIRUDgTaVT6iRAgAABAgQIEKglIPDWYnMRAQIECBAgQIBAKgICbyqdUicBAgQIECBAgEAtgSwD761bt8KpU6fCjRs3Jijnzp0L6+vrC4EuXrwYjhw5EjY2NnbHXrt2LZw+fXr3n48ePRq2trbCysrKwvkMIECAAAECBAgQ6F8gu8B7586dcP78+UnAjcH1+vXr4cKFC2FzczOsra3NFb98+XLY2dmZ/Nnx48fvC7zxzw4fPjyZr5j70KFD4eTJk/13TwUECBAgQIAAAQILBbILvDHgXrp0KZw5cyasrq7uhtQiAO8nMu8O7+z4GIDjXV93eReeLQMIECBAgAABAoMQyC7wxjAa79Zub29PAm/8ikE2fi26K1sm8JadaxDdVQQBAgQIECBAgEDIMvDGu7DTd2DLhtRFgXdemP76t990jAgQIECAAAECrQp839sfaXX+3CfPMvC2cYc3ht141zg+Dzz9LPB33nirszPy0EMHOlvLQgTGJvDd794d25btlwCBhATe9ujDCVU7vFKzC7xtPMO7V9jtup2/9qf/Z/j8N/+862WtRyB7gfe+893hl//u/5L9Pm2QAAECYxXILvAuektD8cqyY8eO3fc2hngA9not2ewd474Oy0//6xfDZ17/Ul/LW5dAtgL/7bv+bvjUP3g+2/3ZGAECBMYukF3gjQ3d7z288wLv9GvJ4vUHDx7cfXQhhuCrV6/ed06m/7zLAyTwdqltrTEJCLxj6ra9EiAwRoEsA2+ujRR4c+2sffUtIPD23QHrEyBAoF0Bgbdd30ZnF3gb5TQZgV0BgddhIECAQN4CAm9C/RV4E2qWUpMSEHiTapdiCRAgUFlA4K1M1t8FAm+39t97/Zvh8U/emCz6xvc/Gv78/T8U3nrn/u9BfPRrd8Kh/+vfh6/9o3ffN/bQv/paeNerr+9u4OZ//3fC6//N33wwSvx6+Btvhnf/9p+FR//yjck/f+VnDodvrb2j2w2PeDWBd8TNt3UCBEYhIPAm1GaBt7tmxeD6g7/3lfC1n3o8vPGDK+Fd/++tEAPw1/7h4+Huow89UMh0YJ0Nxwfe+G74gf/7L8J//PEfmITgOPfjn/iz8Bf/8+OTUBv//Ad/9yuT/z+G4Nm1u9v1eFfKOfC+deVXw92vfnG8zbVzAi0JHHjsveHhY7/U0uymbVpA4G1atMX5BN4WcWemjgH30b/8Trj5P/7g5E/KhtC97vBOTz8v4E7fFZ798+52Pd6Vcg68b37858PdL7863ubaOYGWBA785z8WHvnAb7U0u2mbFhB4mxZtcT6Bt0XcmanjIwjxqwi8xR3c//A//J19HzUoE3hn54p3jn/gD/7ivkcmZtfvbufjXEngHWff7ZrAMgIC7zJ63V8r8HZvXntFgbc2XeULY+B84/vftvucbZOBdzbMxsAb7yhPPy4h8FZu2VIXCLxL8bmYwCgFBN602i7wJtQvgbe7ZrV1hzfO+8jrb94Xbt3h7a6ve60k8PbfAxUQSE1A4E2rYwJvQv0SeLtrVhvP8M4Lu3FHs49BeIa3uz4XKwm83ZtbkUDqAgJvWh0UeBPql8DbXbMWvaUhBuJ3vfpXD7yqbL/XksXqi2eCp3fiLQ3d9dUd3v6tc6/g2n/8nnDm3x6cbPOH3v5m+Gf/1e2w+ujdfbd9/RsPh1//03eE0//l1+eOvfznbwt/9q1Hwokf/ubuPPGarc8eDLffvP+tMUf/9hth871fDysP5y7d//4E3v57UKUCgbeKVs9jBd5uG7Dfe3hnA+/se3Rjpa//2LsmAXfen8U//+Z73rH7aIP38Hbb29nV3OHt1z+X1WMI/WdfeGf4p+/9Rlh751shBtVXbz26ZwC99caB8E//zcHwZ99+ZG44ng7Pzzx2577AO8/s0p++I/zQ974ZNt79nVxIB70PgXfQ7XmgOIE3oX4JvAk1S6lJCQi8SbVrsMXO3omdDcB7FV7nDu/sXIvmGCxawoUJvGk1T+BNqF8Cb0LNUmpSAgJvUu0abLHxDmv8Kh49KO7g/m/v+VZY/9t/vWfdi8LqvEcaZidzd7f7YyHwdm++zIoC7zJ6HV8r8HYMbrnRCAi8o2l1qxudDZ1dBd5FgbnVTY94coE3reYLvAn1S+BNqFlKTUpA4E2qXYMtto87vHfeCuHCF74v/NjqG57d7fhkCLwdgy+5nMC7JGCXlwu8XWpba0wCAu+Yut3eXvt4hjf+Ytv//v99b6m3QbS383HOLPCm1XeBN6F+CbwJNUupSQkIvEm1a7DFLnpLQwzEV7+y8kA4XfRIwl7P8Lq72+9REHj79a+6usBbVazH8QJvj/iWzlpA4M26vZ1ubr/38M4G3unXkhVFTr9+bHqu4s/P/v3bu78At+i1Z51ufISLCbxpNV3gTahfAm9CzVJqUgICb1LtUiyBQQgIvINoQ+kiBN7SVP0PFHj774EK8hQQePPsq10RaFNA4G1Tt/m5Bd7mTVubUeBtjdbEIxcQeEd+AGyfQA0BgbcGWo+XCLw94lddWuCtKmY8gXICAm85J6MIELgnIPCmdRoE3oT6JfAm1CylJiUg8CbVLsUSGISAwDuINpQuQuAtTdX/QIG3/x6oIE8BgTfPvtoVgTYFBN42dZufW+Bt3rS1GQXe1mhNPHIBgXfkB8D2CdQQEHhroPV4icDbI37VpQXeqmLGEygnIPCWczKKAIF7AgJvWqdB4E2oXwJvQs1SalICAm9S7VIsgUEICLyDaEPpIgTe0lT9DxR4+++BCvIUEHjz7KtdEWhTQOBtU7f5uQXe5k1bm1HgbY3WxCMXEHhHfgBsn0ANAYG3BlqPlwi8PeJXXVrgrSpmPIFyAgJvOSejCBC4JyDwpnUaBN6E+iXwJtQspSYlIPAm1S7FEhiEgMA7iDaULkLgLU3V/0CBt/8eqCBPAYE3z77aFYE2BQTeNnWbn1vgbd60tRkF3tZoTTxyAYF35AfA9gnUEBB4a6D1eInA2yN+1aUF3qpixhMoJyDwlnMyigCBewICb1qnQeBNqF8Cb0LNUmpSAgJvUu1SLIFBCAi8g2hD6SIE3tJU/Q8UePvvgQryFBB48+yrXRFoU0DgbVO3+bkF3uZNW5tR4G2N1sQjFxB4R34AbJ9ADQGBtwZaj5cIvD3iV11a4K0qZjyBcgICbzknowgQuCcg8KZ1GgTehPol8CbULKUmJSDwJtUuxRIYhIDAO4g2lC5C4C1N1f9Agbf/HqggTwGBN8++2hWBNgUE3jZ1m59b4G3etLUZBd7WaE08cgGBd+QHwPYJ1BAQeGug9XjJKAPvtWvXwunTpyfshw8fDtvb22F1dXXfNly/fj1cunQpnDlzZuHYtvop8LYla96xCwi8Yz8B9k+guoDAW92szytGF3hjcL1w4ULY3NwMa2tr4fLlyyEG4K2trbCysvJAL27duhVOnToVbty4UToct9VQgbctWfOOXUDgHfsJsH8C1QUE3upmfV4xusAbA+5rr70WTp48OXGfDcB7NcMd3j6PqbUJtCsg8Lbra3YCOQoIvGl1dXSB9+LFi5MOFYG3uIN7/PjxsL6+vmf3BN60DrZqCVQREHiraBlLgEAUEHjTOgejDLxHjhwJGxsbk04tG3j/6htvdNLxRx4+EH7+i78ePvP6lzpZzyIExiQQA+9v/b1/Et58625W244/N1b+xYfC3S+/mtW+bIbAEARi4L3zjz/W2c+Nv/XOR4ew7WRrGGXgbfIOb1d/QR4IIfzsZz8q8Cb7rabwIQvEwPs7T3w45BV3Q4g/N+7+zgcF3iEfPrUlKxAD74Gf/c3Ofm7E/4D1VV9gdIHXM7z1D4srCeQq4JGGXDtrXwTaE/BIQ3u2bcw8usC76C0NMRBfuXLlgVeVeYa3jeNnTgLDEBB4h9EHVRBISUDgTalbIYwu8Mb27Pce3tnAO/1asqK1zzzzzO4vvXXZbq8l61LbWmMSEHjH1G17JdCMgMDbjGNXs4wy8HaF2/Q6Am/TouYj8DcCAq+TQIBAVQGBt6pYv+MF3n79K60u8FbiMphAaQGBtzSVgQQI/CcBgTetoyDwJtQvgTehZik1KQGBN6l2KZbAIAQE3kG0oXQRAm9pqv4HCrz990AFeQoIvHn21a4ItCkg8Lap2/zcAm/zpq3NKPC2RmvikQsIvCM/ALZPoIaAwFsDrcdLBN4e8asuLfBWFTOeQDkBgbeck1EECNwTEHjTOg0Cb0L9EngTapZSkxIQeJNql2IJDEJA4B1EG0oXIfCWpup/oMDbfw9UkKeAwJtnX+2KQJsCAm+bus3PLfA2b9rajAJva7QmHrmAwDvyA2D7BGoICLw10Hq8RODtEb/q0gJvVTHjCZQTEHjLORlFgMA9AYE3rdMg8CbUL4E3oWYpNSkBgTepdimWwCAEBN5BtKF0EQJvaar+Bwq8/fdABXkKCLx59tWuCLQpIPC2qdv83AJv86atzSjwtkZr4pELCLwjPwC2T6CGgMBbA63HSwTeHvGrLi3wVhUznkA5AYG3nJNRBAjcExB40zoNAm9C/RJ4E2qWUpMSEHiTapdiCQxCQOAdRBtKFyHwlqbqf6DA238PVJCngMCbZ1/tikCbAgJvm7rNzy3wNm/a2owCb2u0Jh65gMA78gNg+wRqCAi8NdB6vETg7RG/6tICb1Ux4wmUExB4yzkZRYDAPQGBN63TIPAm1C+BN6FmKTUpAYE3qXYplsAgBATeQbShdBECb2mq/gcKvP33QAV5Cgi8efbVrgi0KSDwtqnb/NwCb/Omrc0o8LZGa+KRCwi8Iz8Atk+ghoDAWwOtx0sE3h7xqy4t8FYVM55AOQGBt5yTUQQI3BMQeNM6DQJvQv0SeBNqllKTEhB4k2qXYgkMQkDgHUQbShch8Jam6n+gwNt/D1SQp4DAm2df7YpAmwICb5u6zc8t8DZv2tqMAm9rtCYeuYDAO/IDYPsEaggIvDXQerxE4O0Rv+rSAm9VMeMJlBMQeMs5GUWAwD0BgTet0yDwJtQvgTehZik1KQGBN6l2KZbAIAQE3kG0oXQRAm9pqv4HCrz990AFeQoIvHn21a4ItCkg8Lap2/zcAm/zpq3NKPC2RmvikQsIvCM/ALZPoIaAwFsDrcdLBN4e8asuLfBWFTOeQDkBgbeck1EECNwTEHjTOg0Cb0L9EngTapZSkxIQeJNql2IJDEJA4B1EG0oXIfCWpup/oMDbfw9UkKeAwJtnX+2KQJsCAm+bus3PLfA2b9rajAJva7QmHrmAwDvyA2D7BGoICLw10Hq8RODtEb/q0gJvVTHjCZQTEHjLORlFgMA9AYE3rdMg8CbUL4E3oWYpNSkBgTepdimWwCAEBN5BtKF0EQJvaar+Bwq8/fdABXkKCLx59tWuCLQpIPC2qdv83I0G3jt37oTz58+HV155JRw8eDBcuHAhPP7445N/t76+HjY2NprfwYhmFHhH1Gxb7VRA4O2U22IEshAQeNNqY6OB9+LFi+HIkSPh6aefDi+++GJ49tlnw9raWrh27Vq4fPly2NraCisrK2kJDahagXdAzVBKVgICb1bttBkCnQgIvJ0wN7ZIY4H31q1b4ezZs+HEiROTu7rTgff69evh0qVL4cyZM2F1dbWx4sc2kcA7to7bb1cCAm9X0tYhkI+AwJtWLzsJvO7wNnMoBN5mHM1CYFZA4HUmCBCoKiDwVhXrd3xjgTduIz62EMPtCy+8EF566aXJIw3xju6pU6fCsWPHPMO7ZK8F3iUBXU5gDwGB19EgQKCqgMBbVazf8Y0G3riVGHhPnz59367OnTs3+aU1X8sJCLzL+bmawF4CAq+zQYBAVQGBt6pYv+MbD7z9budvVo/PE8e7yjdu3Jj886LAvWh8/GW8q1evTuY6fPhw2N7e7uVZZIF3CKdLDTkKCLw5dtWeCLQrIPC269v07I0F3ulfWotvZpj+6vIZ3uLVaMVr0OIvzMXXo21ubk7eGDH7tWh88ZhG8YaJ2X9uuiH7zSfwdqltrTEJCLxj6ra9EmhGQOBtxrGrWToJvF2+pWF2rdlAOwu7aHy8uxu/Tp48Ofm/Mbzv7Oz0cpdX4O3q28I6YxMQeMfWcfslsLyAwLu8YZczdBJ4u7wrOi+QzobW2bvPswF2enwMxPHu8FNPPTUJvcW7hvv4EA2Bt8tvDWuNSUDgHVO37ZVAMwICbzOOXc2ydOAtAuHt27f3rLn41LV5jxQ0vdF5j08sCryzH4oxPb64Q/z1r389fPGLX3zgGd6/vP2dprcwd75HHj4QfuHf/Ub4zOtf6mQ9ixAYk0AMvC/98C+GN9+6m9W248+Nd/zuL4a7X341q33ZDIEhCMTA+81/9Bud/dz4/oNvG8K2k61h6cBb7Hy/Z3i71Gn6Du/sHd0Yjq9cubL7SMN3v9vdX5DP/ZuPCrxdHiZrjUYgBt5P/NcfznK/3/3nHxR4s+ysTfUtEAPvQ+//zc7KeOihA52tleNCjQXeoeAseiZ3ts79xsePSD5//vzk/cHFa9UW/RJcmw4eaWhT19xjFvBIw5i7b+8E6gl4pKGeW19XZRd4F711oXgFWfFBGIvGxzu8N2/eDNNvaZi+w9tl4wTeLrWtNSYBgXdM3bZXAs0ICLzNOHY1S6OBd7/nebt8f+1+79WdDbwRer/xRSB+5ZVXJj3pch+zh0Dg7erbwjpjExB4x9Zx+yWwvIDAu7xhlzM0Fnin75S+733vCy+//HJ4/vnnw8rKyuTNBk8++aRPW1uyswLvkoAuJ7CHgMDraBAgUFVA4K0q1u/4xgLv9C+txS1dunQpnDlzZvKJZF1+8ES/nO2uLvC262v28QoIvOPtvZ0TqCsg8NaV6+e6VgJvDLkf+chHwgsvvDAJvF1+8EQ/jN2sKvB242yV8QkIvOPruR0TWFZA4F1WsNvrGwu8s7/8Nf06ry4/eKJbvm5XE3i79bbaeAQE3vH02k4JNCUg8DYl2c08jQXe2XKnfxGsyw+e6Iatn1UE3n7crZq/gMCbf4/tkEDTAgJv06Ltztda4G237HHOLvCOs+923b6AwNu+sRUI5CYg8KbV0cYC71A+aS0t/mrVCrzVvIwmUFZA4C0rZRwBAoWAwJvWWRB4E+qXwJtQs5SalIDAm1S7FBZsWWkAACAASURBVEtgEAIC7yDaULqIxgJvXNH7dku71xoo8NZicxGBhQIC70IiAwgQmBEQeNM6Eo0G3vj6sekPnEiLYvjVCrzD75EK0xQQeNPsm6oJ9Ckg8PapX33txgLv7MfzzpbS50fyVmcZ5hUC7zD7oqr0BQTe9HtoBwS6FhB4uxZfbr3GAu9yZbi6jIDAW0bJGALVBQTe6mauIDB2AYE3rRMg8CbUL4E3oWYpNSkBgTepdimWwCAEBN5BtKF0EQJvaar+Bwq8/fdABXkKCLx59tWuCLQpIPC2qdv83AJv86atzSjwtkZr4pELCLwjPwC2T6CGgMBbA63HSwTeHvGrLi3wVhUznkA5AYG3nJNRBAjcExB40zoNAm9C/RJ4E2qWUpMSEHiTapdiCQxCQOAdRBtKF9Fo4N3v1WReS1a6J3sOFHiXNzQDgXkCAq9zQYBAVQGBt6pYv+MbDbzxk9bi18mTJ/vdVaarC7yZNta2ehcQeHtvgQIIJCcg8KbVssYCb7y7e/bs2XDixImwtraWlkIi1Qq8iTRKmckJCLzJtUzBBHoXEHh7b0GlAgTeSlz9DhZ4+/W3er4CAm++vbUzAm0JCLxtybYzb2OBN5YXH2k4cuRI2NjYaKfakc8q8I78ANh+awICb2u0JiaQrYDAm1ZrGw28169fDy+//HJ4/vnnw8rKSloSCVQr8CbQJCUmKSDwJtk2RRPoVUDg7ZW/8uKNBd793tAQq/KWhsq9eeACgXd5QzMQmCcg8DoXBAhUFRB4q4r1O76xwNvvNsaxusA7jj7bZfcCAm/35lYkkLqAwJtWBwXehPol8CbULKUmJSDwJtUuxRIYhIDAO4g2lC6i8cB77dq1cPr06fsKOHfuXFhfXy9dlIHzBQReJ4NAOwICbzuuZiWQs4DAm1Z3Gw28Mezu7OyE7e3tsLq6OpGIv8i2ubkZnnvuOW9vWPJsCLxLArqcwB4CAq+jQYBAVQGBt6pYv+MbC7x37twJ58+fn4Ta2bu5MQhfvnw5bG1teXvDEv0WeJfAcymBfQQEXseDAIGqAgJvVbF+xzcWePf7pLV4l/fSpUvhzJkzu3d++912mqsLvGn2TdXDFxB4h98jFRIYmoDAO7SO7F9PY4HXHd72Gy/wtm9shXEKCLzj7LtdE1hGQOBdRq/7axsLvLH0+NjClStXPMPbUh8F3pZgTTt6AYF39EcAAIHKAgJvZbJeL2g08MadeEtDe/0UeNuzNfO4BQTecfff7gnUERB466j1d03jgbe/reS/ssCbf4/tsB8Bgbcfd6sSSFlA4E2rewJvQv0SeBNqllKTEhB4k2qXYgkMQkDgHUQbShch8Jam6n+gwNt/D1SQp4DAm2df7YpAmwICb5u6zc+9dOCNryM7depUePbZZ8PLL78cbty4MbfKw4cP3/fLbM1vJf8ZBd78e2yH/QgIvP24W5VAygICb1rdWzrwprXdtKsVeNPun+qHKyDwDrc3KiMwVAGBd6idmV+XwJtQvwTehJql1KQEBN6k2qVYAoMQEHgH0YbSRTQaeC9evBhu3rw5+Qjh+BU/aviVV14JBw8eDBcuXAhra2ulCzPwQQGB16kg0I6AwNuOq1kJ5Cwg8KbV3cYCb/Es7/Hjx8P6+vrkfbzxgyhi+P3c5z63+/+vrKykJTSgagXeATVDKVkJCLxZtdNmCHQiIPB2wtzYIo0G3rNnz4YTJ05M7uTGu73x6+TJk+H69evh0qVL4cyZM2F1dbWx4sc2kcA7to7bb1cCAm9X0tYhkI+AwJtWLxsLvHfu3Jk8wrCxsTEJvPHNDdN3e3d2djp7S0Nxt7l4Y8S5c+cmd533+lo0vthbfDwjfsV9xX12/SXwdi1uvbEICLxj6bR9EmhOQOBtzrKLmRoLvLHYeCd3c3Mz3L59OzzzzDOTu7tFmHziiScm/9z2VxFOY8CNoTTWFJ8fjnXNe4Z40fjZP2+7/v3mF3j71Ld2zgICb87dtTcC7QgIvO24tjVro4G3rSKrzDv7+MSiwLpofHwO+bXXXuskrC/ap8C7SMifE6gnIPDWc3MVgTELCLxpdb+xwBvv5E4/w9sXQ/xludnHJ6afJ56ta9H4eO3Vq1d3L5t948TdDjf6M//6xfCZ17/U4YqWIjAOgRh4P/kPns9ys299/OfD3S+/muXebIpAnwIx8D78gd/qrIQDna2U50JZBt7i7RDFGyEWBd69xn/wgx/cfS65eAY4jr1y5cru88h/+fp3OjkZjzx8IPzCn/6GwNuJtkXGJhAD70s//Ivhzbe6/E/Y9pXjz413/O4vCrztU1thhAIx8H7zH/5GZz83vv9dbxuhcnNbbizwxpJisHzyySf3/QWx5kqfP9OiO7ZV7vDOC7yzr19rez/T83ukoUtta41JwCMNY+q2vRJoRsAjDc04djVLo4E3Pg/78ssvh+effz709b7dRc/kzsIuGh9D/JEjR3bfytDnoxsCb1ffFtYZm4DAO7aO2y+B5QUE3uUNu5yhscA7+2qv2U0cPny4k9eSLXrrQlHnsWPHJiF20fh4x3h7e3v3k+LiIw3x38UP1Og61Au8XX5rWGtMAgLvmLptrwSaERB4m3HsapbGAm9XBZdZZ7/36s4G3jjfovfwxpAbfxEufnUV3OftU+At031jCFQXEHirm7mCwNgFBN60TkCWgTetFpSvVuAtb2UkgSoCAm8VLWMJEIgCAm9a56DRwDv9iWTF67sef/zxyZsOig+CSItnWNUKvMPqh2ryERB48+mlnRDoSkDg7Uq6mXUaDbzFL3g9/fTT4cUXXwzPPvvs5NPN4jOvs6/+aqb8cc0i8I6r33bbnYDA2521lQjkIiDwptXJxgLv9NsL4l3d6cA7+yaEtIiGU63AO5xeqCQvAYE3r37aDYEuBATeLpSbW6OTwOsObzMNE3ibcTQLgVkBgdeZIECgqoDAW1Ws3/GNBd64jeKVXS+88EJ46aWXJo80rK6uhlOnToXiNWD9bjft1QXetPun+uEKCLzD7Y3KCAxVQOAdamfm19Vo4I1LxLu5p0+fvm+1c+fO9frpa2m1ZO9qBd5cOmkfQxMQeIfWEfUQGL6AwDv8Hk1X2HjgTWv7aVUr8KbVL9WmIyDwptMrlRIYioDAO5ROlKtD4C3nNIhRAu8g2qCIDAUE3gybaksEWhYQeFsGbnh6gbdh0DanE3jb1DX3mAUE3jF3394J1BMQeOu59XVVY4G3+HjeJ554Ipw8ebKv/WS9rsCbdXttrkcBgbdHfEsTSFRA4E2rcY0F3rjt2V9YO3r0aNja2gorKytpqQy0WoF3oI1RVvICAm/yLbQBAp0LCLydky+1YKOBd7aS+JqynZ2dyb8+fPhw2N7enrymzFc9AYG3npurCCwSEHgXCflzAgRmBQTetM5Eq4E3ftTw1atXBd6GzoTA2xCkaQjMCAi8jgQBAlUFBN6qYv2ObzTwTt/RjdvySEOzzRV4m/U0G4FCQOB1FggQqCog8FYV63d8Y4HXL62130iBt31jK4xTQOAdZ9/tmsAyAgLvMnrdX9tY4O2+9PGtKPCOr+d23I2AwNuNs1UI5CQg8KbVzUYDb3GX98aNGw8o+KW15Q+GwLu8oRkIzBMQeJ0LAgSqCgi8VcX6Hd9o4I2/pBa/vIe3naYKvO24mpWAwOsMECBQVUDgrSrW7/jGAm+8u3v27Nlw4sSJsLa21u+uMl1d4M20sbbVu4DA23sLFEAgOQGBN62WCbwJ9UvgTahZSk1KQOBNql2KJTAIAYF3EG0oXURjgTeuGB9pOHLkSNjY2ChdgIHlBQTe8lZGEqgiIPBW0TKWAIEoIPCmdQ4aDbzXr18PL7/8cnj++ed9nHAL50DgbQHVlARCCAKvY0CAQFUBgbeqWL/jGwu8+72hIW7RWxqWb7TAu7yhGQjMExB4nQsCBKoKCLxVxfod31jg7Xcb41hd4B1Hn+2yewGBt3tzKxJIXUDgTauDAm9C/RJ4E2qWUpMSEHiTapdiCQxCQOAdRBtKF9F44L127Vo4ffr0fQWcO3curK+vly7KwPkCAq+TQaAdAYG3HVezEshZQOBNq7uNBt4Ydnd2dsL29nZYXV2dSMRfZNvc3AzPPfectzcseTYE3iUBXU5gDwGB19EgQKCqgMBbVazf8Y0F3jt37oTz589PQu3s3dwYhC9fvhy2tra8vWGJfgu8S+C5lMA+AgKv40GAQFUBgbeqWL/jGwu8+33SWrzLe+nSpXDmzJndO7/9bjvN1QXeNPum6uELCLzD75EKCQxNQOAdWkf2r6exwOsOb/uNF3jbN7bCOAUE3nH23a4JLCMg8C6j1/21jQXeWHp8bOHKlSue4W2pjwJvS7CmHb2AwDv6IwCAQGUBgbcyWa8XNBp44068paG9fgq87dmaedwCAu+4+2/3BOoICLx11Pq7pvHA299W8l9Z4M2/x3bYj4DA24+7VQmkLCDwptU9gTehfgm8CTVLqUkJCLxJtUuxBAYhIPAOog2li2gk8BaPMcz7gIn9/qx0lQZOBAReB4FAOwICbzuuZiWQs4DAm1Z3Gwm8Fy9enOz65MmTc3e/6M/TIuuvWoG3P3sr5y0g8ObdX7sj0IaAwNuGantzLh144/t3T506FY4fP77nxwfP+wS29raU78wCb769tbN+BQTefv2tTiBFAYE3ra41EnjPnj0bTpw4EdbW1ubu3gdPNHMoBN5mHM1CYFZA4HUmCBCoKiDwVhXrd/zSgXe/D5wotuajhZtpssDbjKNZCAi8zgABAssKCLzLCnZ7/dKBN5YbP3Ditdde2/cZ3iNHjoSNjY1ud5fZagJvZg21ncEIuMM7mFYohEAyAgJvMq2aFNpI4C3u8sYJt7a2wsrKymTy4t9/9atfve/T19omKp4rvnHjxmSpeW+PmK6h7Pi+3zgh8LZ9csw/VgGBd6ydt28C9QUE3vp2fVzZSOAtCo93end2du7bR/xlti7v7BYhe319fbJufH74woULYXNzc+4zxmXHT3+C3KIA3VYjBd62ZM07dgGBd+wnwP4JVBcQeKub9XlFo4G3z40Ua8/+gtxsoJ2tscz4Ykx87dr58+f3fSNFmwYCb5u65h6zgMA75u7bO4F6AgJvPbe+rsou8M57Bdp+7wFeNH76DvHq6urCV7C12UiBt01dc49ZQOAdc/ftnUA9AYG3nltfV2UZeOOjFdPPEi8KvHuNf//73x+mX7k2753Db751t5PeHQgh/OxnPxo+8/qXOlnPIgTGJBAD7+888eHQzXdzd7Lx58bd3/lguPvlV7tb1EoERiIQA++Bn/3Nzn5uPPJw/I72VVcgy8AbnyPe3t4O8Y5s/FoUePca/5M/+ZOTZ39v3779gG/xHO9ffeONuvaVrosH/ee/+OsCbyU1gwmUE4iB97f+3j8JXf0HbLmqlh8Vf26s/IsPCbzLU5qBwAMCMfDe+ccf6+znxt9656O6sIRAdoG3zDO5015Vxpf5VLklerHwUo80LCQygEAtAY801GJzEYFRC3ikIa32Zxd4F711oQitx44dm7zFYdH46XYKvGkdbtUSKCsg8JaVMo4AgUJA4E3rLGQXeCP/fu/VnQ28i8YLvGkdaNUSqCMg8NZRcw2BcQsIvGn1P8vAm1YLylfrkYbyVkYSqCIg8FbRMpYAgSgg8KZ1DgTehPol8CbULKUmJSDwJtUuxRIYhIDAO4g2lC5C4C1N1f9Agbf/HqggTwGBN8++2hWBNgUE3jZ1m59b4G3etLUZBd7WaE08cgGBd+QHwPYJ1BAQeGug9XiJwNsjftWlBd6qYsYTKCcg8JZzMooAgXsCAm9ap0HgTahfAm9CzVJqUgICb1LtUiyBQQgIvINoQ+kiBN7SVP0PFHj774EK8hQQePPsq10RaFNA4G1Tt/m5Bd7mTVubUeBtjdbEIxcQeEd+AGyfQA0BgbcGWo+XCLw94lddWuCtKmY8gXICAm85J6MIELgnIPCmdRoE3oT6JfAm1CylJiUg8CbVLsUSGISAwDuINpQuQuAtTdX/QIG3/x6oIE8BgTfPvtoVgTYFBN42dZufW+Bt3rS1GQXe1mhNPHIBgXfkB8D2CdQQEHhroPV4icDbI37VpQXeqmLGEygnIPCWczKKAIF7AgJvWqdB4E2oXwJvQs1SalICAm9S7VIsgUEICLyDaEPpIgTe0lT9DxR4+++BCvIUEHjz7KtdEWhTQOBtU7f5uQXe5k1bm1HgbY3WxCMXEHhHfgBsn0ANAYG3BlqPlwi8PeJXXVrgrSpmPIFyAgJvOSejCBC4JyDwpnUaBN6E+iXwJtQspSYlIPAm1S7FEhiEgMA7iDaULkLgLU3V/0CBt/8eqCBPAYE3z77aFYE2BQTeNnWbn1vgbd60tRkF3tZoTTxyAYF35AfA9gnUEBB4a6D1eInA2yN+1aUF3qpixhMoJyDwlnMyigCBewICb1qnQeBNqF8Cb0LNUmpSAgJvUu1SLIFBCAi8g2hD6SIE3tJU/Q8UePvvgQryFBB48+yrXRFoU0DgbVO3+bkF3uZNW5tR4G2N1sQjFxB4R34AbJ9ADQGBtwZaj5cIvD3iV11a4K0qZjyBcgICbzknowgQuCcg8KZ1GgTehPol8CbULKUmJSDwJtUuxRIYhIDAO4g2lC5C4C1N1f9Agbf/HqggTwGBN8++2hWBNgUE3jZ1m59b4G3etLUZBd7WaE08cgGBd+QHwPYJ1BAQeGug9XiJwNsjftWlBd6qYsYTKCcg8JZzMooAgXsCAm9ap0HgTahfAm9CzVJqUgICb1LtUiyBQQgIvINoQ+kiBN7SVP0PFHj774EK8hQQePPsq10RaFNA4G1Tt/m5Bd7mTVubUeBtjdbEIxcQeEd+AGyfQA0BgbcGWo+XCLw94lddWuCtKmY8gXICAm85J6MIELgnIPCmdRoE3oT6JfAm1CylJiUg8CbVLsUSGISAwDuINpQuQuAtTdX/QIG3/x6oIE8BgTfPvtoVgTYFBN42dZufW+Bt3rS1GQXe1mhNPHIBgXfkB8D2CdQQEHhroPV4icDbI37VpQXeqmLGEygnIPCWczKKAIF7AgJvWqdB4E2oXwJvQs1SalICAm9S7VIsgUEICLyDaEPpIgTe0lT9DxR4+++BCvIUEHjz7KtdEWhTQOBtU7f5uQXe5k1bm1HgbY3WxCMXEHhHfgBsn0ANAYG3BlqPlwi8PeJXXVrgrSpmPIFyAgJvOSejCBC4JyDwpnUaBN6E+iXwJtQspSYlIPAm1S7FEhiEgMA7iDaULiLLwHvr1q1w6tSpcOPGjQnEuXPnwvr6+p4o+42/du1aOH369O61R48eDVtbW2FlZaU0clMDBd6mJM1D4H4BgdeJIECgqoDAW1Ws3/HZBd47d+6E8+fPTwLuxsZGuH79erhw4ULY3NwMa2trD2gvGn/58uVw+PDhyXzF2EOHDoWTJ0923jmBt3NyC45EQOAdSaNtk0CDAgJvg5gdTJVd4I0B99KlS+HMmTNhdXV1N6QWAXjWtOr4GIDjXd8+7vIKvB18R1hilAIC7yjbbtMElhIQeJfi6/zi7AJvDKM7Ozthe3t7Enjj18WLFyf/d95d2arj95ur7e4JvG0Lm3+sAgLvWDtv3wTqCwi89e36uDLLwBvvwk7fgV0UeMuOnxeO77zxVid9e+hACB/4txfDZ17/UifrWYTAmARi4P343z8Zvns3r13HnxsPffIXwt0vv5rXxuyGwAAEYuD97s+81NnPjZVHHx7ArtMtIcvA28Yd3hh2413j+Dzw9LPAX//WX3fS/YcfOhD+1y9cEng70bbI2ARi4P0/3nsivJVZ4o0/Nx59+bjAO7YDbb+dCMTA+8azO5393Pi+7/2eTvaV6yLZBd6qz+SWGb9X2O36UHikoWtx641FwCMNY+m0fRJoTsAjDc1ZdjFTdoF30VsXileQHTt2bPIWh0Xj5z3G0EVj5q0h8PYlb93cBQTe3DtsfwSaFxB4mzdtc8bsAm/E2u+9urOBd9H4+Pzv1atX7+vBwYMHH3i0oc0mFXMLvF0oW2OMAgLvGLtuzwSWExB4l/Pr+uosA2/XiF2tJ/B2JW2dsQkIvGPruP0SWF5A4F3esMsZBN4utZdcS+BdEtDlBPYQEHgdDQIEqgoIvFXF+h0v8PbrX2l1gbcSl8EESgsIvKWpDCRA4D8JCLxpHQWBN6F+CbwJNUupSQkIvEm1S7EEBiEg8A6iDaWLEHhLU/U/UODtvwcqyFNA4M2zr3ZFoE0BgbdN3ebnFnibN21tRoG3NVoTj1xA4B35AbB9AjUEBN4aaD1eIvD2iF91aYG3qpjxBMoJCLzlnIwiQOCegMCb1mkQeBPql8CbULOUmpSAwJtUuxRLYBACAu8g2lC6CIG3NFX/AwXe/nuggjwFBN48+2pXBNoUEHjb1G1+boG3edPWZhR4W6M18cgFBN6RHwDbJ1BDQOCtgdbjJQJvj/hVlxZ4q4oZT6CcgMBbzskoAgTuCQi8aZ0GgTehfgm8CTVLqUkJCLxJtUuxBAYhIPAOog2lixB4S1P1P1Dg7b8HKshTQODNs692RaBNAYG3Td3m5xZ4mzdtbUaBtzVaE49cQOAd+QGwfQI1BATeGmg9XiLw9ohfdWmBt6qY8QTKCQi85ZyMIkDgnoDAm9ZpEHgT6pfAm1CzlJqUgMCbVLsUS2AQAgLvINpQugiBtzRV/wMF3v57oII8BQTePPtqVwTaFBB429Rtfm6Bt3nT1mYUeFujNfHIBQTekR8A2ydQQ0DgrYHW4yUCb4/4VZcWeKuKGU+gnIDAW87JKAIE7gkIvGmdBoE3oX4JvAk1S6lJCQi8SbVLsQQGISDwDqINpYsQeEtT9T9Q4O2/ByrIU0DgzbOvdkWgTQGBt03d5ucWeJs3bW1Ggbc1WhOPXEDgHfkBsH0CNQQE3hpoPV4i8PaIX3VpgbeqmPEEygkIvOWcjCJA4J6AwJvWaRB4E+qXwJtQs5SalIDAm1S7FEtgEAIC7yDaULoIgbc0Vf8DBd7+e6CCPAUE3jz7alcE2hQQeNvUbX5ugbd509ZmFHhbozXxyAUE3pEfANsnUENA4K2B1uMlAm+P+FWXFnirihlPoJyAwFvOySgCBO4JCLxpnQaBN6F+CbwJNUupSQkIvEm1S7EEBiEg8A6iDaWLEHhLU/U/UODtvwcqyFNA4M2zr3ZFoE0BgbdN3ebnFnibN21tRoG3NVoTj1xA4B35AbB9AjUEBN4aaD1eIvD2iF91aYG3qpjxBMoJCLzlnIwiQOCegMCb1mkQeBPql8CbULOUmpSAwJtUuxRLYBACAu8g2lC6CIG3NFX/AwXe/nuggjwFBN48+2pXBNoUEHjb1G1+boG3edPWZhR4W6M18cgFBN6RHwDbJ1BDQOCtgdbjJQJvj/hVlxZ4q4oZT6CcgMBbzskoAgTuCQi8aZ0GgTehfgm8CTVLqUkJCLxJtUuxBAYhIPAOog2lixB4S1P1P1Dg7b8HKshTQODNs692RaBNAYG3Td3m5xZ4mzdtbUaBtzVaE49cQOAd+QGwfQI1BATeGmg9XiLw9ohfdWmBt6qY8QTKCQi85ZyMIkDgnoDAm9ZpEHgT6pfAm1CzlJqUgMCbVLsUS2AQAgLvINpQugiBtzRV/wMF3v57oII8BQTePPtqVwTaFBB429Rtfm6Bt3nT1mYUeFujNfHIBQTekR8A2ydQQ0DgrYHW4yUCbwn8y5cvh52dncnIo0ePhq2trbCyslLiymaHCLzNepqNQCEg8DoLBAhUFRB4q4r1O17gXeB/7dq1Sdjd3t4Oq6ur4eLFi5MrTp482XnnBN7OyS04EgGBdySNtk0CDQoIvA1idjCVwLsAOQbcI0eOhI2NjcnI2QDcQY92lxB4u9S21pgEBN4xddteCTQjIPA249jVLALvPtJ37twJ58+fD+vr67uB9/r16+HChQthc3MzrK2tddWnyToCb6fcFhuRgMA7ombbKoGGBATehiA7mkbgLRF4493dGHrj12zg/Q+vf6eTVn3PwwfCR7/6L8Pnv/nnnaxnEQJjEvjRd7w7fPix/yn89Vt3s9p2/Lnxzv/no+Huv/93We3LZggMQeDAf/ZfhG/8dx/u7OfGD7zrbUPYdrI1CLwlAu9+d3i7CrzJnjCFEyBAgAABAksLCLzLEQq8C/yG9Azvcq12NQECBAgQIEBgnAIC74K+D+ktDeM8onZNgAABAgQIEFhOQOAt4TeU9/CWKNWQDAX6fBVehpy2RIAAAQIjFBB4R9h0W+5OoImw2sQc3e3YSgTyELh161Y4depUOH78+O4vLTe9sz6/t/tcu2lH8xEoIyDwllEyhkBNgSb+Umlijprlu4zAaAUE3tG23sYzFRB4M22sbdUTmH585fDhw7ufsLfXYy3Fa+p+4id+Inz84x+fLPrMM89MPokvPv99+vTp3ULix1J/6EMfCr/0S78U4vjf//3fn/xZfK9zfKdzDLZXr169b474DwJvvV66isAyAtPfj3GeeKf3fe973+T7tfh+L35GxD+Pd4Nv3LgxWfLcuXO7d4Wnf3ZM/9l+Px+eeOKJ3Z8F8efGT//0T4df/dVfDbdv337g4+2n55n+mVX1Z9PW1lZYWVlZhsy1BAYtIPAOuj2K61Ig/sV05cqV3ZD7+c9/fvIXQLzTM/vx0jdv3gzxL4ivfOUrkw8heeqppyYhd/au0GxYLf78sccem1xf/AUT145/ccV/F7/iB54cOnRoMqfA2+UpsBaBvxGYd4c3hsjp7/fpcceOHZt8QFEcc+nSpXDmzJnw9re/Pfze7/1e+Kmf+qnJ9/rsz5i9fj7EwDv986T4efHtb3/7vscs4s+M+LH3xX80x/lfe+21ybWztS762aTvBHIXEHhz77D9lRKY96l6xYWzfylNf/hIHDP9yXvFPMWHlez1F9r0c4Hz0QaSwQAAB4pJREFU1p5+O8hv//ZvT0qJf4n5IkCgG4G9Au/sJ23Ovsln9mfAdLWzH1y06OfD7M+G2X+e97OpCNux/io/m7pRtQqB/gQE3v7srTwggf3+kpp9F3P8i+Ts2bPhxIkTkx1U+Utl3l+i89aevksk8A7ooChlNAJVAu/0o0sFUPFYQ3GnNT6OEL8OHjx432NM0/8xO7tmmcBbPAZVrFs81iDwjuao2mhJAYG3JJRheQu4w5t3f+2OQFWBKoE3Pkow7xnYIuzG53vjJ3a2cYf3yJEjk0cpZr9m11r0vz5V9TGeQGoCAm9qHVNvawKzz9d9+tOfDu95z3smz/JNPycX7/hOP8O73x3e6Wdzi+eB573qaHrt+NyfZ3hba7OJCZQSmPcfwbMhMk5UBOPiGd747+JjDvFrdXX1vv8FaN4zt8Wz+/N+Piy6wzs7X1zzU5/6VHj66acndVX52VQKxSACCQsIvAk3T+nNC0z/ZnaVtzTEX2SJb1qYvYtS/GUYf3t7+i0N897t6S0NzffTjASWEZh+A8L0WxqK7/di7unv8/jvpn92zP5MiX9eXL/o58OiwFuE6+lHKoq3xCy6wzu7trc0LHNSXJuCgMCbQpfUSIAAAQIECBAgUFtA4K1N50ICBAgQIECAAIEUBATeFLqkRgIECBAgQIAAgdoCAm9tOhcSIECAAAECBAikICDwptAlNRIgQIAAAQIECNQWEHhr07mQAAECBAgQIEAgBQGBN4UuqZEAAQIECBAgQKC2gMBbm86FBAgQIECAAAECKQgIvCl0SY0ECBAgQIAAAQK1BQTe2nQuJECAAAECBAgQSEFA4E2hS2okQIAAAQIECBCoLSDw1qZzIQECBAgQIECAQAoCAm8KXVIjAQIECBAgQIBAbQGBtzadCwkQIECAAAECBFIQEHhT6JIaCRAgQIAAAQIEagsIvLXpXEiAAAECBAgQIJCCgMCbQpfUSIAAAQIECBAgUFtA4K1N50ICBAgQIECAAIEUBATeFLqkRgIEehe4detWOHXqVLhx48ZuLQcPHgwXLlwIa2trvdenAAIECBDYW0DgdToIECCwQODy5cthZ2cnHD9+PGxsbOyOLv79uXPnwvr6OkcCBAgQGKiAwDvQxiiLAIFhCFy/fj1sbm5O7u7OC7Xxz+/cuRN+9Ed/NFy7dm03GG9vb4fbt2/vhuRinvjv4tczzzwTTp48Ofn/i7vHMVAXa8Q5z58/P/nnGLKLMT/3cz8X/uAP/iC88sork2tnQ/gw1FRBgACBYQkIvMPqh2oIEBiYQLyLe+XKlRAD7Orq6r7VxcB7+vTpcPTo0bC1tRVWVlYm42dDcxFmDx06NAm9VQJvDMzFYxSLwvjAKJVDgACB3gQE3t7oLUyAQAoCFy9eDDdv3rwvwO5Vd3GHdzYcx9Ac/2w6BE+PjfPFO8hl7vBOj4nXxfriV3G3OAVTNRIgQKBrAYG3a3HrESCQlMC8wBv/3dWrV3f3UTzDu1fgjeOPHDly3/O/8a7u2bNnw4kTJyZ3jusG3nlhOilgxRIgQKADAYG3A2RLECCQrsB+jzTMPoog8KbbZ5UTIJC3gMCbd3/tjgCBJQWK52Sfe+65++7QxmnLBl6PNCzZBJcTIEBgSQGBd0lAlxMgkL/AXq8lKxt4F/3S2uwvsUXR2TXn/WJbvKMcnxf2LuD8z6AdEiCwnIDAu5yfqwkQGInAvA+eiFuffgfvXo80xHH7vZZs+m5x8cEWH/jAB8IXvvCFB15L5oMvRnLgbJMAgUYFBN5GOU1GgACBdgTm3eFtZyWzEiBAID8BgTe/ntoRAQIZCgi8GTbVlggQ6ExA4O2M2kIECBCoLyDw1rdzJQECBAReZ4AAAQIECBAgQCBrAYE36/baHAECBAgQIECAgMDrDBAgQIAAAQIECGQtIPBm3V6bI0CAAAECBAgQEHidAQIECBAgQIAAgawFBN6s22tzBAgQIECAAAECAq8zQIAAAQIECBAgkLWAwJt1e22OAAECBAgQIEBA4HUGCBAgQIAAAQIEshYQeLNur80RIECAAAECBAgIvM4AAQIECBAgQIBA1gICb9bttTkCBAgQIECAAAGB1xkgQIAAAQIECBDIWkDgzbq9NkeAAAECBAgQICDwOgMECBAgQIAAAQJZCwi8WbfX5ggQIECAAAECBAReZ4AAAQIECBAgQCBrAYE36/baHAECBAgQIECAgMDrDBAgQIAAAQIECGQtIPBm3V6bI0CAAAECBAgQEHidAQIECBAgQIAAgawFBN6s22tzBAgQIECAAAECAq8zQIAAAQIECBAgkLWAwJt1e22OAAECBAgQIEBA4HUGCBAgQIAAAQIEshYQeLNur80RIECAAAECBAgIvM4AAQIECBAgQIBA1gICb9bttTkCBAgQIECAAAGB1xkgQIAAAQIECBDIWkDgzbq9NkeAAAECBAgQICDwOgMECBAgQIAAAQJZCwi8WbfX5ggQIECAAAECBAReZ4AAAQIECBAgQCBrAYE36/baHAECBAgQIECAgMDrDBAgQIAAAQIECGQtIPBm3V6bI0CAAAECBAgQ+P8BmWIrqh1ik2sAAAAASUVORK5CYII="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sup_defs.bar_plot(\n",
    "    dataf=None, \n",
    "    x=conv_rates['conversion_rate'].index, \n",
    "    y=conv_rates['conversion_rate'].values,\n",
    "    title='Conversion rate',\n",
    "    x_label='Group',\n",
    "    y_label='Conversion rate').show('png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The sample size is large, so z-test will be used. The calculated p-value is much greater than 0.05, which means that the null hypothesis cannot be rejected. Interestingly, our theoretical estimate of 14% conversion on a new page is not even within the confidence interval."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 254,
     "status": "ok",
     "timestamp": 1644588471923,
     "user": {
      "displayName": "Ви Колаб",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhlJDbx-1O-h8TctKuZMXeldr_Lr4lelXZQ_JrK=s64",
      "userId": "02862000375646745385"
     },
     "user_tz": -120
    },
    "id": "U1cky1ib-PVg",
    "outputId": "18a10e10-b98e-4393-f272-67c12c61440a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "z statistic: 0.395\n",
      "p-value: 0.693\n",
      "ci 95% Control group: [0.110, 0.129]\n",
      "ci 95% Reatment group: [0.107, 0.126]\n"
     ]
    }
   ],
   "source": [
    "g_counts = [ab_required[ab_required['group'] == 'control']['converted'].sum(), \n",
    "            ab_required[ab_required['group'] == 'treatment']['converted'].sum()]\n",
    "nobs = [required_n, required_n]\n",
    "\n",
    "z_stat, pval = proportions_ztest(\n",
    "    g_counts, \n",
    "    nobs=nobs\n",
    "    )\n",
    "ci_low, ci_upp = proportion_confint(\n",
    "    g_counts, \n",
    "    nobs=nobs, \n",
    "    alpha=alpha\n",
    "    )\n",
    "print(f'z statistic: {z_stat:.3f}')\n",
    "print(f'p-value: {pval:.3f}')\n",
    "print(f'ci 95% Control group: [{ci_low[0]:.3f}, {ci_upp[0]:.3f}]')\n",
    "print(f'ci 95% Reatment group: [{ci_low[1]:.3f}, {ci_upp[1]:.3f}]')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyP40GsF4ZlaEPPY3KNu10HE",
   "collapsed_sections": [],
   "name": "A_B_Testing.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}