mammographic_masses.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import libraries and data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "sns.set_style('whitegrid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('mammographic_masses.data', header=None, na_values = '?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BI-RADS assessment</th>\n",
       "      <th>Age</th>\n",
       "      <th>Shape</th>\n",
       "      <th>Margin</th>\n",
       "      <th>Density</th>\n",
       "      <th>Severity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   BI-RADS assessment   Age  Shape  Margin  Density  Severity\n",
       "0                 5.0  67.0    3.0     5.0      3.0         1\n",
       "1                 4.0  43.0    1.0     1.0      NaN         1\n",
       "2                 5.0  58.0    4.0     5.0      3.0         1\n",
       "3                 4.0  28.0    1.0     1.0      3.0         0\n",
       "4                 5.0  74.0    1.0     5.0      NaN         1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns = ['BI-RADS assessment','Age','Shape','Margin','Density','Severity']\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BI-RADS assessment</th>\n",
       "      <th>Age</th>\n",
       "      <th>Shape</th>\n",
       "      <th>Margin</th>\n",
       "      <th>Density</th>\n",
       "      <th>Severity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>959.000000</td>\n",
       "      <td>956.000000</td>\n",
       "      <td>930.000000</td>\n",
       "      <td>913.000000</td>\n",
       "      <td>885.000000</td>\n",
       "      <td>961.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4.348279</td>\n",
       "      <td>55.487448</td>\n",
       "      <td>2.721505</td>\n",
       "      <td>2.796276</td>\n",
       "      <td>2.910734</td>\n",
       "      <td>0.463059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.783031</td>\n",
       "      <td>14.480131</td>\n",
       "      <td>1.242792</td>\n",
       "      <td>1.566546</td>\n",
       "      <td>0.380444</td>\n",
       "      <td>0.498893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>45.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>57.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>55.000000</td>\n",
       "      <td>96.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       BI-RADS assessment         Age       Shape      Margin     Density  \\\n",
       "count          959.000000  956.000000  930.000000  913.000000  885.000000   \n",
       "mean             4.348279   55.487448    2.721505    2.796276    2.910734   \n",
       "std              1.783031   14.480131    1.242792    1.566546    0.380444   \n",
       "min              0.000000   18.000000    1.000000    1.000000    1.000000   \n",
       "25%              4.000000   45.000000    2.000000    1.000000    3.000000   \n",
       "50%              4.000000   57.000000    3.000000    3.000000    3.000000   \n",
       "75%              5.000000   66.000000    4.000000    4.000000    3.000000   \n",
       "max             55.000000   96.000000    4.000000    5.000000    4.000000   \n",
       "\n",
       "         Severity  \n",
       "count  961.000000  \n",
       "mean     0.463059  \n",
       "std      0.498893  \n",
       "min      0.000000  \n",
       "25%      0.000000  \n",
       "50%      0.000000  \n",
       "75%      1.000000  \n",
       "max      1.000000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 961 entries, 0 to 960\n",
      "Data columns (total 6 columns):\n",
      "BI-RADS assessment    959 non-null float64\n",
      "Age                   956 non-null float64\n",
      "Shape                 930 non-null float64\n",
      "Margin                913 non-null float64\n",
      "Density               885 non-null float64\n",
      "Severity              961 non-null int64\n",
      "dtypes: float64(5), int64(1)\n",
      "memory usage: 45.2 KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BI-RADS assessment     2\n",
       "Age                    5\n",
       "Shape                 31\n",
       "Margin                48\n",
       "Density               76\n",
       "Severity               0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    516\n",
       "1    445\n",
       "Name: Severity, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.Severity.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that there are missing values and that the classes are more or less balanced.\n",
    "\n",
    "Do they remain balanced if we drop the NaN?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(830, 6)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_no_null = data.dropna(axis=0)\n",
    "data_no_null.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    427\n",
       "1    403\n",
       "Name: Severity, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_no_null.Severity.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Even when we discard all samples that have at least one NaN value, the classes remain balanced. Let's proceed with that and see how our algorithms will eventually perform."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EDA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's identify the correlation between features and plot some stuff!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbIAAAFTCAYAAABca4k7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0IklEQVR4nO3deXyNZ97H8c852QkhosogJLaqMao6mFq6YGJtLVksMZbWmNZT2xhpOzTUEtWi1aLaGkUQYqnE0inTlpqnrVK1PCoVWytMUEuTkEjO/fyRV89USXJORe5z1/fd13m9cpbc53uU/PK7ruu+bpthGAYiIiIWZTc7gIiIyK1QIRMREUtTIRMREUtTIRMREUtTIRMREUtTIRMREUtTIRMREVN89dVXxMbG3vD4v/71L3r37k10dDSrVq0q8TjetyOciIhIcd566y02bNhAQEDAdY9fu3aN6dOnk5ycTEBAAH379uXhhx+matWqRR5LHZmIiJS52rVrM3fu3BseT09Pp3bt2gQFBeHr68v999/PF198Ueyx1JFZ2LVzR82O4JY69bubHcFtdcpVMzuC23Id18yO4Ja956319xig8933mR3BbSknU2/5GO78zFm7bRdJSUnO+9HR0URHRzvv//GPf+S777674fuysrKoUKGC83758uXJysoq9r1UyER+RaxWxOTX6+eFy1WBgYFkZ2c772dnZ19X2G5GQ4siIuIaR4Hrt18oPDycEydOcPHiRfLy8vjiiy+4777iO2B1ZCIi4hrDcdsOnZKSQk5ODtHR0cTFxTF06FAMw6B3795Uq1b8EL9Nu99bl+bIbj+rzZFZcWhRc2RlozTmyPIyDrr8Wt8a997y+7lKHZmIiLjGcfs6sluhQiYiIq65jUOLt0KFTEREXHMLizhuJxUyERFxjToyERGxMqMg3+wIN6VCJiIirtFiDxERsTQNLYqIiKVpsYeIiFiaOjIREbE0K86RffbZZ4waNYp69ephGAb5+flMnTqV8PBw5s6dS0hICH379r3ue+Li4jh48CCVKlXCMAwuXrzI4MGD6d27t/M1w4cPB2DBggXOxx555BGqV6+O3W4nNzeXe++9l7i4OPz8/Dhx4gRTp06loKCA/Px8mjRpwtixY7Hbf517Hufm5rJhwwYiIyPNjiIi8l9WXbXYqlUrZs+eDcAnn3zCSy+9xJtvvlns94wbN4527doBcPHiRbp160avXr2w2WycPn2anJwcrl27xrfffkutWrWc37do0SL8/PwAmD9/PrNnzyYuLo5Zs2YxYMAA2rVrh2EYjBgxgm3bttGxY8df/ME92dmzZ1m9erUKmYh4FMP4FcyRXb58md/85jduvcG5c+fw9fXFZrMBkJyczKOPPoq/vz/Lly9n/PjxN/2+wYMH06VLF+Li4qhRowbr1q2jfPnyNG3alDlz5uDtfX30M2fOEB8fT25uLhcvXuTpp5+mQ4cOzJ49m08//RSHw0HXrl0ZNGgQiYmJrF+/HrvdTvPmzRk/fjynT59mwoQJ5Obm4ufnx4svvkhwcDAjR44kKyuLq1evMm7cOFq2bElcXBwnT54kNzeXoUOH0qVLF7p3706LFi1IS0ujbt26VKlShS+++AJfX18WLlzI1atXef7557lw4QIAf//732nYsCGdOnWiefPmHDt2jCpVqjB37lwWLFjAkSNHeP311xkxYoRbf94iIreNVefIPv30U2JjY8nLy+Pw4cMldmMAM2fOZMGCBWRkZBAeHs6rr74KgMPhIDU1laSkJLy9venatSsjR47E39//hmP4+/uTm5sLwOjRo1m+fDmzZs0iLS2N9u3bM3HiRCpWrOh8/dGjRxk8eDAtW7Zkz549zJ07lw4dOrB+/XqWLVtGtWrVWLt2LQBr165lwoQJNGvWjOXLl5Ofn8+MGTOIjY2lffv2/O///i8vv/wyw4cP59y5cyxevJjz589z/PhxsrKy+Oyzz1izZg0AO3fuBAov/tatWzfuv/9+IiIiePbZZxk9ejQDBgzgyJEjpKam0qpVK/r168fx48d59tlnWbFiBd9++y3vvvsu1atXJyYmhv379zN8+HDS0tJUxETEs1hxjgyuH1o8evQoMTExbN++3fn8li1bSExMBHB2Vz8OLX788ce8/PLL1K5dG4AdO3aQnZ3N2LFjgcLClpKSctMhtKysLMqXLw8UFtNBgwYxaNAgsrOzmTFjBvPmzSMuLs75+qpVqzJ//nySk5Ox2Wzk5xeO5c6aNYtZs2Zx7tw52rZtC8D06dNZtGgRL7/8Ms2aNcMwDNLS0njzzTd5++23MQwDHx8f6tevT//+/RkzZgz5+fnExsYSGBjIhAkTmDBhAllZWfTo0cOZ4d57Cy9bULFiRcLDw51f5+bmkpaWxqeffsrmzZuBwu4WoHLlylSvXh2A6tWrO4u3iIjHsWpH9lMhISE3PBYREUFERMRNX9++fXu+/PJLJkyYwGuvvUZycjJTpkzhoYceAmD37t1MmTLlpoXsrbfeonPnzkBhh+fl5cWDDz5I+fLlqVu3rnOI7kevvvoqkZGRtG/fnjVr1rBu3Try8vLYsmULs2bNwjAMunbtSteuXVm1ahWTJk3Cz8+PoUOH8uWXXxIWFsaQIUNo3rw56enp7Nq1i8OHD5Odnc3ChQvJzMwkJiaGe++9l4MHD/LGG2+Qm5tL+/bteeyxxwCcw6c3ExYWRo8ePejevTvnz59n9erVRX6P3W7H4aG/+YjIHcyq55H9OLRot9vJzs4mLi7upkOBRXnqqafo1asX69at46uvvnJ2dwD3338/ubm57NmzB4AhQ4Y4f4jfc889/O1vfwNgzpw5TJkyhVdeeQVfX19q1qxJfHz8de8TERHB1KlTefPNN6levToXLlzA19eXoKAgHnvsMYKCgnjwwQepUaMGDRs2pE+fPlSuXJlq1arxu9/9jvHjxzvn2H6cz6pTpw5vvPEG69evx8fHh2eeeYaqVaty9uxZHn/8ccqVK8eQIUNumK+7meHDh/P888+zatUqsrKyih02rFKlCteuXWPmzJmMGzfO5T9rEZHbykNXLeoK0RamK0TffrpC9O2nK0SXjdK4QvTV/13h8mv9W/ct+UWlRCdEi4iIazx0ykOFTEREXKNCJiIiVvarOCFaRETuYB662EOFTEREXKOhRRERsbRfwwnRIiJyB1NHJiIilqaOTERELE0dmYiIWJpWLYqIiKWpIxMREUvTHJmIiFiaOjIpbVbbTf74NylmR3DbvOYTzY7gHi+o4pnTGEX6s/e3Zkdw27C8CmZHMIc6MhG53axWxMRi1JGJiIilFWjTYBERsTJ1ZCIiYmkqZCIiYmla7CEiIpZWSh2Zw+EgPj6ew4cP4+vry5QpUwgNDXU+v2HDBv7xj39gt9vp3bs3/fr1K/Z4KmQiIuKaUlrssXXrVvLy8khKSmLv3r0kJCQwf/585/MvvfQSqamplCtXjq5du9K1a1eCgoKKPJ4KmYiIuKaUOrLdu3fTtm1bAJo1a8aBAweue75hw4b88MMPeHt7YxgGNput2OOpkImIiGvcmCNLSkoiKSnJeT86Opro6GgAsrKyCAwMdD7n5eVFfn4+3t6FJal+/fr07t2bgIAAOnbsSMWKFYt9LxUyERFxieEwXH7tTwvXzwUGBpKdne2873A4nEXs66+/5qOPPmLbtm2UK1eOcePGsXnzZjp37lzke9ldTiUiInc2h8P1WzGaN2/O9u3bAdi7dy8NGjRwPlehQgX8/f3x8/PDy8uL4OBgLl++XOzx1JGJiIhrSmn5fceOHdm5cycxMTEYhsG0adNISUkhJyfH2cn169cPHx8fateuTc+ePYs9ngqZiIi4Jr90Vi3a7XYmT5583WPh4eHOr/v27Uvfvn1dPp4KmYiIuEY7e4iIiKUZri/2KEta7FEGFi5cSJs2bcjNzTU7iojIL1dKiz1KmwpZGUhJSaFLly5s3LjR7CgiIr+cw3D9VoY0tHibffbZZ9SuXZuYmBjGjRtHr1692LdvH5MmTaJ8+fJUqVIFPz8/EhISWLp0KampqdhsNrp06cLAgQPNji8i8l8eej0ydWS32erVq4mMjCQsLAxfX1+++uorXnjhBRISEliyZAm1a9cG4MiRI2zatInly5ezfPlytm7dytGjR01OLyLyX4bD4fKtLKkju40uXbrE9u3b+f7771m6dClZWVksW7aMzMxM6tevD8D999/Ppk2bSEtLIyMjg0GDBjm/9+TJk4SFhZn4CUREfqKMhwxdpUJ2G23YsIHevXszfvx4AK5cucKjjz6Kv78/R44coV69enz11VcAhIWFUa9ePd5++21sNhuLFy++7mx3ERHT6Xpkd57Vq1fz0ksvOe8HBATQqVMnQkJCeO655yhXrhw+Pj5Uq1aNRo0a0bp1a/r27UteXh5NmzalWrVqJqYXEfkZdWR3ng0bNtzwWHx8PImJiSxYsIDg4GBmz56Nj48PAE888QRPPPFEWccUEXGNToiWH1WpUoUhQ4ZQrlw5KlSoQEJCgtmRRERK5qGrFlXITBAREUFERITZMURE3KOhRRERsbKyXlbvKhUyERFxjToyERGxNBUyERGxNJ1HJiIiVmbkq5CJiIiVaWhRREQsTasWRUTE0tSRiYiIpamQiYiIlRkFGlqUUlannLV2x5/XfKLZEdz21J7JZkdwW95rz5kdwS2j5viaHcFtjUO+NzuCOdSRicjtZrUiJtZiqJCJiIilqZCJiIileeYUmQqZiIi4RkOLIiJibfkqZCIiYmHqyERExNo0RyYiIlamjkxERKxNHZmIiFiZh15XU4VMRERcY+SbneDmVMhERMQ16shERMTKNLQoIiKWpkImIiKWVlqFzOFwEB8fz+HDh/H19WXKlCmEhoY6n9+3bx8JCQkYhkHVqlWZOXMmfn5+RR7PXjqx7iwLFy5k0KBBDBkyhKFDh3LgwAFiY2NJT083O5qIyG1jFNhcvhVn69at5OXlkZSUxNixY0lISPjvexgGEyZMYPr06axYsYK2bdty6tSpYo+njsxNR44c4V//+hcrVqzAZrNx6NAhxo8fT1BQkNnRRERuK8NRfIFy1e7du2nbti0AzZo148CBA87njh07RqVKlXj33XdJS0ujffv2hIWFFXs8FTI3BQcHk5GRQXJyMu3ateOee+4hOTmZoUOH8sYbb3Du3DmuXLnCrFmzqFGjBhMnTuTMmTNcuHCBdu3aMWrUKOLi4jAMg9OnT5OTk8OMGTMIDw9n6dKlpKamYrPZ6NKlCwMHDjT744qIOLkztJiUlERSUpLzfnR0NNHR0QBkZWURGBjofM7Ly4v8/Hy8vb25cOECX375JRMmTCA0NJThw4fTpEkTWrduXeR7qZC5KTg4mPnz57Ns2TLeeOMN/P39GT16NADt27fnscceY+7cuWzZsoXOnTvTrFkzIiMjyc3NdRYygFq1ajFjxgw+/vhjZs6cyV//+lc2bdrE8uXLsdlsDBo0iDZt2pT4m4iISFkxDNc7sp8Wrp8LDAwkOzvbed/hcODtXViOKlWqRGhoKPXq1QOgbdu2HDhwQIWsNJ04cYLAwECmT58OwP79+xk2bBghISE0adIEgJCQEM6dO0elSpXYv38/n376KYGBgeTl5TmP06pVKwDuu+8+pk2bRlpaGhkZGQwaNAiAS5cucfLkSRUyEfEYpbXYo3nz5nz44Yd06dKFvXv30qBBA+dztWrVIjs7mxMnThAaGsoXX3xBnz59ij2eCpmbDh8+zIoVK1iwYAF+fn7UrVuXChUq4OXldcNr165dS4UKFZg8eTInTpxg1apVGEbhppsHDx6kRYsW7Nmzh/r16xMWFka9evV4++23sdlsLF68+Lr/uSIiZiutObKOHTuyc+dOYmJiMAyDadOmkZKSQk5ODtHR0UydOpWxY8diGAb33XcfDz30ULHHUyFzU6dOnUhPTycyMpJy5cphGAZ/+9vfePfdd294bevWrRkzZgy7d+8mICCA0NBQMjMzAdi+fTvbtm3D4XAwffp0atWqRevWrenbty95eXk0bdqUatWqlfXHExEpkqOE1YiustvtTJ48+brHwsPDnV+3bt2a5ORkl49nM35sEaTMxMXF0aVLF9q1a3dLx3nwN4+UUqKyEeVVw+wIbntqz+SSX+RB8l57zuwIbguds8fsCG77rHZdsyO4LfzA+7d8jOPNOrr82jp7P7jl93OVOjIREXGJp7Y9KmQm+OnJfyIiVlFac2SlTYVMRERc4s7y+7KkQiYiIi7RpsEiImJpBQ7P3J5XhUxERFyiOTIREbE0rVoUERFLU0cmIiKW5tCqRRERsTKHOjIREbEydWQiImJpOiFaREQsTasWpdTlOq6ZHcEtVTz0H0FxrLibvO8z08yO4Jbg+VFmR3Cb3ctDt7i4zTS0KCK3ndWKmFiLhhZFRMTSClTIRETEyjS0KCIilqahRRERsTRPXeKiQiYiIi4xUEcmIiIWlq+hRRERsTJ1ZCIiYmmaIxMREUtTRyYiIpamjkxERCxNhUxERCytwKahRRERsTCHh86R2c0OYBWfffYZDRs2ZNOmTdc93r17d+Li4n7xcbdv305SUtKtxhMRue0MN25lSR2ZG8LCwkhNTaVLly4AHD58mCtXrtzSMdu1a1ca0UREbjvNkf0KNGrUiOPHj3P58mUqVqzIhg0b6N69O6dPn2bZsmX885//JD8/nwoVKjB37lxSU1NZs2YNDoeDZ555hu+++47ExESCgoLw8fFxFsSjR48SExPD2LFjufvuu/n222/57W9/y6RJk0z+xCIi/+Xw0DkyDS26qWPHjnzwwQcYhsG+ffu47777cDgcXLx4kcWLF7N8+XLy8/PZv38/ABUrVmTFihU0bNiQt99+mxUrVrBo0aKbdnLHjx9n6tSprF69mu3bt3P27Nmy/ngiIkXS0OKvRPfu3YmPj6dWrVq0aNECALvdjo+PD2PGjKFcuXKcOXOG/Px8AOrWrQvAyZMnCQ8PJyAgAID77rvvhmPXrl2bwMBAAKpWrUpubm5ZfCQREZfke2ZDpo7MXbVq1SInJ4elS5fSo0cPALKysti6dStz5sxhwoQJOBwODKPwdxK7vfCPuHbt2hw9epSrV6/icDjYt2/fDce2eWjbLiIChasWXb2VJXVkv0CXLl147733qFu3Lt9++y1eXl4EBATQq1cvfH19qVq1KpmZmdd9T3BwME8++ST9+vWjUqVK5Obm4u3t7ezcREQ8XVkPGbpKhcxFLVu2pGXLlgDExsYSGxsLFK46dGXlYX5+PpmZmaxduxaA/v37U716dR544AHna1atWnXTr0VEPIGjlBoth8NBfHw8hw8fxtfXlylTphAaGnrD6yZMmEBQUBB//etfiz2ehhbLiLe3N1euXKFnz55ERUVxzz33OOfYRESswOHGrThbt24lLy+PpKQkxo4dS0JCwg2vWblyJWlpaS7lUkdWhsaMGcOYMWPMjiEi8osUlFJHtnv3btq2bQtAs2bNOHDgwHXPf/nll3z11VdER0dz9OjREo+njkxERFxSWh1ZVlaWc4U2gJeXl3O9QGZmJq+//joTJ050OZc6MhERcYk7O3skJSVdt/1edHQ00dHRAAQGBpKdnf3f4zoceHsXlqMtW7Zw4cIFhg0bxtmzZ7l69SphYWH06tWryPdSIRMREZcYbgwt/rRw/Vzz5s358MMP6dKlC3v37qVBgwbO5wYOHMjAgQMBWLt2LUePHi22iIEKmYiIuKi09lrs2LEjO3fuJCYmBsMwmDZtGikpKeTk5BRZ/IqjQiYiIi4prUJmt9uZPHnydY+Fh4ff8LqSOrEfqZCJiIhLSmvVYmlTIRMREZfoMi4iImJpKmQiImJp2mtRREQsrbT2WixtKmQiIuISTx1atBk/XjhLLMfb9zdmR3CLn7eP2RHcFuDta3YEtwX7VTQ7glsOHrLelR7uqtPJ7Ahuu5B15JaPMTW0v8uvff5E4i2/n6vUkYn8ilitiIm1eGpHpkImIiIu8dThOxUyERFxiToyERGxNK1aFBERSyvw0MFFFTIREXGJhhZFRMTSHOrIRETEyjyzjKmQiYiIizS0KCIilqahRRERsbQCswMUQYVMRERcYqgjExERK9McmYiIWJqnzpHZzQ7gyT777DNat25NbGwsAwYMICYmhk2bNpXKsUeMGAHA4cOH2bVrV6kcU0TkdjLcuJUldWQlaNWqFbNnzwYgOzub2NhY6tatyz333HNLx3399dcB+Oc//0lISAgPPPDALWcVEbmdPLUjUyFzQ/ny5YmOjmbLli1s2rSJXbt2YRgGgwYNonPnzsTGxtKoUSO++eYbsrKyePXVVwkJCWHkyJFkZWVx9epVxo0bR8uWLXnwwQdZu3Yt69atw8fHh3vvvZfJkyeTnJwMwKhRoxgyZAhNmzY1+VOLiBTSXou/ElWqVGHRokU0btyYlStXkpubS1RUFA8++CAATZs25fnnn2f27Nls3LiRhx9+mHPnzrF48WLOnz/P8ePHnceqVq0aPXv2JCQkhKZNm+Lv78+RI0cICQnhu+++UxETEY+ixR6/EhkZGXTv3p0NGzYQGxsLQH5+PhkZGQA0btwYgLvvvptz585Rv359+vfvz5gxY8jPz3d+z81ERkaydu1aatSoQY8ePW7/hxERcYOW3/8KZGVlsXr1avr06UPLli158cUXcTgczJs3j5o1a970ew4fPkx2djYLFy4kMzOTmJgYHn74YefzNpsNh6Pw95yIiAgWLVpEpUqVePXVV8vkM4mIuEodmUV9+umnxMbGYrfbKSgo4H/+53/o2LEjCQkJ9OvXj5ycHDp06EBgYOBNv79OnTq88cYbrF+/Hh8fH5555pnrnm/SpAkvvfQS4eHhtGrVigceeIDvv/+eSpUqlcGnExFxncPwzI7MZhgemuwOFR8fzx//+Edat25d4mu9fX9TBolKj5+3j9kR3Bbg7Wt2BLcE+1U0O4LbDh5aZXYEt91Vp5PZEdx2IevILR+jX2hPl1+7/MS6W34/V+k8Mg8yZMgQrl696lIRExEpa4Yb/5UlDS16kEWLFpkdQUSkSJojExERS9MJ0SIiYmlafi8iIpamoUUREbG0AsMzS5kKmYiIuMQzy5gKmYiIuEhzZCIiYmmltWrR4XAQHx/P4cOH8fX1ZcqUKYSGhjqfT01N5d1338XLy4sGDRoQHx+P3V70ac86IVpERFxiGIbLt+Js3bqVvLw8kpKSGDt2LAkJCc7nrl69ypw5c1iyZAkrV64kKyuLDz/8sNjjqSMTERGXlNYc2e7du2nbti0AzZo148CBA87nfH19WblyJQEBAUDh1UX8/PyKPZ4KmYiIuKSglEpZVlbWdRute3l5kZ+fj7e3N3a7nZCQEACWLl1KTk6O83qPRVEhExERl7izx3xSUhJJSUnO+9HR0URHRwMQGBhIdna28zmHw4G3t/d192fOnMmxY8eYO3cuNput2PdSIbOwznffZ3YEtwzLq2B2BLc1Dvne7Ahus3t56iLpm7PiTvKZx/9pdgRTuLPY46eF6+eaN2/Ohx9+SJcuXdi7dy8NGjS47vmJEyfi6+vLvHnzil3k8SMVMpFfEasVMbGW0lp+37FjR3bu3ElMTAyGYTBt2jRSUlLIycmhSZMmJCcn06JFC/70pz8BMHDgQDp27Fjk8VTIRETEJaV1YU273c7kyZOveyw8PNz59ddff+3W8VTIRETEJQU6IVpERKxMl3ERERFLc2fVYllSIRMREZeoIxMREUvTpsEiImJpGloUERFL04U1RUTE0jRHJiIilqY5MhERsbTS2tmjtKmQiYiIS9SRiYiIpWmxh4iIWJqGFi1k4cKF/Pvf/8Zut2Oz2Rg9ejRNmjQpteNv376d06dPEx0dTVJSEr169cLHx6fUji8icjtoaNEijhw5wr/+9S9WrFiBzWbj0KFDjB8/ng0bNpTae7Rr18759Ztvvsnjjz9eascWEbld1JFZRHBwMBkZGSQnJ9OuXTvuuecekpOTOXz4MFOmTAGgUqVKTJs2jddff51GjRrRs2dPzp49y5///GfWrl3LK6+8wq5duzAMg0GDBtG5c2diY2OpXLkyly9fpmvXrpw4cYLQ0FDOnj3L6NGjqVevHtWqVaN///5cunSJwYMHs3btWpP/NERE/stTO7KSryF9hwkODmb+/Pns2bOH6OhoIiIi+PDDD5kwYQIvvPACS5cupV27drz99ttERUWxbt06AN577z169erFxx9/zHfffcfKlStZsmQJCxYs4PLlywB0796dxYsX4+XlBUBkZCRVq1Zl9uzZREZGsn79egBSU1Pp3r27KZ9fRKQohuFw+VaW1JH9zIkTJwgMDGT69OkA7N+/n2HDhnH16lUmTZoEwLVr16hbty7h4eEUFBRw6tQpNm3axOLFi0lKSuLgwYPExsYCkJ+fT0ZGBgB169Yt8n1r1apF+fLlOXLkCCkpKcybN+82f1IREfdo1aJFHD58mBUrVrBgwQL8/PyoW7cuFSpUoFq1asyYMYMaNWqwe/duzp49C0CfPn2YOXMm9erVo2LFioSFhdGyZUtefPFFHA4H8+bNo2bNmgDYbLYb3s9ms+FwFP7liIqKYv78+VSrVo3g4OCy+9AiIi7QFlUW0alTJ9LT04mMjKRcuXIYhsHf/vY37r77bsaPH09BQQEAU6dOBSAiIoKpU6cyf/58AB555BE+//xz+vXrR05ODh06dCAwMLDI92vRogXDhg1jyZIldOjQgcmTJzNz5szb/0FFRNzkqbvf2wxPTXYHunLlCgMGDGD16tXY7SVPX3av3a0MUpWeYXkVzI7gtsYh35sdwS12L88c+ilO8/RjZkdwW+bxf5odwW0+IWG3fIzqlRq7/NrTF//vlt/PVVrs4SH27NlDVFQUTz31lEtFTESkrBlu/FeWNLToIZo3b05KSorZMUREiuSpA3gqZCIi4hKtWhQREUvTzh4iImJpGloUERFL03lkIiJiaerIRETE0rTYQ0RELE2LPURExNI0tCgiIpbmqdcjUyETERGXqCMTkdvOUWC35MbBYg2eWsi0+72IiFiatlkXERFLUyETERFLUyETERFLUyETERFLUyETERFLUyETERFLUyETERFLUyETAObNm3fd/VdeecWkJO45fvw4H3/8MWfOnPHYkzVF5PbSCdF3uNWrV5OcnEx6ejr16tUDoKCggPz8fNatW2dyuuItW7aMDz74gEuXLvH4449z8uRJJk6caHasEh0/fpwTJ07QsGFDqlWrhs1mMztSic6fP09ubq7zfo0aNUxMU7wDBw7QpEkTs2O4pVevXvTo0YPHH3+cSpUqmR3HclTI7nB5eXlkZmby5ptvMnz4cADsdjtVqlTB19fX5HTF69u3L8uXL2fgwIEsXbqU3r17s2bNGrNjFcuKxTc+Pp7t27dz1113YRgGNpuNlStXmh2rSKNHj+bUqVP06NGDHj16ULFiRbMjlejy5cukpKSQkpJC9erViYyM5A9/+IPZsSxDey3e4Xx9falZsyaTJk3iwIEDzt+6v/vuOx544AGT0xXvx9/BfuxoPL3wAmzcuNFZfAcNGkTv3r3NjlSiffv2sXXrVux2a8xEzJ49m0uXLpGamsrIkSMJDg4mKiqKli1bmh2tSBUrVqR///60atWKefPmMXbsWGrWrMnTTz/NQw89ZHY8j6dCJgA888wznD9/nurVqwOFxcHTC1nXrl3p378/GRkZPPnkk3To0MHsSCWyYvENDQ0lNzeXgIAAs6O47Ny5c2RkZHDhwgXCw8PZsmUL69evZ/r06WZHu6nExETee+89AgMDiYyMJCEhgfz8fKKiolTIXKBCJkDhP3xPHi66mdjYWP7whz+QlpZGWFgYDRs2NDtSibp162a54nv69GkefvhhQkNDATx+aDEyMhJ/f3+ioqIYOXKk85eFoUOHmpysaJmZmbzyyivUqlXL+ZiPjw+TJ082MZV1aI5MAHj22WcZNWoU1apVMzuKy5599tnr7vv4+HD33XfTv39/goKCTEpVsvT0dL755hvq1q1rieJ76tSpGx77zW9+Y0IS1+zbt4+mTZs673/++ef8/ve/NzFR0QoKCigoKGDMmDHMnj0bwzAwDIMnn3ySJUuWmB3PMtSRCQB79uzh4YcfpnLlys5hr08++cTkVMXLzc2lVq1atGjRgq+++or9+/cTHBzM+PHjWbBggdnxburYsWO8/PLLHDt2jAYNGjB+/HiPLQqrV68mMjKSlStX3rCycsyYMSalKtoXX3zBkSNHWLx4MYMHDwYKC8Xy5ctJTU01Od3NrVmzhgULFnDu3DkiIiIwDAO73U6LFi3MjmYpKmQCwPvvv292BLd9//33zJo1C4C2bdsyZMgQRo0aRf/+/U1OVrTx48fz9NNP07x5c3bv3k1cXBxLly41O9ZN3X333QCEhYWZnMQ1FStW5Ny5c+Tl5XH27FmgcBh03LhxJicrWlRUFFFRUSQnJ9OnTx+z41iWCpkA8M033/DCCy/www8/0L17d+rXr8/DDz9sdqxiZWVlkZ6eTnh4OOnp6eTk5HDhwgVycnLMjlakgIAA2rdvD8BDDz3EP/7xD5MTFa1t27YAN3Rj3t7efPHFFx7XNTRo0IAGDRoQFRXFXXfdZXYcl/zY9Z44ccL5S9mPPLHr9VQqZALAlClTmD59On//+9/p06cPTzzxhMcXsokTJzJu3DgyMzPx9/enZ8+ebNq0yXk+nCeqXr068+bNo1WrVhw8eBBfX1/nEG6bNm1MTndzGzdu5OrVqzRr1ox9+/aRm5uLt7c3jRs35rnnnjM7ntMzzzzDa6+9Rq9evW54zlOHyX/sekNDQ/Hy8jI5jXWpkIlTaGgoNpuN4OBgypcvb3acEjVt2pT4+HiWLVvGzp07OX/+PE8//bTZsYpls9n49ttv+fbbbwEICQlh48aNgOcWsvz8fN59913sdjsOh4Mnn3ySd955h5iYGLOjXee1114DPLdo3cyPXe+mTZtYtGiRyWmsS4VMAAgKCmLlypVcuXKFjRs3evRuCHl5eWzcuJHExER8fX3Jyspi27Zt+Pv7mx2tRD8/jykzM9Pjh8EuXrxIfn4+vr6+5Ofnc+nSJaDw/4Mn2rVrF1euXMEwDF588UVGjhxJ9+7dzY5VrAoVKrBt2zbq1KnjPPG8bt26JqeyDi2/F6BwvmnBggWkpaURHh7On//8Z4/d861NmzZ069aNmJgY6tSpwxNPPMHbb79tdiyXvPbaayxfvpxr165x9epV6tSp4+zIPFVycjJvvfUW9evX5+jRozzxxBNkZmZy5coVRo8ebXa8G0RFRfHyyy8zadIkEhISGDVqFImJiWbHKlZsbOx19202m5bfu0EdmQCFixC6devm/C375MmTHlvIBg4cSGpqKqdOnaJPnz6W2vV++/btbN++nWnTpjF48GAmTZpkdqQS1axZk5UrV3Ly5Elq165N5cqVKSgo8Ng5HT8/P6pUqYK3tzdVq1b12M7xp5YuXcoPP/zAqVOnqFWrliWG9j2JCpkAMGzYMPLy8ggKCnJuDPv666+bHeumhg0bxrBhw/j8889ZvXo1Bw4cYObMmTz22GM0aNDA7HjFqlSpEr6+vmRnZxMaGsqVK1fMjlSiuXPnkpiYSOXKlZ2PeWoRAwgMDGTw4MH069ePxMRE57Zrnuz9999n/vz5FBQUEBERgc1m46mnnjI7lmVoaFEAGDBgAMuWLTM7xi9y+fJl3nvvPdasWcP69evNjlOsv//9787Vf0FBQXz00UekpKSYHatYAwYMICgoiLp16zrnbzx5aXheXh4nT56kXr16pKWlUadOHY/f0zImJoYlS5YwdOhQlixZQu/evVm7dq3ZsSxDHZkA0KJFC3bs2EF4eLjzMU++5tRPVaxYkdjY2BvmGTzR5MmTOXPmDBEREaxbt445c+aYHalEVtih/6fOnz/Phx9+yJYtW5yPjRgxwsREJbPb7fj6+mKz2bDZbJbaoNkTqJAJUPiPf9q0ac7Vip6+MaxVZWRk8P777zuHFDdv3uzxP2S7d+/O/v37yc/PxzAMMjMzzY5UrJEjR9K6dWtLDCn+qEWLFowZM4b//Oc/TJw4kd/+9rdmR7IUFTIBCvcA3Lx5s9kxfvXGjh1L27ZtCQkJMTuKy0aMGMG1a9fIzMykoKCAu+66i27dupkdq0jly5f3yNWUxRkzZgzbt2+ncePGhIWF8cgjj5gdyVJUyAQo3N5n7969NG7c2PmYp88rWJG/v7/Hd2A/l5WVxbJly3j++eeZMGGCc0NeT1W/fn02btzIPffc49xey9PPyerVqxe9e/cmJiaGwMBAs+NYjgqZAIUnkX700UfO+zabjW3btpkX6Ffm2LFjQOFOHikpKdx7772W+SH74wrFK1eu4O/v7/HL2Q8dOsShQ4ec961wTtbChQt57733+NOf/kT9+vWJjIzk/vvvNzuWZWjVotzAk88RsqrY2FhsNpvznLfLly/j5eVFYGCgx/+QTUxM5OLFi/j4+LBt2zYCAgJYvHix2bGKZdVzsjIyMpg5cyY7d+7k888/NzuOZagjE6Bw0YHD4SAvL4+ZM2cydOhQj76irtXExcXx3HPPsXr1aj766CPi4+OpUKGCR+8N+dMLlzocDux2O3fddRfe3p79Y8OK52StX7+edevW4XA46N279w1bmUkJDBHDMPr06WN8//33xqBBg4zc3Fyjf//+Zkf6VRk6dKhx6NAhwzAMo3PnzsaBAweMH374wYiOjjY5WdG6detmdOjQwXjttdeMHTt2GNu3b3fePFl0dLSRm5trDBgwwHA4HEbPnj3NjlSi6dOnG+np6WbHsCzP/tVKyoyfnx9QuOLrx50npPQYhkGjRo34z3/+w5UrV7j33nuBG6/15UlSUlJIS0tjw4YNLFy4kAceeIAePXoQGhpqdrRi2Ww2y52TNWLECN566y3Onj3LQw89RMOGDT3+z9mT2M0OIJ6hZs2a9O7dm969e/P666/TtGlTsyP9qjgcDgB27NhB69atgcIdKDz5IqBQuJr1r3/9K0uWLKFVq1a88sorREVFmR2rWA888ABjx4611DlZzz33HLVq1eL48eOEhITw/PPPmx3JUtSRCQAJCQlkZ2dTvnx5fvvb31rqPCcraN26NTExMZw5c4b58+dz8uRJ4uPj6dKli9nRSpSVlcUHH3xAamoqV65coUePHmZHKtLXX3+N3W7n4MGD9OjRw7nri6e7ePEiffr0YcOGDTRv3txSG2F7AhUyAax5DScrGTZsGI8++ijBwcFUrlyZkydP0rdvXzp27Gh2tCJt3ryZjRs3kpGRQadOnZg0aRI1a9Y0O1aRNm/ezFtvvUXfvn0ZN24cGRkZrFq1iurVq9OhQwez45UoPT0dgDNnzjj3tBTXaPm9ANa8hpPcXo0aNSIsLIxGjRoB18/nvfLKK2bFKlLfvn155513KFeunPOxrKws/vKXv7B06VITk5UsLS2NCRMmcOTIEUJDQ5kyZcp1mxNI8dSRCWDNazjJ7eXp57f9nLe393VFDAov6eLJ50QePHiQ559/ntWrVzN06FDi4+PJzs7m9OnTKmRuUCETwJrXcJLb6/e//73ZEdxS1ArQHxfaeKLZs2eTkJCAj48Pc+bM4a233iI0NJQnnniCRx991Ox4lqFCJgC8+uqr113DKTIy0uxIIm45cuQIY8eOve4xwzCcc0+eqKjTMjRH5h4VMgHgwoULLFiwgAsXLvDHP/6RK1eu8Lvf/c7sWCIuK+rabjExMWUbxA1FnZah8zjdo0ImAM5dzefNm0eLFi2Ii4tj1apVZscScZnVhkLB2qdleBIVMgEgNzeX1q1bM3/+fMLCwpw7fYjI7WPF0zI8kQqZAIXXHtuxYwcOh4O9e/fqWmQiZSQ8PNz5de3ataldu7aJaaxJ55EJUHgS5owZM0hLSyM8PJxx48ZRq1Yts2OJiJRIhUycCgoKMAyDvXv30rRpU3VlImIJGloUAGbOnEmtWrXIyMjg4MGDhISEMGPGDLNjiYiUSCcrCAC7d+8mJiaGL7/8knfeeYczZ86YHUlExCUqZAIUns+yb98+atasSV5eHt9//73ZkUREXKI5MgEgMTGR9evXM23aNFatWkWDBg20u4eIWIIKmdzg9OnT2mtRRCxDiz0EKNzp3N/fn8uXL7N27Vratm3Ls88+a3YsEZESaY5MANi4cSOPP/4427dvZ+PGjRw6dMjsSCIiLlEhE6DwEhhnz54lJCQEm83GpUuXzI4kIuISFTIBoGXLlgwYMIABAwYwbdo0OnXqZHYkERGXaLGH3ODatWv4+PiYHUNExCVa7CEAbNu2jeXLl3Pt2jUMw+DixYukpKSYHUtEpEQaWhQA3njjDUaMGEH16tXp2bMnDRs2NDuSiIhLVMgEgMqVK3PfffcB0KtXL21RJSKWoUImAPj4+LBr1y7y8/PZsWMHZ8+eNTuSiIhLtNhDAPjPf/7D0aNHqVq1Kq+++ioRERF07drV7FgiIiVSIRMREUvT0KKIiFiaCpmIiFiaCpmwdetWAH744QdmzJjB7NmzycnJMTmViIhrVMjucC+//DLvvfceBQUFvPjii+Tk5FC5cmXi4+PNjiYi4hLt7HGHO3jwIP/4xz/Iz8/n448/5qOPPiIgIIC+ffuaHU1ExCXqyO5wXl5eAOzbt4/69esTEBAAFO63KCJiBerI7nBeXl588sknrFu3zrnj/b///W8qVqxocjIREdfoPLI73MmTJ5k1axY1atRg1KhRfPbZZ8ycOZM5c+YQFhZmdjwRkRKpkImIiKVpaPEOl5eXx+zZs3n//ffJy8ujfPnydO3alaeeegpvb/31EBHPp8Ued7gZM2YQFBTE5s2b+eSTT1izZg0Oh4MZM2aYHU1ExCUqZHe4gwcPMnz4cPz8/AAIDAxk1KhRfP311yYnExFxjQrZHc7Hx+emj9tstjJOIiLyy2gSRLh27Ro/X/OjNUAiYhUqZHe4U6dOERERAVxfvNSRiYhVaPm93NTevXtp1qyZ2TFEREqkjkyc8vLySElJITExkby8PFJTU82OJCJSIhUy4bvvviMxMZHNmzdjGAazZ8+mefPmZscSEXGJVi3e4f7yl78wfvx4wsLCSE1NpX79+ipiImIpKmR3OMMw8Pb25urVqzgcDi3yEBHL0WIP4cyZMyQnJ5OSkkJOTg5Tp06lTZs22O36PUdEPJ8KmTgZhsGOHTtITk5m3759fPTRR2ZHEhEpkQqZ3NT69et5/PHHzY4hIlIijR3d4dauXUubNm3o0KEDBw8e5IcffmDkyJG88847ZkcTEXGJOrI7XPfu3Vm2bBlnz54lISGBzMxMHn30UZ566qki92EUEfEkOo/sDlepUiWCgoIICgoiPT2d+Ph42rdvb3YsERGXaWjxDvfT5fY1atRQERMRy1FHdoe7ePEiO3fuxOFwkJWVxSeffOJ8rk2bNiYmExFxjebI7nDPPvtskc9Nnz69DJOIiPwyKmRynf/7v/+jcePGZscQEXGZ5sjkOgkJCWZHEBFxiwqZXEcNuohYjQqZXGfAgAFmRxARcYvmyO5wSUlJRT4XHR1dhklERH4ZLb+/w509e9bsCCIit0QdmVwnLy8PAF9fX5OTiIi4RnNkd7hjx44xfPhwpkyZwq5du2jbti1t27Zl/fr1ZkcTEXGJCtkd7rnnniM2NpbmzZszbNgwkpKS+OCDD1i+fLnZ0UREXKI5sjuct7c3Dz74IABLliyhTp06AJQrV87EVCIirlNHdof76abBP50XczgcZsQREXGbOrI73JEjRxg7diyGYVz3dXp6utnRRERcolWLd7jPP/+8yOd+//vfl2ESEZFfRoVMREQsTXNkIiJiaSpkIiJiaSpkIiJiaSpkIiJiaSpkIiJiaf8P5IuV91UlZpEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "corr_matrix = data_no_null.corr()\n",
    "sns.heatmap(corr_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BI-RADS assessment</th>\n",
       "      <th>Age</th>\n",
       "      <th>Shape</th>\n",
       "      <th>Margin</th>\n",
       "      <th>Density</th>\n",
       "      <th>Severity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>BI-RADS assessment</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.094623</td>\n",
       "      <td>0.180120</td>\n",
       "      <td>0.157771</td>\n",
       "      <td>0.028356</td>\n",
       "      <td>0.223826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Age</th>\n",
       "      <td>0.094623</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.380096</td>\n",
       "      <td>0.420913</td>\n",
       "      <td>0.052417</td>\n",
       "      <td>0.455216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Shape</th>\n",
       "      <td>0.180120</td>\n",
       "      <td>0.380096</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.738014</td>\n",
       "      <td>0.073969</td>\n",
       "      <td>0.564763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Margin</th>\n",
       "      <td>0.157771</td>\n",
       "      <td>0.420913</td>\n",
       "      <td>0.738014</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.124860</td>\n",
       "      <td>0.574498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Density</th>\n",
       "      <td>0.028356</td>\n",
       "      <td>0.052417</td>\n",
       "      <td>0.073969</td>\n",
       "      <td>0.124860</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.068651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Severity</th>\n",
       "      <td>0.223826</td>\n",
       "      <td>0.455216</td>\n",
       "      <td>0.564763</td>\n",
       "      <td>0.574498</td>\n",
       "      <td>0.068651</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    BI-RADS assessment       Age     Shape    Margin  \\\n",
       "BI-RADS assessment            1.000000  0.094623  0.180120  0.157771   \n",
       "Age                           0.094623  1.000000  0.380096  0.420913   \n",
       "Shape                         0.180120  0.380096  1.000000  0.738014   \n",
       "Margin                        0.157771  0.420913  0.738014  1.000000   \n",
       "Density                       0.028356  0.052417  0.073969  0.124860   \n",
       "Severity                      0.223826  0.455216  0.564763  0.574498   \n",
       "\n",
       "                     Density  Severity  \n",
       "BI-RADS assessment  0.028356  0.223826  \n",
       "Age                 0.052417  0.455216  \n",
       "Shape               0.073969  0.564763  \n",
       "Margin              0.124860  0.574498  \n",
       "Density             1.000000  0.068651  \n",
       "Severity            0.068651  1.000000  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see some correlation between the features. However, some of the features are categorical in nature (as per the dataset description) an dthis needs to be taken into account.\n",
    "\n",
    "- Shape: mass shape: round=1 oval=2 lobular=3 irregular=4 (nominal)\n",
    "- Margin: mass margin: circumscribed=1 microlobulated=2 obscured=3 ill-defined=4 spiculated=5 (nominal)\n",
    "- Density: mass density high=1 iso=2 low=3 fat-containing=4 (ordinal)\n",
    "\n",
    "We'll need to transform the features into categorical type and we'll also aply one-hot encoding since for shape and margin there is no inherent order."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mario\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "data_no_null['Shape'] = data_no_null['Shape'].map({1:'round',2:'oval',3:'lobular',4:'irregular'})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mario\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "data_no_null['Margin'] = data_no_null['Margin'].map({1:'circumscribed', 2:'microlobulated', 3:'obscured',\n",
    "                                                     4:'ill-defined',5:'spiculated'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BI-RADS assessment</th>\n",
       "      <th>Age</th>\n",
       "      <th>Shape</th>\n",
       "      <th>Margin</th>\n",
       "      <th>Density</th>\n",
       "      <th>Severity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>lobular</td>\n",
       "      <td>spiculated</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>irregular</td>\n",
       "      <td>spiculated</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>round</td>\n",
       "      <td>circumscribed</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>round</td>\n",
       "      <td>spiculated</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>round</td>\n",
       "      <td>ill-defined</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    BI-RADS assessment   Age      Shape         Margin  Density  Severity\n",
       "0                  5.0  67.0    lobular     spiculated      3.0         1\n",
       "2                  5.0  58.0  irregular     spiculated      3.0         1\n",
       "3                  4.0  28.0      round  circumscribed      3.0         0\n",
       "8                  5.0  57.0      round     spiculated      3.0         1\n",
       "10                 5.0  76.0      round    ill-defined      3.0         1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_no_null.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BI-RADS assessment</th>\n",
       "      <th>Age</th>\n",
       "      <th>Density</th>\n",
       "      <th>Severity</th>\n",
       "      <th>Shape_irregular</th>\n",
       "      <th>Shape_lobular</th>\n",
       "      <th>Shape_oval</th>\n",
       "      <th>Shape_round</th>\n",
       "      <th>Margin_circumscribed</th>\n",
       "      <th>Margin_ill-defined</th>\n",
       "      <th>Margin_microlobulated</th>\n",
       "      <th>Margin_obscured</th>\n",
       "      <th>Margin_spiculated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    BI-RADS assessment   Age  Density  Severity  Shape_irregular  \\\n",
       "0                  5.0  67.0      3.0         1                0   \n",
       "2                  5.0  58.0      3.0         1                1   \n",
       "3                  4.0  28.0      3.0         0                0   \n",
       "8                  5.0  57.0      3.0         1                0   \n",
       "10                 5.0  76.0      3.0         1                0   \n",
       "\n",
       "    Shape_lobular  Shape_oval  Shape_round  Margin_circumscribed  \\\n",
       "0               1           0            0                     0   \n",
       "2               0           0            0                     0   \n",
       "3               0           0            1                     1   \n",
       "8               0           0            1                     0   \n",
       "10              0           0            1                     0   \n",
       "\n",
       "    Margin_ill-defined  Margin_microlobulated  Margin_obscured  \\\n",
       "0                    0                      0                0   \n",
       "2                    0                      0                0   \n",
       "3                    0                      0                0   \n",
       "8                    0                      0                0   \n",
       "10                   1                      0                0   \n",
       "\n",
       "    Margin_spiculated  \n",
       "0                   1  \n",
       "2                   1  \n",
       "3                   0  \n",
       "8                   1  \n",
       "10                  0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_no_null = pd.get_dummies(data_no_null, ['Shape','Margin'])\n",
    "data_no_null.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "According to the dataset description:\n",
    "\n",
    "_Each instance has an associated BI-RADS assessment ranging from 1 (definitely benign)\n",
    "    to 5 (highly suggestive of malignancy) assigned in a double-review process by\n",
    "    physicians. Assuming that all cases with BI-RADS assessments greater or equal\n",
    "    a given value (varying from 1 to 5), are malignant and the other cases benign,\n",
    "    sensitivities and associated specificities can be calculated. These can be an\n",
    "    indication of how well a CAD system performs compared to the radiologists._\n",
    "    \n",
    "So in our analysis we won't use the BI-RADS as a feature, because we would like the Computer Aided Diagnosis (CAD) system to be autonomous with no need for expert's input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Density</th>\n",
       "      <th>Shape_irregular</th>\n",
       "      <th>Shape_lobular</th>\n",
       "      <th>Shape_oval</th>\n",
       "      <th>Shape_round</th>\n",
       "      <th>Margin_circumscribed</th>\n",
       "      <th>Margin_ill-defined</th>\n",
       "      <th>Margin_microlobulated</th>\n",
       "      <th>Margin_obscured</th>\n",
       "      <th>Margin_spiculated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>67.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>58.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>28.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>57.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>76.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Age  Density  Shape_irregular  Shape_lobular  Shape_oval  Shape_round  \\\n",
       "0   67.0      3.0                0              1           0            0   \n",
       "2   58.0      3.0                1              0           0            0   \n",
       "3   28.0      3.0                0              0           0            1   \n",
       "8   57.0      3.0                0              0           0            1   \n",
       "10  76.0      3.0                0              0           0            1   \n",
       "\n",
       "    Margin_circumscribed  Margin_ill-defined  Margin_microlobulated  \\\n",
       "0                      0                   0                      0   \n",
       "2                      0                   0                      0   \n",
       "3                      1                   0                      0   \n",
       "8                      0                   0                      0   \n",
       "10                     0                   1                      0   \n",
       "\n",
       "    Margin_obscured  Margin_spiculated  \n",
       "0                 0                  1  \n",
       "2                 0                  1  \n",
       "3                 0                  0  \n",
       "8                 0                  1  \n",
       "10                0                  0  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_target = data_no_null['Severity']\n",
    "\n",
    "data_features = data_no_null.drop(['BI-RADS assessment','Severity'], axis=1)\n",
    "data_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     1\n",
       "2     1\n",
       "3     0\n",
       "8     1\n",
       "10    1\n",
       "Name: Severity, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_target.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We've seen previously that the features present different magnitudes of standard deviation and mean value. For this, we'll attenpt a scaling of the numerical features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preliminary model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll first attempt a classification with a simple model and no other feture engineering."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data_features, data_target, stratify=data_target, test_size=0.3, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(581, 11)\n",
      "(249, 11)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=300,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg = LogisticRegression(solver='lbfgs',max_iter=300)\n",
    "log_reg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8112449799196787"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Accuracy of 80%, not bad for a very simple classifier, but we definitely can do better!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Do better, fine tune, different classifiers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = {'C':np.logspace(0.01,10,50)}\n",
    "\n",
    "grid_search = GridSearchCV(estimator = log_reg, param_grid = param_grid, cv = 5, verbose = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mario\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\mario\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "C:\\Users\\mario\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=300, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='lbfgs',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'C': array([1.023292...\n",
       "       3.42043729e+06, 5.46964555e+06, 8.74654902e+06, 1.39866686e+07,\n",
       "       2.23661811e+07, 3.57659189e+07, 5.71935348e+07, 9.14585875e+07,\n",
       "       1.46252076e+08, 2.33872734e+08, 3.73987548e+08, 5.98046143e+08,\n",
       "       9.56339832e+08, 1.52928981e+09, 2.44549820e+09, 3.91061352e+09,\n",
       "       6.25348985e+09, 1.00000000e+10])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.023292992280754, class_weight=None, dual=False,\n",
       "                   fit_intercept=True, intercept_scaling=1, l1_ratio=None,\n",
       "                   max_iter=300, multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8037865748709122"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No improvement. Let's try different classifiers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mario\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "                           decision_function_shape='ovr', degree=3,\n",
       "                           gamma='auto_deprecated', kernel='rbf', max_iter=-1,\n",
       "                           probability=False, random_state=None, shrinking=True,\n",
       "                           tol=0.001, verbose=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'C': [0.01, 0.1, 1, 2, 5, 10, 50], 'gamma': ['auto'],\n",
       "                         'kernel': ('linear', 'rbf')},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "svc = SVC()\n",
    "parameters = {'kernel':('linear', 'rbf'), 'C':[0.01, 0.1, 1, 2, 5, 10, 50], 'gamma':['auto']}\n",
    "grid_search_svm = GridSearchCV(estimator=svc, param_grid = parameters, cv = 5, verbose = 0)\n",
    "\n",
    "grid_search_svm.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7917383820998278"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_svm.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7630522088353414"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_svm.best_estimator_.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try a KNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### KNN classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mario\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30,\n",
       "                                            metric='minkowski',\n",
       "                                            metric_params=None, n_jobs=None,\n",
       "                                            n_neighbors=5, p=2,\n",
       "                                            weights='uniform'),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'n_neighbors': array([ 3,  6,  9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "KNN = KNeighborsClassifier()\n",
    "parameters_knn = {'n_neighbors':np.arange(3,50,3)}\n",
    "\n",
    "grid_search_knn = GridSearchCV(estimator = KNN, param_grid= parameters_knn, cv=5, verbose=0)\n",
    "\n",
    "grid_search_knn.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.774526678141136"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_knn.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No improvements. Let's get back to our data and see if we can normalize them."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalize the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The only feature that makes sense to be normalized is the age since this is the one with the highest deviation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_features_norm = data_features\n",
    "data_features_norm['Age'] = (data_features_norm['Age']-data_features_norm['Age'].mean())/data_features_norm['Age'].std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Density</th>\n",
       "      <th>Shape_irregular</th>\n",
       "      <th>Shape_lobular</th>\n",
       "      <th>Shape_oval</th>\n",
       "      <th>Shape_round</th>\n",
       "      <th>Margin_circumscribed</th>\n",
       "      <th>Margin_ill-defined</th>\n",
       "      <th>Margin_microlobulated</th>\n",
       "      <th>Margin_obscured</th>\n",
       "      <th>Margin_spiculated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.764602</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.151179</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.893562</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.083021</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.378024</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Age  Density  Shape_irregular  Shape_lobular  Shape_oval  \\\n",
       "0   0.764602      3.0                0              1           0   \n",
       "2   0.151179      3.0                1              0           0   \n",
       "3  -1.893562      3.0                0              0           0   \n",
       "8   0.083021      3.0                0              0           0   \n",
       "10  1.378024      3.0                0              0           0   \n",
       "\n",
       "    Shape_round  Margin_circumscribed  Margin_ill-defined  \\\n",
       "0             0                     0                   0   \n",
       "2             0                     0                   0   \n",
       "3             1                     1                   0   \n",
       "8             1                     0                   0   \n",
       "10            1                     0                   1   \n",
       "\n",
       "    Margin_microlobulated  Margin_obscured  Margin_spiculated  \n",
       "0                       0                0                  1  \n",
       "2                       0                0                  1  \n",
       "3                       0                0                  0  \n",
       "8                       0                0                  1  \n",
       "10                      0                0                  0  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_features_norm.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "let's give it a shot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_norm, X_test_norm, y_train, y_test = train_test_split(data_features_norm, data_target, stratify=data_target, test_size=0.3, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=300, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='lbfgs',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'C': array([1.023292...\n",
       "       2.03981996e+065, 2.23977340e+067, 2.45932728e+069, 2.70040294e+071,\n",
       "       2.96511005e+073, 3.25576508e+075, 3.57491159e+077, 3.92534245e+079,\n",
       "       4.31012431e+081, 4.73262443e+083, 5.19654013e+085, 5.70593120e+087,\n",
       "       6.26525534e+089, 6.87940726e+091, 7.55376145e+093, 8.29421923e+095,\n",
       "       9.10726042e+097, 1.00000000e+100]),\n",
       "                         'solver': ['lbfgs']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_grid = {'C':np.logspace(0.01,100,50), 'solver':['lbfgs']}\n",
    "\n",
    "grid_search = GridSearchCV(estimator = log_reg, param_grid = param_grid, cv = 5, verbose = 0)\n",
    "\n",
    "grid_search.fit(X_train_norm, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8003442340791739"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What if we include the BI-RADS?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_target = data_no_null['Severity']\n",
    "\n",
    "data_features = data_no_null.drop(['Severity'], axis=1)\n",
    "data_features['Age'] = (data_features['Age']-data_features['Age'].mean())/data_features['Age'].std()\n",
    "data_features['Density'] = (data_features['Density']-data_features['Density'].mean())/data_features['Density'].std()\n",
    "data_features['BI-RADS assessment'] = (data_features['BI-RADS assessment']-data_features['BI-RADS assessment'].mean())/data_features['BI-RADS assessment'].std()\n",
    "\n",
    "data_features = data_features.drop(['Shape_round', 'Margin_obscured'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BI-RADS assessment</th>\n",
       "      <th>Age</th>\n",
       "      <th>Density</th>\n",
       "      <th>Shape_irregular</th>\n",
       "      <th>Shape_lobular</th>\n",
       "      <th>Shape_oval</th>\n",
       "      <th>Margin_circumscribed</th>\n",
       "      <th>Margin_ill-defined</th>\n",
       "      <th>Margin_microlobulated</th>\n",
       "      <th>Margin_spiculated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.320924</td>\n",
       "      <td>0.764602</td>\n",
       "      <td>0.240321</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.320924</td>\n",
       "      <td>0.151179</td>\n",
       "      <td>0.240321</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.208633</td>\n",
       "      <td>-1.893562</td>\n",
       "      <td>0.240321</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.320924</td>\n",
       "      <td>0.083021</td>\n",
       "      <td>0.240321</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.320924</td>\n",
       "      <td>1.378024</td>\n",
       "      <td>0.240321</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    BI-RADS assessment       Age   Density  Shape_irregular  Shape_lobular  \\\n",
       "0             0.320924  0.764602  0.240321                0              1   \n",
       "2             0.320924  0.151179  0.240321                1              0   \n",
       "3            -0.208633 -1.893562  0.240321                0              0   \n",
       "8             0.320924  0.083021  0.240321                0              0   \n",
       "10            0.320924  1.378024  0.240321                0              0   \n",
       "\n",
       "    Shape_oval  Margin_circumscribed  Margin_ill-defined  \\\n",
       "0            0                     0                   0   \n",
       "2            0                     0                   0   \n",
       "3            0                     1                   0   \n",
       "8            0                     0                   0   \n",
       "10           0                     0                   1   \n",
       "\n",
       "    Margin_microlobulated  Margin_spiculated  \n",
       "0                       0                  1  \n",
       "2                       0                  1  \n",
       "3                       0                  0  \n",
       "8                       0                  1  \n",
       "10                      0                  0  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data_features, data_target, stratify=data_target, test_size=0.3, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=300, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='lbfgs',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'C': array([1.023292...\n",
       "       2.03981996e+065, 2.23977340e+067, 2.45932728e+069, 2.70040294e+071,\n",
       "       2.96511005e+073, 3.25576508e+075, 3.57491159e+077, 3.92534245e+079,\n",
       "       4.31012431e+081, 4.73262443e+083, 5.19654013e+085, 5.70593120e+087,\n",
       "       6.26525534e+089, 6.87940726e+091, 7.55376145e+093, 8.29421923e+095,\n",
       "       9.10726042e+097, 1.00000000e+100]),\n",
       "                         'solver': ['lbfgs']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_grid = {'C':np.logspace(0.01,100,50), 'solver':['lbfgs']}\n",
    "\n",
    "grid_search = GridSearchCV(estimator = log_reg, param_grid = param_grid, cv = 5, verbose = 0)\n",
    "\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8123924268502581"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30,\n",
       "                                            metric='minkowski',\n",
       "                                            metric_params=None, n_jobs=None,\n",
       "                                            n_neighbors=5, p=2,\n",
       "                                            weights='uniform'),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'n_neighbors': array([ 3,  6,  9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48])},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KNN = KNeighborsClassifier()\n",
    "parameters_knn = {'n_neighbors':np.arange(3,50,3)}\n",
    "\n",
    "grid_search_knn = GridSearchCV(estimator = KNN, param_grid= parameters_knn, cv=5, verbose=0)\n",
    "\n",
    "grid_search_knn.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8089500860585198"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_knn.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try a random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "             estimator=RandomForestClassifier(bootstrap=True, class_weight=None,\n",
       "                                              criterion='gini', max_depth=None,\n",
       "                                              max_features='auto',\n",
       "                                              max_leaf_nodes=None,\n",
       "                                              min_impurity_decrease=0.0,\n",
       "                                              min_impurity_split=None,\n",
       "                                              min_samples_leaf=1,\n",
       "                                              min_samples_split=2,\n",
       "                                              min_weight_fraction_leaf=0.0,\n",
       "                                              n_estimators='warn', n_jobs=None,\n",
       "                                              oob_score=False,\n",
       "                                              random_state=None, verbose=0,\n",
       "                                              warm_start=False),\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'max_depth': [3, 5, 8, 10],\n",
       "                         'n_estimators': [5, 10, 20, 100, 150, 200]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_rf = {'n_estimators':[5,10,20,100,150,200], 'max_depth':[3,5,8,10]}\n",
    "\n",
    "grid_search_rf = GridSearchCV(estimator=rf, param_grid=param_rf, cv=5, verbose=0)\n",
    "grid_search_rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8416523235800344"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_rf.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=5, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=150,\n",
       "                       n_jobs=None, oob_score=False, random_state=None,\n",
       "                       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_rf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8514056224899599"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search_rf.best_estimator_.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = grid_search_rf.best_estimator_.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "conf_m = confusion_matrix(y_test,preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[110,  18],\n",
       "       [ 19, 102]], dtype=int64)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.86      0.86       128\n",
      "           1       0.85      0.84      0.85       121\n",
      "\n",
      "    accuracy                           0.85       249\n",
      "   macro avg       0.85      0.85      0.85       249\n",
      "weighted avg       0.85      0.85      0.85       249\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(y_test, preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9058303202479339\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "probas_ = grid_search_rf.best_estimator_.predict_proba(X_test)\n",
    "fpr, tpr, thresholds = roc_curve(y_test, probas_[:,1])\n",
    "print(auc(fpr,tpr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAESCAYAAAD5d3KwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABI6UlEQVR4nO3dd3zNZ//H8dc5OZkSiYSqGiFBUDXCXXsVFQm1d1HjRltVikYRjT1ib6q9q9QqaUtQatQqWntFUnsUQRIyZJ7r90d+jqYkkcg5J+PzfDw8knO+63OuHOd9vuu6NEophRBCCPH/tOYuQAghRM4iwSCEECIVCQYhhBCpSDAIIYRIRYJBCCFEKhIMQgghUpFgyCc8PDxo3bo1bdq0oW3btrRo0YIOHTpw9uxZo2yvTZs2PH782CjrNpczZ84wbtw4AM6ePcuQIUOMvk0PDw/Cw8ONvp1/Gzt2LOfOncv0chn93aOioujVq9dLzy/MQ2fuAoTprFy5EmdnZ8Pjr7/+mkmTJrF+/fps39bPP/+c7es0t0uXLnHv3j0A3nrrLebPn2/miozn999/p0uXLpleLqO/+6NHj1J9GcmL75O8QIIhn0pKSuLOnTs4OjoanluyZAk7d+5Er9dTvHhxvvzyS4oWLcr9+/f58ssvuXLlClqtlq5du9KrVy+ioqKYPHkyoaGhJCYmUqdOHT7//HN0Oh0eHh4cPnyYjz76iD59+tCiRQsAAgICABg5ciQ//PADa9euRa/X4+TkhJ+fH+7u7owaNYrIyEhu3rxJ48aNGTlyZKra169fz6pVq9BqtRQuXBg/Pz/KlCnDqFGjsLa25uLFizx8+JB69eoxduxYLC0tuXz5MpMnTyYyMpLk5GR69uxJx44dOXr0KJMnT8bOzo6YmBg2bdrEjBkzOH36NDExMSilmDRpEm+88Qbz588nKiqKL774grZt2zJx4kSCgoIYNWoU9vb2hISEcPfuXTw8PJg+fToFChRg3759zJw5E61WS8WKFfn9999Zs2YNJUqUSPWaTp8+zaRJk3jy5AmWlpZ8/vnn1KlTB4AFCxZw+vRpIiMj6devHz169CA2NhZ/f3+uX79OZGQkBQoUYObMmbi5udGzZ08cHR25cuUK3bp146233iIgIICEhATu379P3bp1mTJlCgB79+5l7ty56PV67OzsGD9+PNu3bycsLIwRI0YwY8YM3Nzc0vw7V65cmaZNm3Lx4kVmzpxJx44dOXz4MMnJyfj6+hIREQFAo0aNGDp0KF988QVxcXG0adOGwMBAKlWqxOHDh3F2dmbZsmX8+OOP6HQ6XF1dmTZtGg4ODsb5DyDSp0S+UL58edWqVSvVqlUrVa9ePfXOO++oiRMnqgcPHiillPrxxx/V0KFDVWJiolJKqXXr1qn+/fsrpZT6+OOP1fTp05VSSj1+/Fj5+Pioa9euqVGjRqnvvvtOKaVUUlKSGjFihFq+fLlhew8fPlQbN25UAwYMMMxTv359dfXqVXX06FHVvXt3FRsbq5RS6sCBA8rLy0sppZSvr6/q3bv3C1/H77//rpo1a6YePnyolFJq06ZNqmXLlkqv1ytfX1/Vtm1bFR0dreLj41WPHj3UqlWrVGJiovL29lbnzp0zvIaWLVuqkydPqiNHjqgKFSqoW7duKaWUOnHihPrkk09UcnKyUkqpZcuWqYEDBxq29fS1HDlyRPn4+Bjq7dKli4qPj1cJCQmqbdu2auPGjSo8PFy9/fbbKjg4WCmlVGBgoCpfvry6efNmqteUkJCg6tWrp/bu3auUUurs2bOqVatWKjk5WZUvX159/fXXSimlzp8/rypXrqwSEhLU9u3b1cSJEw3r8PPzUxMmTFBKKfX++++rL774wjBt2LBh6siRI0oppaKjo1WtWrXU2bNn1f3791WNGjXU+fPnlVJK7dixQ/Xr108ppVSTJk3UmTNnlFIqw7/zjz/+aNjW07/7woULlZ+fn1JKqZiYGDV06FD1+PFjdfPmTVWtWrXn5t+1a5d69913VWRkpFJKqSlTpqjFixe/8D0gjE/2GPKRp4eSzp8/z4ABA6hVqxYuLi5AyjfHs2fP0qFDBwD0ej1PnjwBUg4rPP3W7uDgQFBQEAC//fYbZ8+eZePGjQDExcU9t01vb29mzJjB/fv3uXDhAqVLl6Z06dJs2LCB69ev07VrV8O8jx8/JjIyEoAaNWq88DUcOHAAb29vwyGx9u3bM3nyZG7dugVAu3btKFCgAJBy/Hr37t3Url2bGzduMHr0aMN64uLiuHDhAu7u7hQrVozixYsDUL16dRwdHVm3bh03b97k6NGjhvWlp0GDBlhZWQFQvnx5Hj16xLFjx3B3d6dChQqG2iZNmvTcsqGhoWi1Who3bgxA5cqV2bJli2F6q1atAKhYsSIJCQlER0fj5eVFyZIlWbVqFdevX+ePP/6gevXqhmVq1qxp+H3atGns37+fpUuXcuXKFeLj44mNjeXEiROUK1eOSpUqAfDuu+/y7rvvPldfRn/nf27rn+0xYMAA7ty5Q926dRk+fDgODg48evTohe13+PBhvLy8DHuwX3zxxQvnE6YhwZAPvfnmm3zxxReMGjWKihUrUqJECfR6Pf3796d79+4AJCQkGP4T63Q6NBqNYfmbN29SqFAh9Ho98+bNw93dHUj5YP/nfAC2tra0aNGCoKAgTp48SadOnYCU4GnTpo0hcPR6PWFhYYYPBjs7uxfWrtfrn3tOKUVSUhIAFhYWqZ7XarUkJyfj4OCQ6nj2gwcPcHBw4NSpU6m29dtvvzF58mT69OlD06ZNcXNzY/PmzRm2qY2NjeF3jUaDUgoLCwvUv7oi02qfv97DwsLiuXYLDQ3Fzc0NSGn/p+t9+rrWrFnDhg0b6NGjB61bt8bJyckQjpC6/d5//308PDxo0KABLVu25PTp04b6/rldpRQhISGGIHsqo7/zi/5WVapUYffu3Rw+fJgjR47QqVMnvvrqK5ycnF7Qes+3wePHj3n8+PFzh9yEachVSflUq1atqFKlClOnTgWgfv36bNy4kejoaADmzZvH559/DkCdOnXYtGkTkHJVSe/evbl27Rr169fn22+/RSlFQkICH374IatXr35uW507d+bHH3/kxIkThnMN9evXZ+vWrYSFhQGwdu1aevfunWHdDRo0YNu2bYYrdTZt2oSTkxOurq4AbN++nYSEBOLj4/nxxx9p0qQJZcqUwcbGxhAMd+7coVWrVi+86ubQoUM0adKE7t27U7lyZXbt2kVycjKQ8uH1NIBehqenJ9euXePixYsA7Nix44Xh6ebmhkaj4dChQwCcP3+e3r17vzAEnzp48CDt2rWjU6dOlClThj179hjq/KfHjx9z9uxZRowYwbvvvsvdu3e5ceMGer2eqlWrcvnyZf766y8Adu/ebQjqf77Wl/07/9PMmTNZvHgxzZo1Y8yYMZQtW5a//voLnU5HcnLyc4FZt25dfv31V8P7b8GCBXz77bfpbkMYj+wx5GN+fn689957HDhwgE6dOnHv3j06d+6MRqOhWLFiTJs2DYBx48bh7+9P69atUUoxcOBAKleuzJgxY5g8eTKtW7cmMTGRunXr0r9//+e2U7lyZSwsLPDy8sLa2hpI+bD573//S9++fdFoNNjb27Nw4cLnPjT/rV69enzwwQeGD86nJy2ffhO3sbGhe/fuPH782HBJrlarZfHixUyePJkVK1aQlJTEp59+So0aNTh69Giq9Xft2pXhw4fTunVrkpKSqFevnuGEfLVq1Vi0aBGDBw+mZ8+eGbavk5MTs2fPxtfXF61WS+XKldHpdNja2qaaz8rKigULFjBlyhRmzJiBpaUlCxYsMByaepG+ffsybtw4w+GdatWqERoa+tx8BQsWZMCAAbRr1w47OzuKFi2Kp6cn169fp06dOsycORNfX1+Sk5Oxt7dnzpw5ADRv3pyRI0fi7+//0n/nf+rduzejRo2iVatWWFlZ4eHhgY+PDxYWFlSpUgUfHx++//57w/yNGjXi0qVLdOvWDYCyZcsyceLEDNtYGIdG/Tu6hcilRo0aRbly5ejXr5+5SwEgOjqaxYsX88knn2Bra8v58+cZOHAgBw4cyDAAhTAn2WMQwkjs7e2xtLSkY8eO6HQ6dDodc+fOlVAQOZ7sMQghhEhFTj4LIYRIRYJBCCFEKrniHMOJEyeeu5Ijv4qPjzdc2ZPfSVs8I23xjLTFM/Hx8VSrVi3Ty+WKYNBoNFSsWNHcZeQIwcHB0hb/T9riGWmLZ6QtngkODs7ScnIoSQghRCoSDEIIIVKRYBBCCJGKBIMQQohUJBiEEEKkIsEghBAiFaMFw+nTp1/YA+WePXvo0KEDXbp0YcOGDcbavBBCiCwyyn0MX331FZs3b37uprTExESmTp3Kxo0bsbW1pVu3bjRp0oQiRYoYowwhRF6jFMTeg/CLkPTkhbMUCLsBNtdMW1cOlJCgB9yytKxRgqFUqVIsWLDAMNDLU5cvX6ZUqVKGUbpq1KjBsWPHaNmypTHKEELkVkpB1C0IvwAP//UvPjLdRUsBHDNFkTnXvAO1WHHUkw17PsnS8kYJhhYtWqQaZvCp6OhoHBwcDI8LFChgGLEpPXq9Pst38OU1cXFx0hb/T9rimZzWFgXuH8A+bD+Quc6btUmxWEdfxir6MhbJsS+cJ1lXkHh7N/SW9i+crk/Wo7XI36dPy1csxIXNr2V5eZN2iWFvb09MTIzhcUxMTKqgSItWq5Vb3P+f3O7/jLTFMzmmLWIfwN5P4eKaV1+XbRFwqZT6n3NFLAq8jl06Y1rkmLYwoZs3HxEUFMqHH/4HgJbApQERxMXdzdL6TBoM7u7uXL9+ncjISOzs7Dh27FiOGW1LiHzvwXnYNxwi/sr6Op7ch4Qo0NlCjc/ArmjmltfZgLMHOFcEOzn3mJGkJD3z5x9l3Li9xMQkUrnyazRokDL+eZkyhQgOzsHBsGXLFmJjY+nSpQujRo2iX79+KKXo0KEDRYtm8o0jhMheyYnw53Q4MhGSE159fSWbwLtfgZP7q69LpOno0VsMHBjE6dP3AOjQoSJuboWyZd1GC4YSJUoYLkdt3bq14fl33nmHd955x1ibFUJk1r4RcHJ+yu9v/RdqDgdtFj8atFbgUAJk+FKjiYh4wujRu1m27DhKQenSTixc2BIfn/LZto1c0e22EPnSveNwa/9Lzep87x7EZnHv+/qvKT+bLoJqH2VtHcJkxo/fx9Klx9HptIwYUQc/v0bY2Vlm6zYkGITIaRJj4MBoOLmAl72qpyjAq16U5FzhFVcgjCUpSY9Ol3Kl1dixDbl6NZLJk9+hcuWsX3mUHgkGkXscnQY39xgeloyJgfMFzFiQkYRfhKiboLGASj3B2inDRR6Gh+Pi7Jz1bdq/ASUaZn15YRRxcUlMn36Qn34K4ejR/lhZWVC4sB0//9zVqNuVYBBZp08y7bYOjuaf36DtAR6YrgSTKlIVWvwPilZ/qdnDgoNxyWeXaOZ1u3df4cMPt/LXX+EA7NhxidatPUyybQkGkTUHx8DRKabfrkYL7bcDcOPGDUqVKmX6GoxNZwPF6oBF9h43FrnDvXvRDB++k++/PwtAxYqFWbLEh0aNSpusBgkGAdF/w91M9iEQ8v8dIGq0gAmvQCnbFkq/C0DMk2AoLd+SRd6xevUZPvlkO5GRcdjY6Bg3riHDh9fFysrCpHVIMORn+uSUE5wHx0DSi7sfyFCPYy99uEMIkT69XhEZGYeXV1kWLfLOtvsSMkuCIb85sxzunUj5PewE3P0z5ffiDV7qJGcqTm5QpEq2lidEfhIdncDhwzdp3jzlZsCePavwxhsONG1aBo0Z7wWRYMhPnjyEXwemfs7+DWi2FNxbv3gZIYRR/PTTRT75ZDv378dw7txHlC3rjEajoVmzrHWVnZ0kGPKDC6vh93GQFJfy2KogNJye0p+NexuwcTJreULkJ9evRzJkyC9s3hwCQM2abxAfb8Ir/F6CBENeFnMXnjyAM8vg0dVnzxevD1UHma8uIfKhxMRk5s49gr//PmJjE3FwsGLKlKZ8+GFNLHJYN+ESDHnVvZPwfU1Q+mfPtfwOSjRK6ctGCGFSQ4ZsZ+nS4wB07vwmc+a04I03Mh52wBwkGMzp/lm4uj1TiziHhUHUS9wG/+BMSihYOYBDKbAvnnLYyLpgFosVQryKoUNrs2/fdWbPboGXV1lzl5MuCQZz2t4T7p/O1CJFAUIysUC59uD1baa2IYR4NUopVq8+w7Ztl1izpj0ajQYPj8KcO/cRWm3O73lWgsGc4h+l/Hzrvy99qejD8Ie4OLu83PotLKFy36zVJoTIkpCQB3z44Vb27r0GpFyC6u1dDiBXhAJIMOQMtb4AxzIvNav0iSNEzvTkSSJTpx5k+vRDJCQk4+Jiy6xZ79KyZc4+bPQiEgxCCPGKdu26wqBBQVy+HAFAv37VmT69GS4udmauLGskGIQQ4hX9/vtNLl+O4M03i7B0aSvq18/dnTtKMAghRCYlJ+u5dCkcD4/CAPj61qNwYTv69/c0eYd3xiDBYCqnFqcM1fhPT+6bpxYhRJadPHmHQYO2cuVKBCEhg3F2tsXaWsdHH/3H3KVlGwkGU4h9ALs/fvE0rQ6sHE1bjxAi06Ki4hk3bi/z5/+BXq8oXtyBy5fDcXYubu7Ssp0EgynoE1J+WhWExrNTT3OpBLavMCSjEMKolFIEBgbz6ae/cPt2FFqthmHDajN+fGMcHKzNXZ5RSDCYkmUBeKufuasQQmTC0KG/MH/+HwD85z9vsGxZK6pXL2bmqowrZ/XcJIQQOUy7dhVxdLRm0SJvDh/ul+dDAWSPQQghUjl48AZ7917Fz68RAI0bl+bGjWEULJg3Dxu9iASDEEIADx/G4uu7i6+/PglA06Zu1K1bEiBfhQJIMGS/J+Gwsz/E3nv2XHKC+eoRQqRLKcV3351mxIhfefAgFktLLaNG1ad69dfNXZrZSDBkt5t74NKPL55WsLRJSxFCpC84+D4ffriVffuuA9CkSWkWL/ahQoXCZq7MvCQYstvTgXFKNoF6E1NPe62aycsRQqRt9uzD7Nt3nSJF7Jg9uwU9eryFRpM7ekA1JgkGY7EtDMXrmbsKIcS/PHoUh6OjDQBTpzajQAErxo1rhLOzrZkryznkclUhRL7w999RdOmykdq1vyYhIRmAwoXtmDvXS0LhX2SPIbOUgqOTISL0xdMfXTNpOUKI9CUn61m8+E/GjNlDVFQCdnaWnDhxh9q1ZezztEgwZFZ4CBzyy3g+2yLGr0UIka7jx/9m4MAgjh+/A8B773mwYEFLSpWS/snSI8GQWU/7PbIvAfUnv3geCyso09J0NQkhnuPv/xsTJ+5Hr1eULFmQBQta0qZNBXOXlSsYJRj0ej3+/v6EhIRgZWXFpEmTcHV1NUzfvHkz//vf/9BqtXTo0IHu3bsbowzjsikEb/YydxVCiDS4uRVCo4Hhw+vg798Ye3src5eUaxglGHbt2kVCQgLr16/n1KlTTJs2jSVLlhimz5gxg6CgIOzs7PDx8cHHxwdHR9m1E0Jk3ZUrEfz5522qVEkZKKdnzyrUqlXcMJiOeHlGCYbjx4/ToEEDAKpVq8a5c+dSTffw8CAqKgqdTodSKsPrhvV6PcHBwcYo9aVZRf1Fwbu/oou/TyEgLj6eq2aoKS4uzuxtkVNIWzyTn9siISGZ//0vhKVLg1FKsWFDY+BZWwQHy4BYmWWUYIiOjsbe3t7w2MLCgqSkJHS6lM2VK1eODh06YGtrS/PmzSlYsGC669NqtVSsWNEYpb68DR/Bzd8MD20cCpulpuDgYPO3RQ4hbfFMfm2L/fuvM2hQEMHBDwDo0eMtXFzs82VbvEhWvywY5T4Ge3t7YmJiDI/1er0hFC5evMhvv/3G7t272bNnD+Hh4Wzfvt0YZby6xzdgkxd8/zbcOZry3Fv/hTr+0GypWUsTIj978CCWPn1+plGjbwkOfkC5cs7s2tWT1avb4+JiY+7ycj2j7DF4enqyd+9evL29OXXqFOXLlzdMc3BwwMbGBmtraywsLHB2dubx48fGKCPr9MkQFwEhG+DajmfPay2h3gQokH871xIiJxg0KIhNm4KxtrZg9OgGfP55PWxs5CLL7GKUlmzevDmHDh2ia9euKKWYMmUKW7ZsITY2li5dutClSxe6d++OpaUlpUqVol27dsYoI2uUgu//A2Ennz1Xrj38xxccSkooCGEmer1Cq005Hzl58js8eZLE3LktKFfOxcyV5T1GCQatVsuECRNSPefu7m74vVu3bnTr1s0Ym351yQnPQsHGBXS28FZ/KPa2eesSIp+KjU1k4sR9nDp1j23buqPRaPDwKMzWrbnwMvdcQva90mJhBR8/MHcVQuRrW7eGMnjwdq5di0SjgT/+uE2tWtKVhbFJMDwVdhqOz4akJ+auRIh879atx3z66S8EBqZcVVO1alGWLm0loWAiEgxPnZgLF7579tj2NbOVIkR+tnjxn/j67iI6OoECBSyZOLEJn3xSC51OOoM2FQmGp54Ov1n1IyheX84pCGEmDx7EEh2dQLt2FZg3z4uSJaVXBFOTYPi34nWhYg49MS5EHhQZGcfFiw8M3WD7+tbj7beL4+VV1syV5V+ybyaEMAulFOvWnaNixUW8995awsNTzu9ZW+skFMxMgkEIYXKXLoXj5fU93bpt4u7daMqVc+HRozhzlyX+nxxKEkKYTHx8EjNmHGLy5APExydTqJANM2Y0p2/f6oab14T5STAIIUymS5eN/PxzCAC9elUlIKA5r71WwMxViX+TYBBCmMzQobUJCXnI4sXeNGlSxtzliDRIMAghjEKvV3zzzUmCg+8za1YLABo3Ls25cx9iYSGnN3MyCQYhRLY7e/YegwZt5fffbwIph42qVk3pgFJCIefLMBiio6P56quvuH//Po0bN8bDwyPV+M1CCPFUTEwC48fvY/bswyQnK15/3Z65c1tQpUpRc5cmMiHD6B49ejQlS5bk2rVrFC5cmDFjxpiiLiFELrNlSwiVKi0mIOB39HrFxx//h4sXP6ZLl8oZDt8rcpYMgyEyMpKOHTui0+nw9PREKWWKuoQQucxPP13kxo1HVK/+OkeP9mfhQm8cHWU0tdzopc4xXL58GYC7d++i1crxQSEEJCXpuX37Ma6uTgBMn96c6tWLMWhQTenwLpfL8K83duxYRo8ezYULFxgyZAhffPGFKeoSQuRgR47combN5Xh5fU9CQjIAhQvbMXjw2xIKeUCGewy3b99m/fr1hsfbtm2jUqVKRi1KCJEzRUQ8YfTo3SxbdhyloHRpJ65di6R8eRleMy9JMxj27t3LiRMn2Lp1KydPpgx1qdfr2b17N97e3iYrUAhhfkop1q49x7BhOwgLi0Gn0zJyZF3Gjm2InZ2lucsT2SzNYKhQoQKRkZFYW1tTpkzKHYoajQYfHx+TFSeEyBl69Ahk7dpzADRoUIolS3x4800ZzCqvSjMYihUrRrt27WjTpk2qE85hYWEmKUwIkXN4eZVl587LBAQ0p3fvatLhXR6X4TmGhQsXsmbNGhITE4mLi6N06dJs3brVFLUJIcxk164rXL4czsCBNQHo2bMKrVqVx9nZ1syVCVPI8PKB/fv3s3//flq3bs22bdsoWlTuYBQir7p3L5oePQJp3nwVn376C5cvhwMph5ElFPKPDPcYnJycsLKyIiYmBldXV548eWKKuoQQJqTXK5YvP86oUbt49CgeGxsd48Y1lPGW86kMg+H1119n48aN2NraMmvWLKKjo01RlxDCRE6fvsvAgUEcPXobgJYty7JwoTduboXMXJkwlwyDYcKECdy5cwcvLy9+/PFH5s6da4KyhBCm8vnnuzh69DZvvOHAvHledOhQUfo2yufSPMeQlJTEzp07+eOPPyhevDj29vZ4eXmxYMECU9YnhMhmSiliYhIMj+fP92Lo0FoEB39Mx46VJBRE2nsMI0aMwMLCgvv373Pp0iVKlCjBmDFj6NWrlynrE0Jko+vXI/nkk+3ExCSya1dPNBoNHh6FmTPHy9yliRwkzWC4ceMGgYGBJCQk0KFDBywtLfnuu+9wd3c3ZX3G9zAYom5BzB1zVyKE0SQmJjNnzhHGj99HbGwiDg5W/PVXuHRlIV4ozWCwt7cHwMrKCr1ezzfffIOTk5Op6jKN8BD49l/9PmkszFOLEEZy6NANBg3ayrlzKTendunyJrNnt+CNNxzMXJnIqV6q220XF5e8FwoA0SlXYWDtBEVrgt1r4PquWUsSIjt98sk2Fi78EwA3t0IsWuSNl1dZM1clcro0g+HSpUsMHz4cpZTh96dmzZplkuJM5rXq0OlXc1chRLYrUqQAlpZafH3rMXp0A2xtpcM7kbE0g+Gfl6V27drVFLUIIV7RxYsPuHHjEe++m3Iu0Ne3Hp07v0mFCoXNXJnITdIMhrfffjvLK9Xr9fj7+xMSEoKVlRWTJk3C1dXVMP3MmTNMmzYNpRRFihQhICAAa2vrLG9PiPwuLi4JP789TJ9+CCcnGy5eHIyzsy3W1joJBZFpL3WOIbN27dpFQkIC69ev59SpU0ybNo0lS5YAKddQ+/n5MX/+fFxdXfnhhx+4ffs2bm5uxihFiDxv587L9O+/k5s3U3oleO89D+RWBPEqjBIMx48fp0GDBgBUq1aNc+fOGaZdvXoVJycnVq5cSWhoKI0aNZJQECIL7tyJYtiwHaxffx6AN98swtKlrahfv5SZKxO5XYbBcO/ePQICAoiIiKBFixZ4eHhQtWrVdJeJjo42XO4KYGFhQVJSEjqdjoiICE6ePImfnx+urq4MGjSIypUrU6dOnTTXp9frCQ4OzsTLejl2D67jCsTExnLDCOs3hri4OKO0RW6U39uiW7fdnD79EBsbCwYMKE+/fm9iaRmTr9sE5H2RHTIMBj8/P/r06cPixYupWbMmo0aNYsOGDekuY29vT0xMjOGxXq9Hp0vZlJOTE66urpQtm3LJXIMGDTh37ly6waDVaqlYseJLvaBMuXEH/oACdnbGWb8RBAcH55pajS0/toVSytBlxbx5tsyceZgFC1ry5MmdfNcWacmP74u0ZDUgMxyPIT4+njp16qDRaHBzc3upk8Senp7s378fgFOnTlG+fHnDtJIlSxITE8P169cBOHbsGOXKlctS8ULkF1FR8Qwb9gsDBwYZnmvUqDRbtnSjdGkn8xUm8qQM9xisrKw4cOAAer2eU6dOYWVlleFKmzdvzqFDh+jatStKKaZMmcKWLVuIjY2lS5cuTJ482XCPRPXq1WncuHF2vJbMUQr+/t302xUiE5RSBAYG8+mnv3D7dhQ6nZbRoxtIGAijyjAYJk6cyPTp04mIiOCbb77B398/w5VqtVomTJiQ6rl/9rFUp04dNm7cmPlqs8uja7Dzv3BjV8rjN+qarxYh0nD1agSDB29n27a/AHj77eIsXeojoSCMLsNg2LFjB/7+/jg65pGRnGLDYF39lO4wbFzgnQVQQW7gEzmHUooZMw4xfvw+njxJwtHRmqlTmzJgQA0sLDI8+ivEK8swGJKSkujTpw9lypShc+fO1KpVyxR1GYc+Gbb2SAmFYnWg7U8p/SMJkYNoNBpCQx/y5EkS3bpVZvbsFrz+un3GCwqRTTL8+tGvXz8CAwPp3bs3a9as4d13c3Enc4cnpBw+si0CrX+QUBA5xoMHsYbeTwGmT2/Ozp3vs2ZNBwkFYXIZ7jHExcWxY8cOfvrpJ5RSDBkyxBR1Zb/bh+DIREADPmvBobi5KxICpRQrV55mxIidFClSgNOnB2FlZUHhwnY0b57Hxj4RuUaGwfDee+/RokUL/P39U/V3lOvc2A0oqDIQXJuauxohCA6+z6BBW9m/P+XS7apVXyci4glFi8oegjCvNIPh6Z3KP/74I5aWKV31JiSkjBP7Mpes5lhy+EiYWWxsIpMn7ycg4HcSE/UUKWLH7Nkt6NHjLRlvWeQIaQaDr68vs2bNonXr1mg0GpRSQMqJsd27d5usQCHyEqUU77yzkqNHUwaJGjiwBlOnNqVQIVszVybEM2kGw9PBeObOnUuVKlUMzx89etT4VQmRR2k0Gj766D/ExiaybFkr6tQpae6ShHhOmsFw7NgxLl26xLfffkufPn2AlD6Pvv/+e4KCgtJaTAjxD8nJehYv/pPERD2ffZbSH1jPnlXo1q0ylpYyvrjImdIMhoIFC/LgwQMSEhK4f/8+kPJtZ+TIkSYrTojc7Nixvxk0KIjjx+9gbW1B166VeeMNBzQajYSCyNHSDIby5ctTvnx5OnfuzGuvyQlbIV7Wo0dxjB27h0WL/kQpKFmyIAsWtOSNNxzMXZoQLyXNYBgyZAjz58+nffv2z007ePCgUYsSIjdSSvHDDxcYOvQX7tyJxsJCw7Bhtfnyy8bY2+fiK/lEvpNmMMyfPx+QEBAiM5YtO86dO9HUrl2CpUt9qFr1dXOXJESmZdglxp9//sn+/fvZt28fzZo1Y8uWLaaoS4hcIT4+iXv3UsZa1mg0LF7szdKlPhw61FdCQeRaGQZDQEAApUuX5rvvvmPt2rWsW7fOFHUJkePt23eNatWW0b17oOE+Hw+PwgwcWBOtVm5UE7lXhl1iWFtb4+Ligk6no0iRIoa7n4XIr+7fj2HkyF9ZufI0kHJJ6r17MdLZncgzMtxjsLe3p0+fPrRs2ZLvv/+eYsWKmaIuIXIcvV7x9dcnqFBhEStXnsba2oLx4xtz5syHEgoiT8lwj2HevHncuHGDsmXL8tdff9GpUydT1CVEjqKUokWL1ezadQWAZs3cWLzYm3LlXMxcmRDZL8M9hvDwcObPn4+Pjw9z584lLCwso0WEyHM0Gg0NGpSiaNECrFnTnp0735dQEHlWhsEwduxY2rRpw9q1a2nXrh1jxowxRV1CmN3WraH89NNFw2Nf33pcvDiYbt2kF1SRt2UYDPHx8TRt2pSCBQvSrFkzkpKSTFGXEGZz69ZjOnTYQKtWa/nvf7cQHv4EAGtrHU5ONmauTgjjyzAYkpOTCQkJASAkJES+KYk8KylJz5w5h6lYcRGBgcEUKGDJ6NH1KVjQ2tylCWFSGZ58Hjt2LKNHj+b+/fu89tprTJo0yRR1CWFSf/xxm4EDgzh16i4A7dpVYN48L0qWdDRzZUKYXrrBEB0dTZkyZdi0aZOp6hHC5PR6RZ8+P3Phwn1KlXJk4cKWtG7tYe6yhDCbNA8lrV69mvfee482bdpw4MABU9YkhNEppYiLSzlfptVqWLTIm88/r8uFCx9JKIh8L809hqCgIH755Reio6P5/PPPadCggSnrEsJoLl0K56OPtlKyZEG+/roNAI0bl6Zx49LmLUyIHCLNPQYrKyusrKxwdnYmMTHRlDUJYRTx8UlMmLCPypUX8+uvV/jppxAePow1d1lC5DgZnnwGDB2E5Vr6ZIgINXcVwoz27LnKhx9uJTT0IQC9e1clIKA5Li52Zq5MiJwnzWC4dOkSw4cPRyll+P2pWbNmmaS4bPHwAuzoC3eOpjx2qWTeeoRJJSfr6dPnZ1atOgOAh4cLS5e2ksNGQqQjzWCYO3eu4feuXbuaopbslxgDa2pDQhTYF4fmy8DNx9xVCROysNCi02mxsdExdmwDRoyoi7X1S+0oC5Fvpfk/5O233zZlHcYRF5kSCjaF4IPzYC3XpOcHZ8/eIy4uif/8pzgAAQHNGTOmAe7uzmauTIjcIcM7n/MEna2EQj4QE5PAyJE7qV59Gb16/URCQjIALi52EgpCZELe3Ke+shX+mJ5yKEnkC5s3h/DJJ9u5ceMRGg00a1aGxMRkrKwszF2aELlOhsFw7949AgICiIiIoEWLFnh4eFC1atV0l9Hr9fj7+xMSEoKVlRWTJk3C1dX1ufn8/PxwdHRkxIgRWX8FL3JyIdz+x015BUtn7/pFjnHjxiOGDNnOzz+n9Ofl6VmMZctaUbPmG2auTIjcK8NDSX5+fnTo0IGEhARq1qzJ5MmTM1zprl27SEhIYP369QwfPpxp06Y9N8+6desIDTXSJaRKn/Kz8Wzoegg67jTOdoRZJSfradz4W37+OQQHByvmzfPi6NH+EgpCvKKX6na7Tp06aDQa3NzcsLbOuKfJ48ePG+6UrlatGufOnUs1/eTJk5w+fZouXbpkseyX5FIJitcFywLG3Y4wqaf31VhYaPH3b0zHjpUIDv6YIUNqodPlj9NmQhhThoeSrKysOHDgAHq9nlOnTmFlZZXhSqOjo7G3fzYGroWFBUlJSeh0OsLCwli4cCELFy5k+/btL1WkXq8nODj4peYFKBkTgz1w48YNYp68/HK5QVxcXKbaIi959CiBOXPO8PrrdgwaVIm4uDhq1rTmP/+pzOPHt3n8+La5SzSb/Py++Ddpi1eXYTBMnDiR6dOnExERwTfffIO/v3+GK7W3tycm5tmJX71ej06XsqlffvmFiIgIBgwYwP3794mLi8PNzY327dunuT6tVkvFihVf4uX8v/MF4AGUKlUKSmdiuVwgODg4c22RByilWLPmLJ999ithYTE4OFgxYYIPf/99Nd+1RVry4/siLdIWz2Q1IDMMhtdff505c+ZkaqWenp7s3bsXb29vTp06Rfny5Q3TevXqRa9evQAIDAzkypUr6YaCyN9CQx/y0Udb2b37KgANGpRiyRIfHB1t+PtvMxcnRB6VYTDUr1/f8HtkZCQlS5bM8BBQ8+bNOXToEF27dkUpxZQpU9iyZQuxsbHGP68g8oSkJD2TJu1n6tSDJCQk4+JiS0BAcz74oJqMIiiEkWUYDAcPHjT8fvv2bRYuXJjhSrVaLRMmTEj1nLu7+3PzyZ6CSIuFhYYDB26QkJBM377VmD69OYULS4d3QphCpm5wK168OFeuXDFWLSKfu3cvmri4JFxdndBoNCxd6sOdO9E0bPj8PTBCCOPJMBg+++wzw657WFgYLi4uRi9K5C96vWL58uOMGrWLmjXf4Ndfe6LRaChXzoVy5eT9JoSpZRgM3t7eFCxYEABra2sqV65s9KJE/nHq1F0GDQri6NGUS02trCyIjk7AwSHj+2WEEMaRYTB8/fXXrF271hS1iHwkKiqeL7/8jXnzjqLXK954w4F587zo0KGinFwWwswyDAZHR0dWrlxJmTJl0GpT7ir955VKQmRWQkIynp7LuXQpHK1Ww6ef1mLChCYULCh7CULkBBkGQ6FChbh48SIXL140PCfBIF6FlZUFPXtWYcuWUJYu9aFGDenbSIicJM1gGDp0KHPnzmXq1KmmrEfkQYmJycyZc4RSpRzp2jXlHNWoUfUZM6YBFhbSt5EQOU2awRAeHm7KOkQedejQDQYN2sq5c2EUKWJHq1blsbe3knEShMjB0gyGmzdvMnv27BdO++yzz4xWkMgbwsOf4Ov7KytWnATAza0Qixd7Y2+fcSeMQgjzSjMYbGxsKFOmjClrEXmAUopVq84wfPhOHjyIxdJSi69vPUaPboCtraW5yxNCvIQ0g6Fw4cK0a9fOlLWIPCAxUc/UqQd58CCWRo1cWbLEh4oVi5i7LCFEJqQZDHIjm3hZT54kkpCQjKOjDVZWFixf3oorVyLo1auq3JMgRC6U5iUhvr6+pqxD5FI7dlyicuUlfPbZDsNzDRq40ru39IIqRG6VqU70hHjqzp0ohg3bwfr15wEoUMCS2NhE7OzkPIIQuZ1cRC4yJTlZz8KFf1ChwiLWrz+Pra2O6dObcfz4AAkFIfII2WMQLy0uLomGDf/Hn3+mDJ3WqlV5FixoSenSTuYtTAiRrSQYxEuzsdFRufJr3LkTzfz5XrRtW0HOIwiRB0kwiDQppQgMDKZoUXvq1y8FwOzZLbCw0Ei32ELkYRIM4oWuXo1g8ODtbNv2FxUqFObUqYFYW+twcrIxd2lCCCOTYBCpJCQkM2vW70ycuJ8nT5JwdLTm009rodPJdQpC5BcSDMLgwIHrDBq0lQsX7gPQvftbzJr1Lq+/bm/myoQQpiTBIICUu5c7dvyBsLAYypZ1ZvFib5o3dzd3WUIIM5BgyMeUUiQnK3Q6Lba2lsye/S6hoQ/54osG2NjIW0OI/Er+9+dTFy7cZ9CgIJo3d8PPrxEAPXpUMXNVQoicQM4o5jOxsYmMHr2bqlWXcuDADVasOEl8fJK5yxJC5CB5c49B6c1dQY60fftffPzxNq5ejQRg4MAaTJ3aFGvrvPk2EEJkTd76REiIgv2j4MaulMc6W/PWk0PExCTwwQc/s3HjBQCqVCnK0qU+1KlT0syVCSFyorwTDFG3YF19eHwdtDqoNQaK1zd3VTmCnZ0l4eFPKFDAkvHjG/Ppp7XlvgQhRJryTjD8/XtKKDiWgfd+hNeqmrsiszp27G+cnGwoW9YZjUbDihWtsbDQUqqUo7lLE0LkcHnja2NiDFwJSvm9aM18HQqPHsXxySfbePvtrxg0KAilFABlyhSSUBBCvJTcv8dw6yD80hseXQGNBbi/Z+6KzEIpxYYN5xk6dAd370ZjYaHB07MYSUl6LC0tzF2eECIXyf3BsKUjxN6DIlWgxf+gqKe5KzK5y5fD+fjjbezYcRmAOnVKsHRpK6pUKWrmyoQQuVHuD4bE6JSfnfeBjZNZSzGHqKh4atb8isjIOJycbJg+vRn9+3ui1co4CUKIrMn9wfCUNu+8lMxwcLBm2LDaXLoUzsyZ7/LaawXMXZIQIpczyqepXq/H39+fkJAQrKysmDRpEq6urobpQUFBrFy5EgsLC8qXL4+/vz9abd44D25s4eFxfPDBTzRtWoaePVNOsvv5NZSR1IQQ2cYon8a7du0iISGB9evXM3z4cKZNm2aYFhcXx9y5c/nuu+9Yt24d0dHR7N271xhl5Cl6vWLFihN4e//CypWnGTNmD4mJyQASCkKIbGWUPYbjx4/ToEEDAKpVq8a5c+cM06ysrFi3bh22til3JSclJWFtnf4wkXq9nuDg4BdO89Dr0QIXQy6idHnzMMpffz1i/PjjnDjxAIA6dYri5+fJpUuhZq7MvOLi4tJ8X+Q30hbPSFu8OqMEQ3R0NPb2zwZ3sbCwICkpCZ1Oh1arpXDhwgCsWrWK2NhY6tWrl+76tFotFStWfPZE1C04tQiSnoBKBKCCRwWwylsDyjx5koi//2/Mnn2EpCQ9RYsWYMSItxg+/F3ZSwCCg4NTvy/yMWmLZ6QtnslqQBolGOzt7YmJiTE81uv16HS6VI8DAgK4evUqCxYsyPyH3MmF8Of0Z491tmBh9apl5zharYbNm0NJTtbz0Uc1mTy5KXfuXJVQEEIYlVGCwdPTk7179+Lt7c2pU6coX758qunjxo3DysqKxYsXZ+2kc1Jsys+y7aBEw5S7nfNIMNy69Rg7O0ucnW2xttbx7bdtAKhVqwQAd+6YszohRH5glGBo3rw5hw4domvXriilmDJlClu2bCE2NpbKlSuzceNGatasSe/evQHo1asXzZs3z/yGSjYGzyHZW7yZJCXpWbDgKOPG/UbnzpX4+uvUgSCEEKZilGDQarVMmDAh1XPu7s/GD7548aIxNptrHT16i4EDgzh9+h4Ajx7Fk5Sklx5QhRBmkT/vCsshIiPjGD16N0uXHkMpcHV1ZOFCb1q1Kp/xwkIIYSQSDGYSEfGESpUWc/duNDqdluHD6+Dn15ACBfLGuRIhRO4lwWAmhQrZ0rJlWUJDH7JkiQ9vvSUd3gkhcgYJBhOJj09i+vRDNGrkSqNGpQFYuNAbGxuddHgnhMhRJBhMYM+eq3z44VZCQx9SsWJhzp79EAsLLXZ2luYuTQghniPBYERhYTEMH76T1avPAFChQmEWL/bBwkKuNhJC5FwSDEbwtMM7X99dREbGYWOjY+zYBowcWQ8rKxlNTQiRs0kwGMGjR3GMGbOHyMg4WrRwZ9Eib9zdnc1dlhBCvBQJhmwSE5OATqfF2lpHoUK2LF3qQ3KyolOnStK3kRAiV5GD3dlg8+YQKlVazIwZhwzPdehQic6d35RQEELkOhIMr+DGjUe0bbuONm3WcePGI3bsuIxer8xdlhBCvBIJhixITExm5szfqVhxET//HIKDgxXz5nmxb98Hck+CECLXk3MMmfTgQSxNm37HmTMpHd516lSJOXNaULx4QTNXJoQQ2UOCIZNcXGwpXNiOMmWcWLjQG2/vcuYuSQghspUEQwaUUnz//Vnefrs45cu7oNFoWL26HY6ONnLnssgWiYmJ3Lp1i7i4uFdah4xznCI/toWNjQ0lSpTA0jJ7PpMkGNIREvKAjz7axp49V2natAy//toTjUZDsWIO5i5N5CG3bt3CwcGB0qVLZ/kqtidPnmBra5vNleVO+a0tlFI8fPiQW7duUaZMmWxZp5x8foG4uCS+/HIvVaosZc+eq7i42PL++1XMXZbIo+Li4nBxcZFLm0WWaDQaXFxcXmmP899kj+Ffdu26wocfbuXSpXAA+vatxowZzXFxsTNzZSIvk1AQryK73z8SDP9w7140rVqtIT4+mUqVirB0qQ8NGriauywhhDCpfB8Mer1Co0lJ3KJF7ZkwoQl6veKzz+pIh3ciXzh69ChDhw6lbNmyAMTExFCiRAlmzpyJlZUV4eHhTJ8+nb///pvk5GSKFSvGqFGjKFKkCADHjh1j0aJFJCUlERsbS/v27enRo4c5XxIRERHMmTPnubHnTS0uLo6RI0fy8OFDChQowPTp03F2Tt1v2vLly9m6dSv29vb079+fJk2aGKb9+uuv/PLLL8yaNQuAefPm4ePjY/hbGUu+DoZTp+4yaFAQH3/8H3r2rArA55/XM3NVIl8L9IGr2zK9WLqnWst4Q/ut6S5fu3Zt5syZY3g8fPhw9uzZQ4sWLRg8eDB9+/alWbNmAPz+++8MHDiQH374gb///ptJkyaxYsUKChcuTFxcHL169aJkyZI0bNgw068ju8ydO5fu3bubbftPrV27lvLly/PJJ5+wdetWFi9ezNixYw3TQ0JCCAoK4ocffgCga9eu1K5dG1tbWyZNmsTBgwepWLGiYf4+ffowYsQIli9fbtS682UwREXF8+WXvzFv3lH0ekV8fDLvv19FjvMKASQkJBAWFoajoyPnzp3DwcHBEAoAdevWpVSpUvz5558cO3aMtm3bUrhwYSDlssmvv/4aO7vU5+SuXbvG2LFjSUxMxMbGhjlz5jBjxgy8vb1p2LAh+/fvZ9u2bUybNo0mTZrg5uZGqVKlOHjwID///DN2dnasWLECnU5HixYt8PPzIz4+HmtrayZOnEixYsUM24qOjubs2bOMHz8egNWrV7Nz506SkpJwcHBgwYIFBAUFsWnTJvR6PUOGDCEyMpJvv/0WrVZLjRo1GDFiBHfv3sXf35/4+HgiIyP5+OOPU7XD9evXU33IA7Rq1YouXboYHh8/fpz+/fsD0LBhQxYvXpxq/suXL/P2229jbW0NgKurKyEhIVSrVg1PT0+aNWvG+vXrDfMXLFgQa2trLl68SIUKFTL/x31JuSsYnjyE6Nvw5EGWFldK8dNPFxky5Bdu3XqMVqvh009rMWFCEwkFkTNk8M0+La96ieaRI0fo2bMnDx8+RKvV0rlzZ+rUqcO2bdsoWbLkc/OXLFmSv//+m7CwsOc+oBwcnr+ce/r06QwYMICGDRuybds2Lly4kGYtd+7cITAwkEKFChEQEMDOnTtp27Yt27Zt4+uvv2b8+PH07NmTRo0acfjwYWbOnGk41AJw5swZw2Wber0+1Yd+v379OHv2LJDyIbtkyRIiIyPp3r07mzZtwtbWlpEjR3Lo0CE0Gg19+vShVq1anDhxggULFqQKBldXV1atWpVuu0ZHRxvao0CBAkRFRaWa7uHhwfLly4mOjiYxMZGTJ08agsXb25ujR48+t04PDw/++OMPCQYgJRSWl4SkJ/948uU/zB88iKVPn58JCgoFoGbNN1i2rBWensUyWFKIvO/poaSIiAj69u1LiRIlAChatCi3b99+bv7r169Tt25dwsLCuHv3bqppFy9eRCmV6hDI1atXqV69OpDygQcQFBRkmK7Us84nCxUqRKFChQDo1KkT/v7+uLm5Ubp0aQoVKkRoaCjLli1jxYoVKKWeu6krMjLSsAej1WqxtLTks88+w87Ojrt375KUlARgCI8bN24QHh7OgAEDgJRzLDdv3qRGjRosWbKEjRs3otFoDMv9sw0y2mOwt7cnJibGsN6CBVN3nePu7k6PHj3473//i6urK1WrVjW89rQUKVKEe/fupTvPq8o9wRB1KyUULKyhUHmwKQRuPi+9uIODFZcuhVOwoDVTprzDoEE1ZYhNIf7l6bf0Xr168dNPP+Hp6cmDBw/Ys2cP77zzDgD79+/n+vXrvP3225QsWZKPP/4Yb29vnJ2diYmJYdy4cXz88cepgsHd3Z2zZ89St25dNm/ezKNHj7CysuL+/fsAqfYgtNpn/y9Lly6NUooVK1bQrVs3ANzc3Ojbty+enp5cvnyZP//8M9VrcHZ25vHjx0BKSO3atYsffviBJ0+e0L59e0MIPd1OiRIlKFasGN988w2WlpYEBgZSsWJF5s2bR6dOnWjUqBGbNm3ixx9/TLWdl9lj8PT0ZN++fVSpUoX9+/dTo0aNVNPDw8OJiIhg7dq1REVF0bdvX8qVS7+bnUePHuHi4pLuPK8q9wTDU84e0Ov0S8166NANKlQojIuLHdbWOtat68BrrxWQO5eFSEfZsmXp2bMnkyZNYv78+SxdupQpU6awbNkyAF5//XWWL1+OhYUFJUqUYOTIkQwePBgLCwtiYmLo2LEjjRo1SrXOzz//nHHjxrFkyRJsbGwICAjg5s2bjB49mi1btlC6dOk06+nYsSPz5s2jdu3aAPj6+hqO/cfFxTFmzJhU81epUoUFCxYAKR/etra2tG/fHisrK4oUKUJYWFiq+Z2dnfnggw/o2bMnycnJFC9enJYtW+Ll5cXkyZNZtmwZxYoVIyIiItNt2a1bN3x9fenWrRuWlpaGQ17/+9//KFWqFO+88w63bt2iQ4cOWFpa8vnnn2Nhkf7VkGfOnGHYsGGZriVTVC5w4sQJpe6dUmomSq2skuH8Dx7EqP79f1bgr/r1+9kEFZrOhQsXzF1CjpFX2iI7XkdsbGw2VJI3xMbGKj8/P3X+/Hlzl5LtIiIi1MCBA1847UXvo6y+t/LUsRSlFCtXnqJChUWsWHESS0stb7zhkOr4pRAi7/v0009Zs2aNucvIdt9++63x9xbIjYeS0nDx4gMGDQpi377rADRuXJolS3yoUKGwmSsTQpiai4sLkyZNMncZ2W7o0KEm2U6eCIZbtx5TtepSEhKSKVzYjlmz3qVnT7kvQeQeSil5v4osy+6jInkiGEqUKEjPnlXQajVMm9YMZ+f80+WuyP1sbGx4+PCh9LAqskT9f7fbNjY22bbO3BMM+gTDr3fuRDFs2A4GDapJ48alAVi+vLWMtyxypRIlSnDr1i3DpZtZkZiYmG2DtOR2+bEtng7Uk11yRTBoEx/DT91I1mtYsrcSYz5ZxOPH8Vy6FM6ff/4XjUYjoSByLUtLy1ceYCU4ODjVfQP5mbTFqzNKMOj1evz9/QkJCcHKyopJkybh6vqs++o9e/awaNEidDodHTp0oHPnzumuzyr2BidCYODmoRy74gjE07p1eRYsaCm73kIIkc2MEgy7du0iISGB9evXc+rUKaZNm8aSJUuAlN28qVOnsnHjRmxtbenWrRtNmjQxdOH7IjciHPGePxC9PuV8woIFLWnTxkNCQQghjMAo9zEcP36cBg0aAFCtWjXOnTtnmHb58mVKlSqFo6MjVlZW1KhRg2PHjqW7vvBYWzQaDZ99Vpvg4I9p27aChIIQQhiJUfYYoqOjsbe3Nzy2sLAgKSkJnU6XqrdBSOlxMDo6Ot31VXyzCGfPdgLg5s3Lxig5VwkODjZ3CTmGtMUz0hbPSFukiI+Pz9JyRgmGf/YoCCnnHHQ63QunxcTEvLCb3n+qVq2aMcoUQgjxAkY5lOTp6cn+/fsBOHXqFOXLlzdMc3d35/r160RGRpKQkMCxY8cM3fEKIYQwP40yQkdCT69KCg0NRSnFlClTuHDhArGxsXTp0sVwVZJSig4dOph9fFghhBDPGCUYhBBC5F55qndVIYQQr06CQQghRCoSDEIIIVLJUcGg1+sZN24cXbp0oWfPnly/fj3V9D179tChQwe6dOnChg0bzFSlaWTUFkFBQXTq1ImuXbsybtw49Hq9mSo1roza4Sk/Pz9mzpxp4upMK6O2OHPmDN27d6dbt24MGTIky9ew5wYZtcXmzZtp164dHTp0yJMD9rzI6dOn6dmz53PPZ+lzM0vjvhnJjh07lK+vr1JKqZMnT6pBgwYZpiUkJKhmzZqpyMhIFR8fr9q3b6/CwsLMVarRpdcWT548UU2bNjUM5zhs2DC1a9cus9RpbOm1w1Nr165VnTt3VgEBAaYuz6TSawu9Xq/ee+89de3aNaWUUhs2bFCXL182S52mkNH7ol69eioiIkLFx8cbPjfysuXLl6tWrVqpTp06pXo+q5+bOWqPIbu70sjN0msLKysr1q1bh61tyrgTSUlJWFtbm6VOY0uvHQBOnjzJ6dOn6dKliznKM6n02uLq1as4OTmxcuVK3n//fSIjI3FzczNXqUaX0fvCw8ODqKgoEhIS8sUgSKVKlWLBggXPPZ/Vz80cFQxpdaXxdFpmu9LIzdJrC61WS+HCKUOWrlq1itjYWOrVq2eWOo0tvXYICwtj4cKFjBs3zlzlmVR6bREREcHJkyfp3r07//vf/zhy5AiHDx82V6lGl15bAJQrV44OHTrg4+ND48aNKViwoDnKNJkWLVoYepf4p6x+buaoYMjurjRys/Ta4unj6dOnc+jQIRYsWJBnvxGl1w6//PILERERDBgwgOXLlxMUFERgYKC5SjW69NrCyckJV1dXypYti6WlJQ0aNHjuW3Rekl5bXLx4kd9++43du3ezZ88ewsPD2b59u7lKNausfm7mqGCQrjSeSa8tAMaNG0d8fDyLFy82HFLKi9Jrh169ehEYGMiqVasYMGAArVq1on379uYq1ejSa4uSJUsSExNjOAl77NgxypUrZ5Y6TSG9tnBwcMDGxgZra2ssLCxwdnbm8ePH5irVrLL6uZmjRnBr3rw5hw4domvXroauNLZs2WLoSmPUqFH069fP0JVG0aJFzV2y0aTXFpUrV2bjxo3UrFmT3r17Aykfks2bNzdz1dkvo/dEfpJRW0yePJnhw4ejlKJ69eo0btzY3CUbTUZt0aVLF7p3746lpSWlSpWiXbt25i7ZpF71c1O6xBBCCJFKjjqUJIQQwvwkGIQQQqQiwSCEECIVCQYhhBCpSDAIIYRIJUddrioEwK1bt3jvvfd48803Dc/VqlWLwYMHv3D+UaNG4e3tTcOGDbO0vXfeeYdixYqh1WpRSuHk5MS0adNS3VmbkeXLl1O7dm08PDzYvHkznTp1IjAwEEdHR5o2bfrKdSUnJxMbG8vEiRN566230lxm9erVvP/++1nanhBPSTCIHKls2bKsWrXKZNv75ptvDP1NBQQEEBgYSK9evV56+QEDBgApofbDDz/QqVOnbLnZ7p91HThwgIULF7Js2bI051+yZIkEg3hlEgwi10hOTmbcuHHcvXuXiIgIGjZsyNChQw3Tr169yhdffIFOp8PCwoIZM2ZQtGhRZs2axZ9//olSig8++ICWLVumuQ29Xk9UVBRlypQhMTGR0aNHc/PmTZKTk+nTpw/e3t58//33/PTTT2i1Wjw9PfH19TXstezcuZNLly6xcOFClFIULlyYa9euUaFCBdq1a8f9+/cZOHAggYGBmaoL4O+//zb0+fPLL7/w/fffG6bNmzeP9evX8+jRI/z9/RkzZgxffvkl169fR6/XM3ToUGrVqvVqfwCRb0gwiBzp0qVLqfqWnzlzJomJiVSrVo1OnToRHx//XDD8/vvvvPnmm4waNYpjx47x6NEjLl68yK1bt1i3bh3x8fF07tyZevXqPdepWt++fdFqtWg0GqpUqULbtm1Zt24dhQoVIiAggOjoaNq3b0/t2rUJDAzEz8+PatWqsWbNmlSdtw0aNIjQ0FAGDx5s6O2yc+fOjB8/nnbt2vHzzz/Tvn179u3b99J1xcfHExYWRoMGDfD19QXg2rVrLF++HFtbW8aNG8fBgwf58MMPWb16Nf7+/qxZs4ZChQoxZcoUIiIieP/999m6dWt2/5lEHiXBIHKkFx1Kio6O5uzZsxw5cgR7e3sSEhJSTe/YsSNfffUV/fv3x8HBgWHDhhEaGsr58+cNIZOUlJTqm/dT/zxk89Tly5epW7cukNIZmbu7Ozdv3mTq1Kl88803zJw5k2rVqpFR5wHu7u4kJydz+/Zttm3bxrfffsv69eszVdfs2bO5desWLi4uALi4uODr60uBAgW4cuUK1apVS7VcaGgox48f58yZM4b1R0REUKhQoXRrFQLkqiSRiwQGBuLg4MCsWbPo27cvcXFxqT6Ud+/eTY0aNVi5ciVeXl6sWLECNzc3atWqxapVq1i5ciUtW7akRIkSL7U9d3d3Q9/10dHRhIaGUqJECTZs2MD48eNZvXo1wcHBnDx50rCMVqt94Wh6HTt2JCAggLJly1KwYMFM1zV06FDCwsJYs2YNUVFRzJ8/nzlz5jBp0iSsra0N7fD0p5ubGz4+PqxatYqvvvoKLy8vHB0dX+p1CyHBIHKNOnXqsH//frp27Yq/vz+urq6EhYUZpleuXJm5c+fSvXt31q1bx/vvv88777yDnZ0d3bt3N5wMftmrjTp37kxkZCTdunWjV69eDB48GBcXFzw8POjYsSO9evXC2dmZqlWrGpZxcXEhMTGRgICAVOvy8vLi4MGDdOrUCSDTdWm1WiZPnsySJUuIjY3F09OTdu3a0aNHD2xsbAzt4O7uzogRI+jatStXrlzh/fffp2vXrhQvXhytVv67i5cjnegJIYRIRb5CCCGESEWCQQghRCoSDEIIIVKRYBBCCJGKBIMQQohUJBiEEEKkIsEghBAilf8DMIGFJLMOVPgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "lw = 2\n",
    "plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=lw, label='ROC curve (area = %0.2f)' % auc(fpr,tpr))\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristic')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Random Forest classifier seems to have a nice performance. Of course we can decide to increase the recall even more since the problem is to predict the masses, in which case we care about detecting all true malignant masses even at the cost of some false alerts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}