index.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oO7h9f9ehst_"
   },
   "source": [
    "# Intro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "g1rY7WydiJN9"
   },
   "source": [
    "> Name: Jack Risse\n",
    "\n",
    "> School: Flatiron School\n",
    "\n",
    "> Date: 11.12.2019\n",
    "\n",
    "> Instructors: Howard Smith & Amber Yandow\n",
    "\n",
    "> Data: Telecom Customer Churn Prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "h70YO4EyzdYj"
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "agFHlcM9Vj9q"
   },
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'plotly.graph_objects'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-f3f771410d09>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mplotly\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph_objects\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mgo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mplotly\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpress\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'plotly.graph_objects'"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "import numpy as np \n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import plotly.graph_objects as go\n",
    "import plotly.express as px\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import metrics\n",
    "\n",
    "from sklearn.metrics import mean_absolute_error, confusion_matrix\n",
    "from sklearn.metrics import accuracy_score, classification_report, roc_curve\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score\n",
    "from sklearn.linear_model import LogisticRegression as Logit\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.models import load_model\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9Bq7SEa1P6qQ"
   },
   "source": [
    "# Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "colab_type": "code",
    "id": "PmmYj6yzWSDC",
    "outputId": "2c6b8b07-810b-4062-d30b-4d4abab6cf73"
   },
   "outputs": [],
   "source": [
    "tlc = pd.read_csv('/content/Mod4_Project/WA_Fn-UseC_-Telco-Customer-Churn.csv')\n",
    "tlc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "colab_type": "code",
    "id": "HvBO9-swdcgI",
    "outputId": "39153ad4-6fcb-4920-b832-8a6d17d8ad8c"
   },
   "outputs": [],
   "source": [
    "tlc.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WGq2sUbVARYp"
   },
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "thm2aZdCTdMe"
   },
   "outputs": [],
   "source": [
    "def binary(x):\n",
    "  \"\"\"\n",
    "  A simple function to turn categorical data\n",
    "  to binary data meant to be passed in map(),\n",
    "  applymap(), or apply().\n",
    "  \"\"\"\n",
    "\n",
    "  if x == 'Yes':\n",
    "    x = 1\n",
    "  else:\n",
    "    x = 0\n",
    "\n",
    "  return x\n",
    "\n",
    "def thresh_pred(model, x, threshold=.5):\n",
    "  \"\"\"\n",
    "  Utilizes the sklearn predict_proba() to predict \n",
    "  classes with different thresholds.\n",
    "\n",
    "  Parameters\n",
    "  ----------\n",
    "  model : fitted model\n",
    "    Any kind of model from sklearn\n",
    "\n",
    "  x : list, numpy array of ints or floats\n",
    "    The input data that was used to train the model\n",
    "\n",
    "  threshold : float \n",
    "    Needs to be a float between 0,1 because dealing \n",
    "    with probablities\n",
    "\n",
    "  Returns\n",
    "  ---------\n",
    "  a list of the predicted classes for each instance\n",
    "  \"\"\"\n",
    "  predicts = []\n",
    "  for x in model.predict_proba(x):\n",
    "    if x[1] >= threshold:\n",
    "      predicts.append(1)\n",
    "    else:\n",
    "      predicts.append(0)\n",
    "\n",
    "  return predicts\n",
    "\n",
    "def model_report(ytrue, ypred):\n",
    "  \"\"\"\n",
    "  Simple function to print out a few metrics \n",
    "  specific to a classification model.\n",
    "  \"\"\"\n",
    "  print('Accuracy: ', accuracy_score(ytrue, ypred))\n",
    "  print('-----------------')\n",
    "  print('Confusion Matrix: \\n', confusion_matrix(ytrue, ypred))\n",
    "  print('-----------------')\n",
    "  print('Classification Report: \\n', classification_report(ytrue, ypred))\n",
    "  print('-----------------')\n",
    "  print('MAE Score: ', mean_absolute_error(ytrue, ypred))\n",
    "  print('_________________')\n",
    "\n",
    "def plot_roc_curve(y, pred):\n",
    "  \"\"\"\n",
    "  Function that plots the ROC curve and prints\n",
    "  out the area under the curve (AUC) score.\n",
    "\n",
    "  Parameters\n",
    "  ----------\n",
    "  y : numpy array\n",
    "    Data used to train/validate/test your model\n",
    "\n",
    "  pred : numpy array\n",
    "    Predicted values\n",
    "  \"\"\"\n",
    "\n",
    "  fpr, tpr, thresholds = roc_curve(y, pred)\n",
    " \n",
    "  layout = {\"title\": \"ROC Curve Churn Classificaiton\", \n",
    "            \"xaxis\": {\"title\": \"False Positive Rate\"}, \n",
    "            \"yaxis\": {\"title\": \"True Positive Rate\"}\n",
    "            }\n",
    "\n",
    "  fig = go.Figure(layout=layout)\n",
    "\n",
    "  fig.add_trace(go.Scatter(x=fpr,\n",
    "                           y=tpr,\n",
    "                           name='ROC Curve',\n",
    "                           line={'dash': 'solid',\n",
    "                                 'color': 'red',\n",
    "                                 'width': 2}))\n",
    "\n",
    "  fig.add_trace(go.Scatter(x=[0,1], \n",
    "                           y=[0,1],\n",
    "                           name='No name',\n",
    "                           line={'dash': 'dash',\n",
    "                                 'color': 'black',\n",
    "                                 'width': 2}))     \n",
    "  fig.show()\n",
    "\n",
    "  print('AUC Score: ', roc_auc_score(y, pred))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lGAR9aYaAYy-"
   },
   "source": [
    "# EDA & Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = go.Figure()\n",
    "fig.addtrace(go.Bar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9_pqgZJnXZ_o"
   },
   "outputs": [],
   "source": [
    "binary_fts = ['Partner',            # columns with only two categories\n",
    "              'Dependents', \n",
    "              'PhoneService', \n",
    "              'OnlineSecurity', \n",
    "              'DeviceProtection', \n",
    "              'TechSupport', \n",
    "              'StreamingTV', \n",
    "              'StreamingMovies', \n",
    "              'PaperlessBilling', \n",
    "              'Churn',\n",
    "              'gender',\n",
    "              'OnlineBackup']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6ms6zIssWf_S"
   },
   "outputs": [],
   "source": [
    "tlc[binary_fts] = tlc[binary_fts].applymap(binary) # changes columns with two classes to binary data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 309
    },
    "colab_type": "code",
    "id": "pro7GTKyZJtV",
    "outputId": "c2c87333-0881-44dd-87c5-2c287ae7a778"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerID</th>\n",
       "      <th>gender</th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>Partner</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>PhoneService</th>\n",
       "      <th>MultipleLines</th>\n",
       "      <th>InternetService</th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>OnlineBackup</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "      <th>Contract</th>\n",
       "      <th>PaperlessBilling</th>\n",
       "      <th>PaymentMethod</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "      <th>TotalCharges</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7590-VHVEG</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5575-GNVDE</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>One year</td>\n",
       "      <td>0</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3668-QPYBK</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7795-CFOCW</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>One year</td>\n",
       "      <td>0</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9237-HQITU</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   customerID  gender  SeniorCitizen  ...  MonthlyCharges  TotalCharges  Churn\n",
       "0  7590-VHVEG       0              0  ...           29.85         29.85      0\n",
       "1  5575-GNVDE       0              0  ...           56.95        1889.5      0\n",
       "2  3668-QPYBK       0              0  ...           53.85        108.15      1\n",
       "3  7795-CFOCW       0              0  ...           42.30       1840.75      0\n",
       "4  9237-HQITU       0              0  ...           70.70        151.65      1\n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tlc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 459
    },
    "colab_type": "code",
    "id": "wsQZ1XKJZng4",
    "outputId": "6667e724-ff0f-448d-f30b-29702051d50d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7043 entries, 0 to 7042\n",
      "Data columns (total 21 columns):\n",
      "customerID          7043 non-null object\n",
      "gender              7043 non-null int64\n",
      "SeniorCitizen       7043 non-null int64\n",
      "Partner             7043 non-null int64\n",
      "Dependents          7043 non-null int64\n",
      "tenure              7043 non-null int64\n",
      "PhoneService        7043 non-null int64\n",
      "MultipleLines       7043 non-null object\n",
      "InternetService     7043 non-null object\n",
      "OnlineSecurity      7043 non-null int64\n",
      "OnlineBackup        7043 non-null int64\n",
      "DeviceProtection    7043 non-null int64\n",
      "TechSupport         7043 non-null int64\n",
      "StreamingTV         7043 non-null int64\n",
      "StreamingMovies     7043 non-null int64\n",
      "Contract            7043 non-null object\n",
      "PaperlessBilling    7043 non-null int64\n",
      "PaymentMethod       7043 non-null object\n",
      "MonthlyCharges      7043 non-null float64\n",
      "TotalCharges        7043 non-null object\n",
      "Churn               7043 non-null int64\n",
      "dtypes: float64(1), int64(14), object(6)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "tlc.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uLHxorIdybX0"
   },
   "outputs": [],
   "source": [
    "target = tlc['Churn'] # the target variable "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mVM_x0GHhYuY"
   },
   "source": [
    "> Pie chart showing imbalance of the data given at hand which needs to handled at some point in time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "colab_type": "code",
    "id": "GnX5HTcqzg5k",
    "outputId": "479aad82-24a0-4e32-d425-f10c16c9204b"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'go' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-22c3696e3e82>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrace\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPie\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Non_churn'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Churn'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue_counts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhole\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mlayout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLayout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Class Embalance of the Target'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlayout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'go' is not defined"
     ]
    }
   ],
   "source": [
    "trace = go.Pie(labels=['Non_churn', 'Churn'], values=target.value_counts(), hole=.5)\n",
    "layout = go.Layout(title='Class Embalance of the Target')\n",
    "fig = go.Figure(data=trace, layout=layout)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ow7ahtJe1ZcW"
   },
   "outputs": [],
   "source": [
    "mon_churn = tlc['MonthlyCharges'].loc[tlc['Churn']==1] # Monthly Charges for customers who churned\n",
    "mon_no_churn = tlc['MonthlyCharges'].loc[tlc['Churn']==0] # Monthly Charfes for custers who haven't churned"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8Vw5xSNxi8Yd"
   },
   "source": [
    "> A histogram showing tenure and monthly charges of customers that churned and haven't churned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "colab_type": "code",
    "id": "alKR5fYJ6a4B",
    "outputId": "54b3994c-aeca-4856-a004-e147789cc8e5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"7a934241-bb7b-434e-a7b0-5b0be08cef1b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"7a934241-bb7b-434e-a7b0-5b0be08cef1b\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '7a934241-bb7b-434e-a7b0-5b0be08cef1b',\n",
       "                        [{\"name\": \"Tenure\", \"type\": \"histogram\", \"x\": [1, 34, 2, 45, 2, 8, 22, 10, 28, 62, 13, 16, 58, 49, 25, 69, 52, 71, 10, 21, 1, 12, 1, 58, 49, 30, 47, 1, 72, 17, 71, 2, 27, 1, 1, 72, 5, 46, 34, 11, 10, 70, 17, 63, 13, 49, 2, 2, 52, 69, 43, 15, 25, 8, 60, 18, 63, 66, 34, 72, 47, 60, 72, 18, 9, 3, 47, 31, 50, 10, 1, 52, 64, 62, 3, 56, 46, 8, 30, 45, 1, 11, 7, 42, 49, 9, 35, 48, 46, 29, 30, 1, 66, 65, 72, 12, 71, 5, 52, 25, 1, 1, 38, 66, 68, 5, 72, 32, 43, 72, 55, 52, 43, 37, 64, 3, 36, 10, 41, 27, 56, 6, 3, 7, 4, 33, 27, 72, 1, 71, 13, 25, 67, 1, 2, 43, 23, 64, 57, 1, 72, 8, 61, 64, 71, 65, 3, 1, 30, 15, 8, 7, 70, 62, 6, 14, 22, 22, 16, 10, 13, 20, 2, 53, 11, 69, 4, 72, 58, 16, 43, 2, 14, 53, 32, 34, 15, 7, 15, 61, 1, 1, 8, 33, 13, 1, 20, 3, 13, 40, 43, 6, 69, 72, 59, 20, 24, 59, 72, 1, 27, 14, 71, 13, 44, 33, 72, 1, 19, 64, 2, 1, 61, 29, 23, 57, 72, 66, 65, 8, 4, 71, 1, 4, 12, 24, 31, 1, 30, 47, 54, 50, 1, 72, 29, 2, 10, 18, 11, 16, 72, 72, 41, 65, 13, 4, 41, 15, 1, 42, 51, 2, 1, 32, 10, 67, 61, 50, 2, 29, 3, 13, 57, 31, 45, 61, 50, 19, 59, 71, 16, 57, 1, 20, 1, 5, 52, 21, 14, 5, 6, 10, 1, 68, 18, 22, 20, 1, 8, 10, 24, 35, 23, 6, 12, 1, 71, 35, 40, 1, 23, 4, 4, 68, 38, 52, 32, 29, 38, 48, 1, 22, 43, 5, 5, 51, 71, 38, 24, 35, 54, 72, 1, 9, 69, 52, 11, 2, 28, 17, 35, 8, 46, 7, 2, 68, 43, 68, 36, 63, 32, 71, 66, 63, 41, 1, 2, 70, 23, 64, 37, 17, 7, 4, 21, 10, 16, 64, 27, 42, 5, 41, 58, 47, 18, 5, 23, 1, 71, 72, 33, 2, 24, 56, 37, 43, 1, 25, 61, 17, 41, 1, 72, 1, 48, 11, 55, 42, 44, 1, 27, 27, 2, 19, 42, 66, 33, 34, 33, 23, 32, 11, 69, 68, 20, 72, 60, 32, 1, 1, 3, 46, 29, 51, 48, 16, 70, 40, 22, 1, 5, 7, 29, 44, 10, 55, 52, 10, 18, 68, 61, 72, 2, 12, 41, 26, 36, 72, 35, 1, 16, 49, 54, 18, 36, 60, 1, 52, 8, 72, 64, 22, 60, 28, 61, 24, 28, 30, 2, 1, 6, 24, 4, 7, 72, 70, 64, 72, 44, 13, 17, 1, 9, 24, 1, 24, 35, 7, 5, 15, 11, 48, 20, 72, 8, 72, 15, 72, 0, 1, 63, 2, 2, 61, 1, 22, 28, 70, 5, 12, 34, 71, 70, 52, 69, 20, 11, 2, 6, 1, 20, 61, 5, 56, 30, 40, 28, 5, 27, 12, 67, 29, 55, 23, 34, 52, 72, 58, 35, 56, 24, 70, 2, 68, 1, 12, 63, 33, 69, 60, 72, 11, 1, 10, 13, 34, 39, 65, 50, 15, 72, 72, 55, 23, 32, 56, 1, 38, 11, 1, 56, 3, 7, 59, 7, 71, 15, 71, 35, 11, 60, 47, 11, 56, 28, 61, 31, 9, 35, 2, 12, 1, 4, 1, 3, 1, 52, 5, 72, 71, 72, 46, 63, 30, 1, 12, 16, 4, 51, 65, 16, 2, 66, 46, 32, 72, 38, 51, 72, 65, 9, 9, 66, 44, 50, 15, 8, 66, 57, 7, 10, 62, 40, 20, 7, 25, 23, 66, 72, 49, 43, 46, 72, 10, 40, 65, 31, 68, 56, 10, 68, 43, 1, 49, 15, 20, 1, 50, 2, 24, 3, 1, 35, 17, 8, 10, 68, 45, 2, 37, 4, 10, 1, 65, 57, 3, 2, 49, 4, 70, 53, 53, 1, 22, 52, 65, 48, 2, 3, 45, 1, 61, 3, 40, 1, 1, 51, 2, 52, 51, 1, 31, 47, 3, 22, 1, 72, 3, 47, 72, 66, 35, 29, 2, 4, 25, 65, 27, 29, 29, 1, 20, 58, 14, 72, 46, 71, 32, 26, 68, 2, 61, 4, 3, 33, 9, 22, 5, 30, 65, 45, 5, 25, 72, 27, 32, 30, 70, 42, 72, 47, 2, 10, 61, 5, 72, 72, 3, 48, 63, 27, 70, 7, 0, 2, 20, 66, 3, 15, 72, 1, 22, 3, 72, 65, 11, 22, 14, 41, 17, 11, 15, 1, 5, 33, 72, 3, 2, 59, 2, 71, 5, 27, 1, 63, 46, 72, 34, 24, 72, 60, 68, 8, 34, 6, 2, 31, 20, 1, 62, 70, 10, 39, 46, 6, 72, 18, 71, 40, 1, 58, 70, 42, 34, 5, 25, 2, 55, 21, 70, 61, 43, 47, 5, 62, 16, 7, 14, 60, 34, 50, 38, 70, 37, 4, 60, 62, 1, 36, 44, 55, 72, 12, 13, 1, 15, 65, 12, 72, 72, 72, 52, 2, 5, 68, 62, 72, 1, 66, 72, 26, 64, 20, 3, 22, 4, 62, 5, 59, 3, 72, 57, 66, 60, 45, 3, 15, 51, 60, 33, 10, 26, 6, 67, 49, 1, 7, 27, 37, 63, 31, 50, 32, 1, 63, 30, 71, 53, 12, 50, 2, 9, 17, 56, 67, 9, 4, 19, 8, 71, 10, 15, 72, 12, 72, 1, 23, 72, 26, 21, 60, 12, 16, 63, 22, 32, 3, 13, 68, 30, 16, 33, 72, 4, 12, 4, 0, 6, 65, 15, 24, 13, 24, 72, 54, 3, 4, 32, 35, 35, 2, 8, 22, 15, 22, 1, 71, 4, 25, 32, 7, 17, 8, 56, 1, 8, 7, 3, 71, 2, 1, 49, 58, 44, 59, 71, 1, 11, 62, 35, 20, 40, 39, 1, 72, 33, 12, 1, 27, 34, 56, 58, 22, 10, 13, 35, 34, 4, 72, 2, 7, 27, 4, 37, 21, 53, 18, 2, 32, 23, 3, 71, 9, 1, 18, 12, 71, 64, 4, 23, 39, 28, 5, 45, 37, 60, 8, 47, 26, 3, 50, 27, 8, 62, 71, 66, 68, 13, 56, 38, 14, 16, 14, 32, 8, 43, 52, 3, 29, 1, 12, 16, 40, 5, 40, 36, 5, 10, 2, 23, 26, 72, 34, 10, 14, 23, 47, 24, 49, 20, 2, 2, 22, 7, 1, 59, 58, 41, 59, 3, 32, 46, 0, 2, 52, 13, 11, 32, 17, 16, 51, 29, 70, 71, 41, 1, 7, 25, 67, 5, 15, 20, 3, 54, 42, 9, 63, 69, 69, 40, 60, 4, 71, 37, 32, 39, 38, 52, 48, 70, 20, 50, 19, 25, 12, 39, 7, 23, 27, 47, 26, 14, 11, 2, 26, 72, 63, 71, 11, 14, 13, 6, 11, 18, 1, 32, 29, 3, 2, 13, 41, 1, 7, 52, 45, 70, 53, 62, 60, 3, 23, 1, 67, 12, 71, 25, 5, 26, 1, 70, 72, 60, 32, 1, 14, 13, 6, 46, 15, 43, 39, 21, 57, 53, 18, 1, 58, 71, 35, 3, 38, 35, 7, 47, 14, 20, 66, 15, 42, 17, 37, 12, 53, 60, 18, 1, 3, 9, 1, 56, 17, 11, 7, 69, 19, 3, 54, 62, 24, 62, 17, 9, 64, 2, 1, 16, 72, 30, 49, 61, 47, 20, 34, 70, 54, 61, 3, 13, 16, 3, 25, 30, 21, 1, 15, 23, 45, 24, 11, 1, 56, 1, 1, 7, 55, 2, 72, 45, 47, 46, 2, 2, 12, 68, 69, 56, 4, 64, 59, 62, 63, 53, 5, 49, 62, 55, 71, 72, 36, 25, 72, 36, 1, 72, 59, 7, 1, 30, 64, 63, 72, 8, 62, 67, 6, 70, 20, 5, 24, 11, 72, 66, 45, 69, 15, 28, 70, 36, 16, 18, 34, 42, 48, 47, 39, 11, 7, 3, 8, 1, 32, 60, 10, 71, 4, 1, 43, 59, 23, 72, 22, 1, 69, 50, 1, 2, 15, 31, 1, 66, 0, 3, 8, 64, 28, 57, 14, 19, 10, 51, 67, 11, 72, 66, 18, 9, 9, 48, 10, 9, 13, 4, 4, 72, 51, 59, 10, 61, 54, 33, 27, 1, 23, 1, 45, 39, 5, 72, 58, 70, 61, 2, 46, 1, 22, 48, 64, 72, 12, 34, 72, 29, 33, 1, 62, 41, 64, 4, 24, 14, 3, 4, 18, 8, 35, 1, 66, 8, 71, 43, 2, 29, 15, 65, 35, 64, 58, 18, 67, 63, 60, 9, 70, 15, 48, 12, 71, 44, 1, 45, 23, 43, 35, 9, 12, 65, 2, 27, 40, 5, 8, 58, 52, 3, 41, 20, 1, 4, 23, 6, 8, 18, 52, 31, 29, 36, 16, 42, 1, 60, 5, 22, 36, 4, 9, 1, 12, 23, 62, 37, 8, 31, 13, 24, 45, 69, 2, 61, 41, 44, 39, 72, 13, 51, 71, 22, 2, 56, 1, 23, 66, 1, 19, 11, 8, 52, 3, 51, 15, 64, 37, 13, 49, 45, 18, 1, 68, 54, 23, 17, 71, 67, 14, 1, 63, 41, 17, 56, 5, 2, 3, 37, 29, 8, 63, 7, 3, 72, 19, 59, 2, 35, 14, 14, 69, 7, 69, 72, 8, 4, 63, 72, 46, 5, 30, 63, 60, 63, 25, 1, 6, 22, 31, 39, 26, 53, 1, 12, 16, 2, 39, 1, 7, 4, 10, 55, 72, 10, 11, 15, 23, 1, 3, 47, 15, 66, 68, 17, 7, 12, 21, 21, 56, 6, 65, 42, 68, 48, 50, 7, 63, 17, 42, 4, 62, 2, 2, 48, 27, 70, 1, 46, 30, 15, 69, 65, 72, 13, 17, 51, 51, 72, 67, 34, 67, 49, 53, 27, 23, 69, 2, 35, 46, 54, 56, 9, 20, 11, 30, 68, 38, 17, 48, 1, 63, 3, 48, 66, 68, 17, 7, 72, 29, 37, 34, 42, 59, 11, 60, 27, 1, 1, 17, 58, 1, 3, 53, 35, 50, 68, 47, 65, 5, 51, 46, 9, 8, 14, 45, 8, 1, 66, 72, 41, 23, 29, 4, 6, 67, 7, 56, 72, 72, 23, 35, 27, 26, 12, 40, 7, 70, 60, 39, 72, 1, 54, 3, 63, 71, 42, 47, 66, 21, 11, 1, 55, 69, 3, 4, 30, 5, 71, 29, 52, 68, 46, 8, 72, 17, 3, 2, 9, 51, 6, 3, 17, 30, 31, 45, 64, 1, 1, 61, 1, 9, 72, 1, 7, 66, 1, 40, 16, 2, 67, 41, 56, 72, 3, 54, 52, 50, 14, 27, 72, 62, 12, 44, 54, 68, 20, 50, 58, 35, 2, 63, 58, 27, 71, 63, 71, 41, 13, 2, 68, 1, 65, 72, 28, 72, 2, 18, 60, 26, 1, 4, 68, 38, 42, 57, 54, 12, 44, 42, 72, 71, 19, 23, 30, 35, 10, 1, 22, 7, 36, 34, 72, 36, 1, 23, 32, 71, 23, 17, 1, 12, 72, 1, 72, 60, 61, 6, 32, 31, 19, 72, 32, 65, 45, 42, 8, 32, 22, 57, 1, 1, 1, 24, 1, 54, 4, 65, 56, 45, 71, 59, 69, 19, 55, 38, 10, 47, 2, 1, 1, 1, 46, 38, 65, 19, 52, 71, 1, 52, 6, 26, 48, 64, 3, 1, 72, 1, 51, 41, 72, 43, 72, 47, 72, 3, 1, 2, 26, 29, 35, 27, 24, 67, 16, 23, 14, 1, 1, 4, 16, 46, 68, 38, 30, 5, 17, 4, 12, 72, 3, 56, 41, 40, 7, 69, 7, 5, 72, 44, 65, 3, 24, 44, 72, 24, 1, 22, 70, 25, 37, 22, 59, 49, 47, 31, 1, 3, 53, 1, 20, 3, 51, 51, 13, 1, 1, 63, 3, 46, 1, 8, 71, 55, 70, 2, 67, 65, 14, 20, 1, 1, 49, 72, 46, 24, 5, 33, 42, 23, 8, 66, 24, 24, 69, 53, 60, 7, 20, 23, 72, 11, 21, 1, 31, 57, 45, 10, 58, 14, 27, 14, 12, 69, 25, 58, 35, 16, 45, 17, 1, 22, 1, 67, 67, 2, 23, 9, 5, 54, 57, 24, 49, 5, 2, 4, 70, 5, 53, 47, 31, 13, 28, 10, 38, 1, 67, 52, 62, 16, 5, 12, 72, 71, 24, 15, 67, 2, 5, 15, 1, 41, 43, 1, 1, 26, 22, 71, 7, 28, 16, 7, 69, 1, 3, 21, 69, 71, 69, 48, 47, 2, 45, 51, 22, 72, 37, 71, 7, 66, 51, 30, 34, 64, 65, 47, 1, 49, 67, 39, 14, 43, 56, 14, 1, 16, 70, 72, 23, 21, 1, 1, 32, 17, 4, 36, 50, 48, 50, 72, 10, 18, 1, 1, 9, 2, 40, 69, 37, 18, 11, 8, 3, 55, 33, 46, 34, 3, 30, 33, 45, 40, 71, 1, 72, 22, 46, 55, 1, 12, 31, 5, 67, 1, 40, 41, 1, 51, 42, 23, 1, 1, 56, 15, 12, 54, 7, 33, 16, 21, 30, 3, 11, 62, 18, 6, 46, 21, 68, 1, 25, 24, 30, 2, 51, 57, 15, 72, 2, 28, 29, 70, 13, 59, 13, 7, 62, 21, 2, 1, 4, 19, 30, 67, 72, 53, 5, 71, 50, 56, 2, 2, 24, 46, 71, 29, 69, 71, 1, 56, 56, 1, 28, 19, 66, 17, 52, 19, 36, 7, 72, 67, 34, 57, 7, 1, 8, 69, 50, 10, 12, 14, 70, 64, 66, 71, 20, 72, 71, 38, 28, 17, 33, 23, 58, 70, 4, 45, 10, 36, 54, 23, 41, 5, 27, 1, 67, 72, 56, 44, 66, 34, 69, 1, 40, 30, 11, 15, 11, 64, 72, 72, 1, 15, 60, 56, 8, 3, 49, 2, 6, 70, 12, 52, 72, 40, 1, 3, 40, 1, 30, 23, 1, 44, 65, 7, 72, 8, 16, 66, 1, 3, 53, 8, 69, 5, 72, 13, 4, 54, 72, 12, 1, 1, 54, 69, 48, 48, 8, 71, 2, 67, 34, 3, 9, 71, 57, 72, 48, 18, 43, 72, 35, 4, 49, 71, 11, 63, 65, 49, 29, 15, 4, 72, 26, 35, 57, 28, 25, 47, 57, 16, 5, 17, 56, 72, 21, 48, 68, 30, 3, 14, 4, 71, 8, 61, 72, 5, 49, 8, 3, 9, 67, 46, 67, 55, 33, 62, 1, 49, 1, 14, 18, 1, 1, 72, 64, 69, 1, 71, 66, 2, 71, 11, 47, 35, 32, 60, 11, 29, 21, 48, 3, 43, 5, 1, 71, 8, 8, 20, 33, 71, 31, 38, 1, 2, 12, 9, 11, 6, 71, 42, 8, 5, 2, 45, 28, 43, 60, 42, 7, 25, 40, 27, 10, 27, 11, 4, 68, 1, 18, 57, 26, 17, 1, 38, 59, 30, 2, 50, 9, 3, 14, 31, 7, 8, 17, 32, 2, 7, 72, 31, 27, 18, 7, 14, 11, 72, 28, 15, 4, 71, 5, 47, 57, 50, 8, 48, 70, 1, 8, 1, 1, 60, 49, 4, 29, 67, 53, 67, 6, 47, 53, 69, 3, 4, 56, 59, 61, 2, 46, 12, 14, 28, 24, 31, 68, 39, 42, 13, 6, 35, 38, 18, 4, 27, 41, 50, 72, 70, 44, 2, 34, 72, 71, 64, 72, 1, 29, 23, 52, 25, 64, 16, 1, 24, 2, 34, 36, 53, 47, 72, 72, 1, 9, 8, 45, 7, 71, 41, 67, 69, 70, 25, 72, 34, 65, 70, 72, 35, 13, 12, 62, 25, 52, 8, 2, 56, 12, 47, 2, 18, 8, 45, 3, 38, 72, 46, 71, 66, 25, 18, 13, 65, 60, 15, 72, 30, 42, 71, 1, 39, 35, 53, 1, 31, 48, 30, 10, 12, 57, 58, 37, 44, 27, 8, 3, 25, 57, 12, 62, 65, 71, 21, 71, 7, 72, 1, 72, 64, 72, 29, 13, 31, 1, 7, 61, 39, 10, 14, 1, 67, 72, 6, 1, 25, 33, 18, 71, 28, 2, 17, 56, 60, 33, 1, 2, 63, 7, 55, 65, 1, 63, 70, 36, 52, 22, 22, 5, 47, 33, 18, 1, 56, 2, 35, 64, 15, 24, 1, 70, 1, 4, 39, 29, 14, 61, 13, 66, 2, 59, 62, 33, 66, 72, 1, 19, 51, 63, 27, 22, 4, 42, 29, 4, 30, 4, 71, 46, 4, 7, 69, 72, 19, 28, 5, 72, 8, 7, 22, 72, 8, 52, 68, 71, 2, 34, 35, 61, 1, 1, 53, 72, 2, 3, 13, 41, 24, 28, 8, 1, 54, 41, 19, 72, 62, 56, 15, 10, 32, 21, 62, 2, 27, 5, 25, 2, 49, 63, 4, 1, 11, 52, 60, 64, 43, 61, 1, 5, 66, 67, 42, 1, 31, 7, 4, 34, 3, 19, 31, 1, 3, 46, 1, 69, 5, 1, 26, 10, 25, 64, 30, 13, 64, 46, 12, 15, 17, 13, 67, 24, 6, 53, 16, 10, 13, 9, 25, 7, 38, 43, 4, 25, 27, 72, 71, 24, 50, 57, 15, 4, 28, 9, 55, 3, 10, 55, 20, 62, 32, 43, 9, 60, 58, 7, 2, 37, 65, 39, 66, 68, 62, 3, 72, 41, 29, 4, 53, 1, 41, 39, 63, 15, 13, 1, 1, 8, 60, 12, 40, 66, 42, 66, 49, 1, 41, 41, 23, 3, 4, 52, 4, 11, 2, 26, 24, 12, 60, 64, 66, 60, 17, 42, 1, 47, 10, 70, 67, 1, 7, 1, 4, 66, 12, 24, 26, 6, 57, 14, 42, 25, 64, 22, 19, 61, 22, 70, 12, 31, 11, 68, 72, 67, 60, 1, 1, 58, 47, 1, 1, 22, 48, 37, 13, 43, 6, 71, 1, 72, 6, 12, 25, 21, 6, 20, 18, 43, 35, 1, 32, 52, 32, 72, 51, 68, 8, 49, 72, 9, 28, 54, 11, 50, 69, 1, 68, 40, 31, 33, 55, 68, 12, 71, 40, 64, 53, 12, 53, 72, 46, 40, 12, 9, 51, 49, 41, 56, 4, 20, 26, 20, 7, 7, 51, 4, 1, 27, 22, 12, 3, 34, 24, 51, 14, 59, 3, 65, 5, 59, 72, 62, 28, 3, 19, 1, 24, 57, 72, 67, 52, 71, 26, 35, 55, 33, 72, 1, 10, 37, 12, 1, 62, 1, 18, 69, 2, 19, 12, 9, 27, 27, 1, 24, 14, 32, 11, 1, 38, 9, 54, 29, 44, 59, 3, 18, 67, 22, 33, 5, 2, 72, 9, 67, 16, 8, 5, 23, 1, 50, 17, 68, 1, 25, 67, 32, 67, 72, 71, 1, 46, 2, 1, 48, 61, 32, 2, 3, 5, 71, 37, 65, 67, 49, 50, 25, 17, 64, 25, 23, 24, 37, 21, 1, 10, 6, 51, 10, 6, 47, 61, 52, 35, 71, 6, 45, 2, 4, 2, 4, 51, 60, 9, 3, 17, 8, 46, 68, 1, 4, 1, 28, 39, 11, 71, 2, 30, 17, 55, 58, 5, 1, 9, 26, 50, 72, 43, 56, 1, 72, 72, 36, 5, 13, 44, 70, 44, 32, 69, 16, 68, 16, 68, 4, 26, 29, 5, 70, 24, 72, 1, 70, 36, 38, 17, 41, 1, 2, 14, 2, 1, 13, 6, 4, 5, 15, 47, 8, 17, 15, 26, 23, 4, 29, 25, 9, 18, 3, 69, 14, 19, 39, 31, 24, 14, 64, 50, 52, 28, 21, 25, 17, 58, 17, 51, 72, 52, 27, 3, 64, 45, 3, 71, 1, 58, 34, 8, 15, 66, 12, 58, 3, 43, 9, 3, 22, 40, 68, 54, 50, 1, 72, 40, 72, 6, 5, 48, 1, 64, 17, 40, 41, 51, 41, 1, 2, 68, 24, 70, 3, 2, 3, 7, 13, 7, 12, 53, 12, 63, 15, 36, 4, 24, 61, 16, 65, 26, 16, 54, 1, 5, 19, 10, 23, 3, 72, 10, 10, 11, 37, 17, 36, 17, 66, 61, 22, 1, 6, 31, 68, 34, 52, 10, 29, 72, 47, 24, 65, 4, 12, 1, 33, 34, 14, 4, 13, 65, 23, 55, 49, 60, 69, 40, 67, 35, 19, 13, 41, 4, 24, 5, 5, 1, 72, 24, 42, 4, 68, 33, 1, 31, 4, 69, 38, 3, 48, 15, 25, 1, 48, 1, 1, 37, 66, 26, 63, 10, 2, 18, 64, 9, 28, 1, 4, 38, 66, 1, 18, 51, 0, 1, 12, 41, 12, 55, 7, 12, 68, 5, 49, 40, 16, 10, 72, 2, 23, 71, 11, 1, 16, 1, 12, 54, 68, 4, 1, 27, 21, 13, 64, 1, 57, 21, 19, 31, 52, 46, 11, 53, 11, 57, 2, 2, 71, 1, 68, 72, 2, 1, 41, 72, 6, 4, 12, 58, 7, 65, 1, 56, 4, 58, 62, 26, 62, 58, 68, 61, 42, 18, 56, 4, 4, 35, 64, 31, 67, 4, 70, 3, 53, 2, 29, 47, 68, 12, 8, 54, 69, 26, 72, 70, 1, 10, 28, 1, 21, 51, 53, 53, 24, 70, 61, 11, 2, 25, 41, 18, 72, 71, 34, 29, 40, 36, 46, 58, 39, 4, 52, 70, 65, 1, 70, 29, 1, 67, 1, 26, 30, 48, 55, 7, 37, 31, 4, 72, 5, 1, 15, 8, 35, 56, 42, 65, 2, 65, 18, 23, 4, 70, 4, 19, 18, 38, 2, 47, 52, 9, 26, 8, 44, 3, 2, 9, 1, 25, 2, 43, 1, 58, 59, 44, 66, 68, 9, 19, 4, 70, 1, 8, 53, 51, 11, 60, 17, 3, 70, 1, 43, 16, 57, 37, 72, 11, 50, 5, 1, 16, 2, 17, 16, 15, 10, 46, 64, 1, 25, 71, 8, 72, 49, 29, 72, 31, 50, 71, 70, 71, 61, 32, 1, 68, 62, 7, 20, 6, 33, 28, 27, 7, 26, 5, 30, 63, 1, 53, 14, 21, 17, 16, 35, 32, 28, 1, 59, 72, 36, 40, 40, 9, 63, 3, 40, 8, 34, 5, 9, 9, 31, 50, 2, 1, 8, 9, 2, 3, 25, 1, 45, 51, 55, 38, 2, 38, 34, 70, 13, 39, 61, 12, 41, 21, 55, 69, 26, 69, 18, 47, 72, 33, 2, 72, 37, 62, 71, 23, 16, 9, 17, 4, 1, 24, 1, 72, 72, 11, 9, 2, 60, 29, 49, 30, 53, 39, 9, 39, 8, 51, 71, 71, 70, 1, 38, 28, 32, 49, 37, 10, 67, 7, 51, 9, 9, 4, 71, 50, 24, 22, 44, 33, 1, 54, 42, 1, 1, 30, 1, 16, 1, 9, 46, 1, 71, 43, 50, 13, 19, 41, 1, 24, 40, 3, 37, 67, 32, 6, 32, 59, 30, 20, 27, 20, 9, 68, 69, 26, 69, 11, 1, 10, 55, 44, 46, 69, 11, 11, 29, 57, 28, 42, 2, 23, 18, 62, 1, 16, 3, 67, 62, 57, 2, 23, 25, 72, 2, 8, 5, 35, 24, 2, 72, 41, 4, 26, 7, 1, 4, 48, 2, 12, 60, 55, 1, 1, 4, 1, 42, 1, 7, 3, 72, 15, 4, 11, 5, 1, 72, 55, 40, 57, 1, 1, 1, 52, 41, 43, 47, 3, 66, 55, 29, 12, 66, 35, 10, 27, 58, 54, 9, 2, 6, 26, 9, 8, 12, 15, 43, 42, 31, 66, 18, 1, 61, 10, 1, 18, 24, 3, 50, 1, 2, 17, 69, 72, 3, 50, 53, 58, 46, 72, 1, 6, 72, 4, 52, 0, 2, 65, 43, 4, 25, 51, 12, 57, 24, 64, 4, 26, 15, 64, 36, 27, 1, 35, 4, 8, 10, 2, 58, 51, 46, 1, 46, 50, 53, 61, 5, 47, 54, 19, 26, 70, 17, 30, 1, 19, 26, 21, 50, 68, 3, 9, 51, 9, 41, 22, 21, 71, 1, 26, 71, 4, 12, 18, 3, 72, 11, 1, 13, 72, 42, 17, 7, 68, 56, 38, 72, 48, 52, 35, 67, 1, 53, 34, 3, 1, 19, 60, 11, 47, 18, 60, 72, 39, 59, 2, 1, 20, 6, 71, 24, 67, 1, 48, 37, 11, 3, 18, 50, 67, 25, 2, 9, 10, 70, 9, 4, 2, 1, 19, 7, 1, 1, 9, 3, 9, 5, 56, 18, 49, 70, 72, 6, 17, 29, 6, 63, 16, 59, 3, 8, 7, 68, 68, 52, 72, 32, 72, 1, 42, 25, 45, 43, 37, 20, 4, 63, 3, 66, 28, 8, 71, 1, 72, 16, 66, 11, 51, 8, 14, 4, 70, 70, 54, 28, 24, 69, 42, 2, 39, 45, 72, 38, 72, 1, 72, 55, 51, 63, 1, 23, 1, 2, 52, 36, 1, 28, 7, 14, 72, 1, 10, 42, 7, 4, 72, 20, 63, 56, 5, 72, 68, 67, 8, 52, 18, 59, 60, 7, 59, 46, 5, 59, 70, 14, 44, 64, 58, 46, 58, 72, 30, 11, 34, 54, 3, 72, 40, 2, 54, 14, 1, 10, 1, 1, 56, 68, 14, 68, 55, 16, 9, 14, 58, 53, 70, 14, 22, 10, 29, 1, 49, 68, 1, 30, 72, 10, 7, 9, 1, 20, 1, 29, 1, 3, 20, 64, 1, 6, 50, 6, 7, 72, 8, 67, 24, 72, 33, 2, 70, 22, 59, 36, 51, 53, 20, 63, 40, 35, 26, 27, 53, 34, 19, 43, 6, 56, 57, 34, 10, 1, 13, 56, 55, 36, 47, 12, 1, 24, 63, 35, 67, 25, 21, 13, 35, 71, 29, 71, 7, 57, 65, 27, 6, 72, 1, 11, 39, 59, 26, 2, 72, 65, 72, 6, 32, 50, 61, 15, 72, 9, 1, 12, 37, 61, 18, 21, 68, 12, 2, 62, 29, 1, 5, 1, 62, 36, 28, 69, 11, 63, 23, 10, 71, 45, 70, 22, 52, 55, 65, 72, 10, 7, 5, 24, 72, 21, 69, 44, 61, 24, 1, 6, 4, 72, 72, 14, 7, 48, 55, 1, 45, 3, 71, 8, 3, 69, 1, 72, 11, 71, 1, 33, 16, 56, 1, 5, 57, 56, 8, 22, 1, 40, 46, 63, 68, 69, 56, 10, 63, 24, 19, 22, 29, 13, 70, 49, 43, 3, 42, 57, 2, 72, 46, 66, 62, 72, 35, 17, 72, 28, 56, 31, 45, 1, 2, 6, 48, 25, 64, 50, 52, 4, 32, 45, 9, 66, 3, 54, 1, 64, 31, 14, 12, 67, 35, 45, 10, 29, 24, 66, 51, 45, 49, 29, 40, 37, 25, 22, 72, 7, 33, 23, 24, 1, 69, 3, 56, 65, 71, 14, 2, 32, 40, 1, 1, 7, 15, 17, 19, 71, 54, 31, 11, 18, 72, 71, 5, 38, 5, 2, 52, 8, 68, 69, 42, 50, 1, 1, 33, 7, 64, 1, 59, 6, 3, 15, 13, 23, 31, 29, 49, 56, 63, 63, 24, 36, 9, 3, 21, 13, 1, 25, 71, 66, 45, 22, 67, 68, 0, 49, 4, 63, 2, 21, 55, 1, 17, 30, 22, 9, 1, 21, 19, 69, 1, 72, 70, 66, 7, 46, 39, 32, 24, 6, 37, 8, 72, 71, 16, 57, 66, 17, 21, 66, 17, 1, 58, 8, 27, 34, 30, 33, 1, 14, 16, 49, 19, 70, 32, 18, 37, 4, 16, 17, 19, 60, 51, 28, 43, 42, 3, 1, 3, 63, 3, 68, 30, 60, 15, 45, 70, 10, 4, 1, 68, 22, 38, 1, 18, 29, 16, 1, 12, 31, 4, 48, 15, 50, 7, 41, 68, 26, 57, 3, 1, 19, 3, 59, 1, 42, 7, 67, 1, 66, 61, 4, 42, 64, 54, 1, 54, 18, 3, 1, 72, 60, 11, 12, 61, 39, 55, 17, 37, 72, 72, 8, 22, 1, 38, 17, 70, 72, 28, 15, 72, 11, 8, 57, 1, 46, 30, 10, 23, 32, 13, 39, 44, 9, 67, 9, 15, 71, 1, 30, 1, 17, 3, 67, 1, 1, 32, 41, 1, 1, 12, 62, 22, 17, 72, 56, 9, 72, 20, 19, 2, 53, 27, 6, 9, 8, 71, 10, 1, 71, 68, 34, 26, 22, 7, 20, 60, 72, 72, 4, 16, 62, 10, 31, 71, 58, 70, 71, 69, 1, 72, 26, 33, 10, 57, 10, 39, 11, 21, 68, 18, 6, 18, 52, 56, 45, 67, 3, 65, 63, 11, 1, 55, 25, 72, 72, 65, 54, 7, 72, 21, 2, 4, 3, 72, 6, 52, 69, 8, 8, 63, 60, 12, 13, 22, 5, 1, 72, 2, 40, 44, 71, 2, 26, 1, 1, 65, 3, 13, 33, 1, 4, 2, 72, 37, 15, 23, 30, 42, 32, 22, 42, 8, 65, 2, 70, 22, 4, 2, 67, 25, 20, 2, 51, 46, 25, 13, 25, 26, 43, 19, 10, 2, 72, 18, 9, 27, 24, 69, 46, 72, 22, 70, 2, 31, 56, 16, 52, 13, 35, 59, 72, 66, 49, 2, 21, 54, 24, 1, 6, 1, 49, 56, 56, 6, 32, 50, 58, 65, 64, 66, 38, 20, 36, 64, 1, 60, 1, 50, 1, 72, 60, 46, 69, 31, 19, 71, 12, 39, 44, 56, 72, 5, 11, 24, 15, 72, 56, 64, 34, 2, 35, 22, 5, 9, 11, 23, 4, 68, 33, 31, 1, 56, 1, 66, 72, 34, 58, 2, 37, 71, 1, 71, 35, 6, 3, 69, 44, 53, 24, 5, 2, 62, 19, 9, 53, 5, 71, 1, 18, 72, 4, 59, 1, 31, 3, 65, 49, 2, 53, 55, 72, 36, 10, 1, 72, 28, 38, 61, 52, 67, 34, 54, 1, 15, 4, 9, 46, 22, 38, 55, 1, 64, 53, 58, 56, 72, 1, 72, 22, 8, 16, 39, 12, 54, 18, 32, 41, 67, 65, 25, 1, 67, 7, 43, 24, 9, 69, 37, 20, 7, 37, 5, 41, 54, 3, 69, 53, 18, 64, 31, 20, 57, 63, 13, 48, 2, 57, 71, 7, 16, 34, 37, 16, 48, 58, 72, 7, 38, 48, 10, 30, 31, 46, 50, 28, 66, 8, 41, 72, 7, 38, 44, 47, 53, 4, 20, 2, 57, 44, 24, 15, 3, 4, 37, 1, 24, 5, 33, 58, 72, 71, 28, 51, 30, 72, 36, 14, 72, 22, 2, 15, 51, 70, 71, 39, 61, 52, 1, 64, 62, 30, 4, 63, 1, 15, 27, 4, 72, 45, 45, 36, 17, 1, 16, 3, 4, 71, 10, 20, 4, 26, 4, 5, 4, 29, 2, 29, 1, 1, 8, 13, 59, 1, 50, 18, 17, 47, 26, 6, 19, 3, 68, 2, 7, 18, 71, 13, 3, 72, 66, 24, 1, 56, 22, 14, 61, 40, 42, 72, 12, 71, 26, 7, 6, 58, 51, 72, 18, 7, 47, 2, 62, 16, 6, 19, 69, 11, 64, 39, 15, 25, 6, 66, 61, 43, 12, 23, 71, 34, 5, 41, 72, 14, 41, 23, 71, 1, 72, 6, 23, 10, 72, 7, 6, 9, 12, 1, 48, 20, 16, 2, 10, 2, 20, 20, 19, 19, 22, 35, 1, 39, 54, 1, 66, 56, 18, 16, 68, 53, 72, 9, 30, 36, 18, 55, 39, 21, 2, 33, 44, 30, 71, 4, 35, 1, 23, 22, 49, 42, 33, 7, 67, 15, 67, 53, 21, 40, 22, 39, 45, 2, 57, 8, 7, 6, 7, 49, 65, 55, 71, 35, 3, 11, 1, 17, 72, 28, 18, 40, 52, 47, 23, 66, 8, 47, 7, 71, 50, 46, 1, 66, 42, 5, 7, 29, 27, 15, 25, 11, 57, 67, 47, 13, 8, 44, 71, 24, 15, 1, 2, 55, 71, 50, 1, 5, 66, 49, 3, 66, 11, 28, 65, 62, 2, 2, 55, 41, 17, 30, 17, 16, 72, 9, 1, 23, 8, 19, 7, 1, 61, 57, 9, 15, 1, 12, 54, 4, 7, 20, 26, 36, 53, 3, 68, 72, 12, 34, 68, 50, 1, 41, 30, 1, 29, 23, 60, 72, 22, 72, 66, 72, 47, 51, 70, 9, 59, 3, 38, 37, 37, 24, 14, 72, 53, 8, 72, 17, 2, 8, 48, 10, 0, 1, 29, 65, 8, 61, 45, 72, 12, 7, 9, 43, 58, 16, 2, 8, 40, 9, 41, 26, 33, 68, 65, 55, 20, 19, 45, 70, 2, 27, 12, 72, 12, 5, 71, 35, 70, 31, 52, 37, 69, 30, 33, 54, 59, 55, 69, 66, 37, 9, 69, 10, 40, 13, 6, 69, 66, 11, 46, 6, 56, 70, 33, 72, 3, 19, 5, 71, 8, 1, 1, 61, 71, 68, 46, 33, 53, 50, 57, 54, 60, 28, 1, 29, 10, 43, 13, 43, 19, 1, 69, 61, 43, 6, 1, 56, 70, 1, 49, 6, 32, 72, 37, 69, 26, 58, 24, 5, 15, 30, 55, 25, 10, 44, 47, 13, 49, 64, 1, 20, 37, 30, 38, 1, 37, 52, 71, 26, 66, 72, 25, 69, 53, 12, 26, 21, 1, 48, 26, 60, 18, 10, 5, 4, 65, 70, 18, 62, 66, 65, 3, 34, 16, 54, 50, 71, 10, 1, 18, 4, 58, 56, 2, 32, 56, 36, 4, 53, 10, 4, 1, 51, 12, 6, 63, 1, 48, 5, 35, 6, 2, 50, 33, 31, 9, 54, 46, 34, 71, 63, 51, 26, 64, 1, 61, 15, 64, 18, 57, 14, 18, 72, 70, 38, 68, 13, 65, 30, 51, 31, 9, 72, 10, 37, 2, 55, 33, 46, 1, 20, 9, 32, 19, 70, 61, 26, 45, 62, 1, 3, 41, 67, 1, 71, 37, 60, 1, 6, 13, 11, 7, 10, 34, 62, 64, 1, 25, 26, 10, 53, 7, 33, 71, 29, 24, 20, 1, 54, 5, 72, 52, 9, 1, 1, 33, 55, 69, 1, 54, 33, 45, 11, 6, 21, 65, 6, 8, 11, 43, 49, 1, 15, 60, 17, 16, 35, 44, 12, 1, 28, 70, 5, 18, 70, 9, 67, 1, 18, 4, 71, 30, 1, 55, 59, 1, 7, 45, 54, 51, 72, 44, 2, 66, 68, 31, 21, 21, 55, 9, 71, 1, 22, 1, 61, 67, 14, 59, 21, 4, 3, 70, 3, 21, 20, 22, 1, 63, 70, 13, 5, 72, 13, 61, 1, 56, 4, 35, 18, 72, 49, 44, 3, 37, 61, 70, 1, 41, 70, 1, 51, 42, 70, 48, 68, 48, 26, 11, 1, 27, 46, 1, 46, 25, 4, 13, 31, 23, 2, 65, 22, 55, 9, 7, 35, 6, 1, 17, 10, 15, 40, 13, 29, 3, 58, 45, 72, 68, 1, 38, 2, 11, 20, 72, 3, 23, 40, 62, 22, 11, 7, 13, 1, 39, 3, 58, 6, 1, 22, 14, 64, 1, 6, 1, 39, 20, 1, 1, 64, 1, 46, 28, 33, 39, 42, 1, 7, 70, 65, 1, 18, 24, 63, 44, 4, 1, 37, 10, 34, 35, 4, 39, 43, 17, 61, 49, 4, 64, 3, 1, 40, 1, 8, 1, 34, 1, 39, 58, 45, 6, 43, 41, 5, 72, 4, 9, 72, 33, 72, 22, 70, 21, 15, 29, 15, 71, 72, 19, 1, 1, 2, 11, 12, 70, 20, 23, 49, 4, 32, 2, 69, 6, 24, 32, 27, 27, 58, 1, 18, 47, 70, 13, 36, 67, 10, 19, 71, 72, 48, 1, 18, 1, 67, 69, 19, 72, 38, 40, 61, 10, 32, 21, 59, 13, 47, 69, 2, 22, 15, 53, 28, 22, 1, 16, 48, 30, 3, 57, 68, 23, 65, 44, 71, 37, 12, 69, 35, 5, 58, 72, 72, 1, 39, 53, 27, 1, 1, 18, 46, 72, 36, 4, 25, 40, 63, 15, 14, 72, 39, 47, 19, 5, 13, 17, 34, 42, 5, 71, 19, 2, 57, 72, 6, 17, 61, 1, 48, 16, 9, 3, 1, 65, 70, 60, 69, 35, 22, 66, 1, 1, 34, 72, 31, 30, 9, 20, 19, 65, 30, 6, 2, 53, 7, 61, 70, 13, 35, 2, 3, 3, 62, 72, 63, 20, 35, 21, 62, 15, 55, 11, 17, 61, 71, 2, 35, 17, 21, 47, 3, 3, 44, 1, 44, 5, 24, 1, 18, 10, 65, 1, 53, 3, 33, 3, 34, 14, 13, 46, 23, 47, 17, 49, 59, 69, 11, 10, 12, 45, 39, 71, 71, 33, 67, 37, 49, 9, 52, 70, 1, 14, 1, 1, 52, 6, 7, 47, 26, 25, 69, 72, 4, 59, 67, 26, 27, 72, 6, 62, 20, 6, 51, 61, 62, 72, 13, 5, 3, 26, 13, 38, 8, 34, 18, 56, 36, 9, 1, 12, 57, 42, 33, 70, 68, 1, 37, 4, 1, 20, 72, 31, 18, 11, 33, 62, 1, 16, 22, 49, 36, 42, 4, 12, 31, 5, 66, 15, 64, 10, 7, 29, 57, 46, 53, 17, 38, 15, 22, 14, 57, 11, 1, 12, 3, 36, 16, 65, 2, 42, 72, 62, 6, 48, 35, 52, 1, 6, 71, 67, 60, 23, 39, 15, 53, 24, 37, 5, 50, 54, 3, 68, 5, 33, 41, 34, 13, 20, 51, 3, 41, 13, 35, 12, 4, 43, 12, 68, 25, 7, 66, 53, 63, 70, 27, 1, 5, 37, 3, 12, 38, 9, 13, 29, 47, 61, 16, 41, 43, 36, 6, 58, 19, 11, 39, 8, 26, 53, 70, 1, 59, 2, 7, 12, 59, 61, 72, 13, 64, 1, 10, 65, 62, 55, 25, 1, 1, 59, 64, 36, 3, 61, 26, 1, 1, 68, 2, 72, 71, 57, 4, 1, 72, 21, 71, 29, 69, 64, 16, 4, 52, 2, 1, 18, 2, 19, 40, 66, 21, 8, 72, 48, 69, 72, 14, 6, 8, 17, 65, 57, 13, 19, 56, 14, 52, 58, 47, 67, 2, 6, 71, 46, 5, 67, 3, 3, 52, 42, 50, 23, 67, 25, 39, 69, 1, 32, 9, 16, 60, 72, 5, 26, 3, 2, 2, 36, 7, 60, 19, 45, 4, 31, 47, 1, 1, 1, 59, 10, 35, 4, 32, 43, 4, 54, 11, 66, 61, 72, 44, 41, 50, 47, 8, 18, 72, 1, 42, 18, 13, 68, 4, 69, 17, 25, 43, 59, 5, 21, 69, 13, 42, 52, 46, 61, 29, 25, 5, 15, 19, 44, 6, 58, 62, 70, 1, 10, 26, 66, 7, 51, 72, 65, 2, 70, 72, 1, 1, 5, 3, 58, 22, 33, 1, 54, 72, 1, 3, 72, 72, 54, 59, 54, 60, 60, 3, 69, 1, 50, 56, 60, 69, 1, 1, 3, 60, 13, 62, 45, 25, 44, 2, 33, 1, 22, 35, 29, 27, 54, 2, 57, 62, 15, 2, 70, 21, 23, 6, 4, 3, 23, 26, 8, 26, 2, 67, 71, 59, 39, 21, 1, 48, 31, 64, 46, 52, 67, 67, 5, 71, 9, 26, 71, 32, 2, 71, 60, 55, 54, 2, 6, 48, 63, 1, 12, 54, 30, 30, 4, 40, 9, 17, 62, 28, 70, 46, 23, 47, 68, 60, 67, 14, 57, 55, 1, 1, 23, 13, 47, 38, 38, 2, 1, 15, 26, 35, 3, 50, 42, 10, 61, 68, 10, 65, 72, 55, 1, 7, 2, 9, 27, 7, 64, 70, 2, 67, 45, 24, 4, 44, 72, 1, 66, 1, 13, 10, 65, 1, 38, 23, 10, 4, 72, 35, 1, 58, 70, 38, 60, 26, 8, 41, 36, 54, 71, 55, 72, 3, 54, 72, 52, 60, 39, 15, 69, 43, 63, 2, 72, 32, 40, 58, 67, 51, 31, 69, 32, 21, 52, 72, 72, 52, 41, 41, 6, 67, 16, 17, 35, 58, 1, 52, 70, 19, 1, 35, 32, 17, 67, 9, 31, 4, 58, 60, 58, 1, 27, 66, 15, 47, 41, 59, 50, 17, 6, 51, 44, 49, 2, 59, 50, 59, 18, 10, 14, 35, 8, 18, 60, 1, 6, 19, 53, 72, 60, 1, 13, 5, 1, 13, 37, 64, 5, 61, 1, 1, 26, 1, 24, 17, 26, 1, 40, 52, 1, 1, 21, 67, 44, 70, 3, 56, 13, 58, 42, 1, 46, 63, 11, 15, 72, 29, 1, 6, 1, 63, 2, 18, 43, 15, 10, 55, 49, 6, 70, 2, 63, 25, 18, 28, 53, 35, 1, 70, 2, 26, 34, 19, 15, 62, 42, 9, 24, 68, 31, 1, 21, 63, 2, 61, 1, 18, 6, 33, 16, 56, 23, 9, 14, 15, 5, 61, 70, 15, 8, 8, 4, 34, 68, 45, 9, 22, 2, 70, 10, 72, 49, 54, 71, 22, 50, 43, 45, 64, 23, 68, 1, 2, 26, 55, 14, 71, 64, 7, 57, 13, 3, 72, 40, 14, 2, 66, 38, 1, 22, 1, 5, 29, 1, 3, 71, 9, 43, 48, 26, 9, 1, 46, 2, 1, 64, 12, 6, 59, 7, 72, 16, 25, 34, 1, 10, 24, 10, 69, 57, 50, 28, 16, 25, 3, 61, 2, 51, 71, 20, 6, 6, 29, 36, 28, 7, 63, 48, 49, 27, 72, 1, 72, 47, 1, 36, 43, 27, 9, 38, 35, 0, 59, 27, 2, 7, 36, 41, 13, 19, 60, 48, 3, 69, 43, 11, 45, 72, 2, 12, 67, 37, 39, 41, 25, 8, 71, 5, 30, 40, 54, 72, 28, 18, 2, 59, 22, 1, 72, 14, 50, 48, 49, 28, 68, 13, 11, 3, 57, 3, 72, 70, 49, 67, 46, 64, 37, 2, 13, 72, 68, 15, 24, 24, 27, 12, 71, 67, 63, 1, 4, 40, 12, 52, 10, 68, 54, 4, 52, 1, 70, 43, 52, 12, 56, 0, 42, 22, 51, 27, 51, 4, 1, 35, 71, 1, 69, 14, 57, 72, 48, 4, 31, 38, 37, 1, 57, 62, 3, 72, 29, 13, 3, 11, 21, 19, 61, 11, 35, 25, 1, 67, 19, 56, 72, 43, 55, 2, 27, 13, 70, 14, 19, 20, 43, 5, 70, 40, 6, 39, 4, 15, 1, 45, 64, 57, 72, 1, 72, 3, 55, 59, 18, 32, 4, 66, 27, 4, 60, 8, 8, 35, 7, 53, 18, 15, 67, 6, 6, 13, 11, 1, 5, 13, 9, 29, 1, 1, 18, 2, 30, 66, 38, 44, 54, 2, 42, 58, 58, 25, 71, 37, 14, 4, 48, 3, 8, 1, 67, 13, 45, 49, 52, 63, 68, 31, 64, 62, 1, 6, 21, 72, 32, 71, 34, 3, 12, 8, 35, 3, 3, 53, 4, 48, 6, 3, 54, 1, 62, 22, 1, 51, 30, 56, 35, 64, 30, 25, 41, 9, 1, 70, 57, 9, 69, 43, 72, 44, 72, 33, 54, 27, 54, 3, 53, 1, 15, 56, 5, 48, 25, 3, 58, 10, 1, 71, 65, 5, 28, 67, 35, 72, 61, 68, 1, 3, 70, 48, 68, 47, 32, 5, 49, 48, 13, 15, 12, 67, 9, 13, 38, 42, 24, 27, 9, 49, 61, 50, 25, 22, 1, 4, 18, 56, 53, 51, 24, 62, 24, 70, 1, 16, 8, 72, 23, 31, 37, 30, 35, 23, 20, 36, 8, 71, 50, 43, 57, 41, 27, 13, 3, 67, 3, 64, 26, 38, 23, 40, 72, 3, 23, 1, 4, 62, 40, 41, 34, 1, 51, 1, 39, 12, 12, 72, 63, 44, 18, 9, 13, 68, 6, 2, 55, 1, 38, 67, 19, 12, 72, 24, 72, 11, 4, 66]}, {\"name\": \"Monthly Charges of Churn\", \"type\": \"histogram\", \"x\": [53.85, 70.7, 99.65, 104.8, 103.7, 55.2, 39.65, 20.15, 99.35, 30.2, 64.7, 69.7, 106.35, 97.85, 80.65, 99.1, 80.65, 95.45, 94.4, 79.35, 75.15, 78.95, 21.05, 98.5, 110.0, 96.75, 76.5, 100.25, 74.4, 78.05, 58.6, 35.45, 44.35, 70.45, 71.15, 45.65, 95.0, 82.4, 70.9, 45.3, 104.4, 94.85, 74.45, 76.45, 29.95, 84.5, 79.25, 24.8, 91.0, 79.9, 106.6, 46.0, 70.15, 50.05, 55.2, 84.6, 54.4, 95.0, 74.4, 48.55, 70.4, 40.2, 44.6, 41.15, 106.9, 19.35, 94.45, 24.8, 70.6, 85.4, 105.05, 95.15, 70.0, 74.5, 44.85, 76.1, 73.6, 95.45, 74.9, 80.6, 80.3, 93.15, 82.45, 70.35, 73.85, 80.6, 75.8, 104.6, 103.4, 90.4, 84.8, 41.9, 80.25, 30.75, 96.5, 85.65, 104.95, 50.65, 90.85, 19.95, 85.45, 73.95, 99.45, 19.9, 19.6, 81.35, 83.3, 75.3, 19.4, 45.4, 105.9, 69.55, 81.05, 101.15, 99.8, 55.95, 55.0, 74.7, 80.25, 96.1, 69.0, 45.3, 83.55, 74.35, 74.4, 43.75, 28.45, 99.7, 94.1, 94.2, 80.5, 74.35, 104.8, 75.2, 75.6, 100.05, 85.0, 86.05, 45.55, 86.3, 80.35, 100.25, 100.3, 19.0, 75.3, 89.2, 85.7, 61.65, 105.25, 29.95, 65.0, 20.0, 90.05, 110.75, 105.5, 104.55, 85.25, 56.15, 89.55, 94.55, 45.7, 89.5, 69.55, 74.6, 19.65, 19.85, 24.25, 69.65, 45.65, 75.05, 49.15, 34.7, 80.0, 49.25, 75.1, 50.15, 79.85, 19.55, 85.95, 45.35, 94.5, 91.7, 87.25, 98.55, 98.55, 85.9, 89.25, 70.3, 93.35, 19.9, 88.9, 95.8, 82.0, 45.35, 52.2, 110.0, 96.75, 98.5, 85.35, 101.3, 69.55, 103.25, 104.0, 86.2, 111.2, 89.45, 95.6, 90.95, 108.55, 78.85, 44.4, 100.0, 19.8, 89.9, 100.15, 50.8, 73.65, 95.1, 94.65, 80.6, 85.55, 50.8, 105.1, 110.1, 83.9, 100.55, 103.85, 24.6, 98.9, 98.3, 93.85, 100.5, 72.85, 73.55, 79.4, 81.15, 84.6, 79.05, 74.4, 99.05, 44.95, 44.7, 45.7, 100.3, 19.25, 96.0, 90.55, 30.35, 108.05, 69.9, 103.75, 86.6, 80.6, 85.3, 70.0, 94.3, 95.35, 75.5, 90.1, 68.95, 99.55, 57.45, 53.65, 100.6, 83.75, 88.3, 92.1, 79.45, 90.45, 69.75, 19.65, 43.65, 39.5, 97.1, 80.0, 84.7, 89.55, 90.6, 90.05, 99.05, 69.75, 49.05, 98.05, 114.5, 80.95, 74.3, 89.7, 100.45, 90.4, 56.15, 94.4, 78.95, 44.85, 105.65, 74.65, 64.7, 104.05, 35.55, 95.15, 96.65, 80.4, 31.35, 89.75, 94.4, 19.85, 109.9, 101.35, 60.05, 60.15, 90.45, 94.25, 74.9, 80.25, 69.0, 66.35, 86.0, 80.3, 90.55, 75.9, 45.85, 49.95, 78.3, 76.95, 96.15, 45.3, 19.4, 90.15, 45.05, 75.8, 79.3, 30.9, 95.25, 89.6, 110.15, 89.5, 75.0, 44.75, 44.05, 85.6, 115.55, 86.6, 85.2, 97.65, 109.55, 89.55, 19.45, 20.35, 69.25, 99.5, 25.2, 45.0, 20.15, 105.0, 54.7, 87.25, 79.95, 88.35, 94.75, 95.05, 78.45, 70.2, 41.05, 85.6, 79.2, 70.0, 49.95, 69.25, 94.25, 73.0, 100.05, 99.8, 35.0, 76.0, 93.85, 84.3, 84.4, 101.1, 50.45, 99.95, 91.4, 75.55, 80.8, 100.0, 20.55, 85.3, 70.4, 98.8, 74.4, 98.75, 106.0, 104.7, 96.7, 55.05, 88.2, 19.75, 75.65, 74.7, 76.65, 25.8, 42.6, 68.85, 90.0, 75.35, 100.85, 69.95, 107.5, 45.85, 106.1, 91.7, 95.6, 74.95, 95.35, 45.0, 99.5, 98.6, 59.5, 80.45, 77.95, 74.7, 88.8, 90.25, 64.65, 89.05, 87.4, 94.75, 86.45, 98.25, 75.75, 85.35, 106.1, 45.05, 109.8, 84.65, 79.5, 85.8, 79.1, 44.3, 105.95, 69.75, 94.65, 96.05, 50.15, 113.6, 78.9, 60.05, 34.7, 85.3, 102.45, 104.4, 99.75, 74.4, 74.25, 59.85, 69.6, 45.4, 19.5, 69.9, 87.15, 84.75, 89.95, 113.2, 90.5, 79.0, 20.15, 71.65, 20.35, 84.25, 78.1, 89.65, 98.7, 76.35, 79.15, 85.0, 85.3, 86.55, 73.85, 44.85, 45.1, 96.0, 20.05, 108.65, 45.55, 35.1, 46.2, 45.15, 43.3, 57.15, 73.2, 85.35, 45.95, 83.75, 70.05, 86.0, 100.5, 72.65, 21.0, 45.1, 50.4, 78.95, 94.45, 84.8, 50.9, 99.8, 107.35, 19.55, 79.1, 25.5, 80.75, 91.85, 75.35, 75.45, 49.55, 78.6, 81.1, 74.95, 93.55, 102.1, 90.9, 29.2, 89.15, 108.85, 84.75, 45.7, 69.95, 44.6, 74.95, 95.25, 89.85, 100.45, 47.15, 80.2, 87.1, 75.9, 98.75, 86.45, 45.3, 104.1, 81.0, 90.6, 88.15, 20.2, 90.8, 79.35, 96.8, 83.35, 49.4, 79.2, 48.7, 95.6, 59.75, 108.15, 71.35, 20.15, 69.75, 93.2, 80.85, 33.65, 79.95, 19.3, 99.2, 85.2, 75.25, 54.35, 24.95, 42.35, 75.3, 94.8, 70.55, 86.15, 85.0, 80.55, 75.8, 98.5, 95.25, 74.4, 85.15, 88.55, 54.75, 91.3, 79.5, 70.1, 111.3, 95.25, 86.25, 100.8, 86.3, 89.95, 76.45, 70.0, 64.4, 102.45, 80.1, 80.2, 98.15, 112.95, 70.9, 86.85, 99.85, 74.5, 109.15, 65.2, 99.5, 71.55, 93.9, 108.4, 48.75, 85.65, 106.7, 25.15, 110.35, 103.25, 47.8, 98.6, 83.05, 70.4, 104.75, 94.25, 45.95, 70.15, 85.55, 70.3, 106.35, 103.7, 88.35, 94.2, 96.25, 70.7, 33.55, 94.55, 112.95, 35.9, 99.25, 24.45, 103.35, 84.8, 99.95, 90.0, 104.85, 106.1, 74.2, 89.35, 77.15, 35.25, 48.55, 98.4, 93.85, 100.7, 80.45, 55.25, 54.45, 75.35, 75.7, 74.25, 89.7, 96.4, 69.5, 85.25, 19.4, 83.3, 80.1, 62.7, 25.3, 98.15, 93.7, 105.95, 82.0, 95.75, 94.05, 54.75, 95.65, 94.6, 98.9, 76.05, 69.75, 34.5, 105.55, 30.1, 70.3, 80.2, 94.35, 44.6, 19.6, 110.45, 70.7, 99.25, 84.8, 105.0, 54.45, 85.3, 83.35, 33.5, 90.35, 90.75, 89.6, 49.25, 19.75, 95.15, 101.35, 70.05, 74.65, 50.4, 79.65, 79.85, 78.75, 80.45, 50.1, 76.4, 98.7, 100.8, 90.1, 51.55, 79.25, 94.85, 85.7, 44.4, 89.5, 86.05, 44.55, 35.4, 95.65, 20.95, 34.7, 19.75, 92.55, 60.05, 74.3, 85.35, 74.3, 98.1, 109.65, 74.55, 100.95, 94.9, 104.9, 24.1, 104.05, 88.35, 94.65, 74.4, 79.9, 69.7, 98.1, 20.5, 99.6, 81.95, 69.35, 91.1, 86.1, 94.7, 85.1, 99.15, 102.95, 45.05, 71.0, 70.55, 79.7, 59.0, 60.35, 69.25, 70.95, 89.0, 99.4, 73.5, 95.5, 92.35, 74.55, 83.8, 54.45, 99.8, 105.85, 75.45, 93.85, 100.75, 31.0, 80.0, 70.25, 100.15, 94.5, 33.9, 89.95, 84.45, 79.5, 89.15, 74.95, 74.4, 55.3, 55.15, 84.95, 43.25, 48.35, 79.55, 71.05, 84.5, 108.6, 30.3, 30.4, 45.4, 84.15, 19.75, 85.4, 75.2, 55.85, 106.4, 84.4, 56.25, 85.55, 68.65, 77.85, 105.35, 87.05, 53.5, 25.1, 79.65, 54.2, 19.45, 103.4, 84.45, 85.95, 59.4, 30.6, 99.55, 70.0, 45.3, 29.85, 95.1, 19.8, 76.65, 99.15, 44.95, 49.55, 95.9, 81.5, 84.1, 36.45, 25.05, 49.85, 55.55, 69.5, 104.3, 93.85, 50.45, 74.65, 94.2, 106.55, 20.1, 83.8, 55.15, 86.65, 106.75, 78.45, 87.45, 70.75, 101.35, 81.7, 68.25, 79.15, 95.45, 100.95, 69.1, 69.6, 103.45, 77.95, 78.45, 29.5, 70.75, 84.45, 85.65, 70.15, 97.8, 46.3, 106.3, 89.4, 83.15, 74.15, 99.5, 44.15, 53.9, 85.45, 50.85, 59.2, 83.2, 54.9, 103.9, 19.25, 75.45, 54.85, 95.1, 75.5, 73.75, 96.05, 88.95, 50.7, 53.4, 101.9, 83.0, 50.6, 89.5, 69.8, 93.8, 101.05, 94.7, 104.15, 61.45, 90.55, 88.8, 85.2, 91.4, 106.15, 45.3, 73.25, 51.05, 85.35, 75.35, 75.2, 20.9, 43.3, 109.5, 84.8, 95.05, 50.35, 74.6, 19.7, 59.45, 105.2, 45.15, 93.8, 96.2, 79.6, 85.25, 80.5, 89.8, 50.75, 89.65, 65.4, 72.9, 40.0, 99.45, 86.25, 45.65, 70.1, 70.2, 55.35, 84.2, 45.85, 75.25, 100.15, 105.75, 48.35, 24.4, 20.95, 49.55, 20.25, 50.15, 95.9, 25.25, 45.15, 84.4, 75.1, 70.15, 101.75, 45.8, 20.1, 40.65, 101.0, 69.1, 75.0, 100.0, 71.15, 54.15, 108.75, 49.15, 19.4, 79.15, 80.4, 102.0, 20.65, 70.8, 89.05, 96.6, 104.5, 50.6, 49.35, 94.25, 68.95, 93.85, 76.05, 24.45, 89.35, 80.35, 55.8, 79.05, 95.0, 25.2, 98.4, 79.4, 101.4, 89.8, 109.75, 19.3, 91.25, 49.1, 65.25, 85.45, 94.1, 71.65, 104.65, 74.25, 74.2, 101.0, 100.2, 89.05, 70.1, 105.65, 96.5, 70.85, 85.95, 73.9, 109.45, 100.2, 74.05, 101.55, 111.95, 55.95, 45.75, 74.95, 80.15, 109.25, 35.2, 78.8, 106.4, 49.4, 69.1, 90.2, 93.55, 80.85, 82.05, 20.55, 78.55, 78.65, 42.9, 34.25, 58.5, 96.2, 74.4, 105.3, 73.85, 84.55, 24.75, 85.8, 24.45, 105.7, 24.25, 76.5, 93.5, 101.5, 74.9, 69.6, 24.05, 50.55, 100.55, 20.5, 74.8, 24.45, 71.0, 40.3, 50.6, 84.85, 65.7, 70.5, 94.85, 19.75, 65.45, 98.7, 67.5, 69.6, 84.35, 100.2, 78.05, 73.6, 74.75, 20.2, 93.55, 25.05, 95.55, 39.3, 20.25, 23.9, 85.45, 99.9, 84.8, 75.2, 49.35, 74.9, 101.25, 18.95, 28.6, 24.35, 98.65, 79.9, 103.3, 96.2, 104.15, 102.95, 90.6, 50.8, 90.05, 75.85, 68.5, 73.5, 69.95, 70.05, 94.0, 61.15, 84.45, 75.3, 109.55, 19.6, 66.4, 100.05, 102.0, 90.2, 25.15, 79.85, 49.6, 105.95, 91.85, 40.1, 73.9, 89.8, 72.25, 96.55, 40.1, 101.25, 79.55, 84.05, 89.1, 94.4, 78.25, 44.15, 75.75, 19.1, 80.3, 74.9, 80.85, 24.75, 90.85, 79.7, 57.55, 86.5, 103.95, 25.1, 48.95, 83.55, 89.15, 70.45, 60.3, 24.75, 103.3, 78.5, 19.85, 79.9, 99.7, 58.6, 98.5, 50.45, 94.65, 100.25, 88.45, 69.85, 81.7, 79.9, 25.4, 83.75, 20.9, 69.4, 74.9, 74.15, 70.1, 78.3, 96.8, 94.4, 84.3, 92.45, 99.55, 81.25, 19.95, 99.25, 80.45, 79.35, 79.75, 100.05, 69.35, 85.8, 79.7, 50.55, 80.55, 84.5, 84.3, 67.75, 85.25, 80.05, 20.0, 48.8, 85.05, 102.8, 99.95, 117.8, 79.5, 79.65, 94.7, 92.5, 19.45, 75.55, 70.0, 74.8, 104.4, 86.2, 51.2, 19.5, 74.9, 74.9, 100.25, 102.8, 100.0, 99.85, 94.2, 83.85, 88.3, 104.05, 94.7, 100.65, 80.55, 78.85, 44.45, 84.9, 20.25, 74.35, 62.05, 80.0, 104.1, 39.1, 76.5, 49.2, 23.15, 80.85, 60.1, 78.95, 45.15, 79.4, 53.95, 99.3, 80.65, 71.1, 106.25, 20.8, 94.0, 89.85, 46.0, 66.25, 90.0, 70.45, 19.75, 53.95, 105.5, 81.0, 69.8, 73.75, 54.55, 76.25, 93.9, 70.4, 91.15, 70.95, 99.5, 83.9, 117.45, 104.4, 29.4, 44.65, 25.25, 84.05, 85.7, 74.7, 90.8, 106.15, 85.35, 90.85, 35.75, 46.0, 89.4, 19.8, 50.9, 81.0, 98.35, 51.0, 91.65, 90.85, 69.4, 94.45, 20.15, 95.7, 74.95, 47.95, 63.6, 36.85, 44.1, 50.75, 104.4, 39.3, 106.45, 96.05, 75.1, 44.7, 55.0, 74.35, 104.7, 55.7, 19.85, 70.3, 85.55, 94.5, 85.35, 79.65, 79.6, 90.6, 20.2, 106.05, 99.15, 55.9, 82.35, 40.4, 75.35, 85.15, 93.8, 20.4, 94.7, 30.5, 77.0, 20.1, 75.3, 99.2, 85.5, 20.05, 70.4, 49.75, 75.3, 94.0, 118.35, 30.45, 96.1, 104.25, 80.2, 104.2, 69.65, 80.8, 80.3, 84.1, 69.35, 79.6, 36.45, 89.4, 80.35, 40.65, 60.45, 84.9, 73.55, 20.15, 34.7, 107.55, 98.75, 101.15, 88.7, 79.75, 94.55, 69.05, 96.25, 101.1, 104.7, 55.25, 90.1, 53.75, 89.3, 95.2, 45.7, 74.5, 79.65, 101.3, 80.05, 89.2, 83.65, 19.5, 51.25, 45.6, 49.65, 20.0, 34.0, 92.5, 88.35, 107.95, 74.45, 64.75, 89.8, 95.5, 29.65, 84.5, 90.7, 70.3, 103.95, 74.75, 50.8, 80.25, 78.9, 84.75, 85.45, 19.9, 99.45, 70.6, 81.0, 84.85, 89.15, 24.8, 105.65, 80.85, 89.45, 72.15, 76.4, 61.6, 79.9, 96.15, 65.3, 19.65, 84.65, 69.55, 31.05, 51.0, 44.35, 79.7, 55.0, 84.85, 73.65, 73.9, 77.75, 84.35, 84.95, 73.85, 51.8, 46.0, 79.4, 71.8, 88.4, 46.0, 51.75, 94.4, 70.15, 34.7, 81.1, 20.55, 106.6, 106.75, 89.45, 97.9, 55.7, 79.8, 39.05, 83.9, 69.85, 53.55, 93.9, 113.15, 84.95, 80.5, 19.3, 49.65, 54.35, 88.45, 90.5, 94.85, 69.95, 48.6, 55.8, 80.15, 75.5, 20.2, 74.75, 104.05, 96.65, 20.05, 103.45, 19.9, 46.35, 19.95, 79.6, 75.9, 76.2, 19.25, 84.3, 79.0, 70.25, 84.6, 24.1, 60.0, 86.55, 20.85, 50.75, 105.5, 85.15, 54.2, 109.5, 104.4, 105.1, 75.7, 90.8, 97.95, 108.15, 80.5, 104.3, 71.35, 89.4, 20.6, 99.95, 81.0, 70.15, 70.15, 96.55, 95.55, 69.65, 89.3, 87.55, 96.3, 85.2, 35.1, 81.3, 19.6, 85.45, 71.0, 85.05, 44.4, 70.3, 79.35, 90.8, 70.45, 45.25, 75.8, 71.3, 69.1, 88.25, 70.0, 97.35, 25.9, 55.65, 69.65, 100.6, 86.0, 106.95, 29.6, 59.9, 95.6, 41.6, 100.85, 40.2, 84.5, 78.95, 20.2, 85.2, 98.0, 94.45, 105.0, 93.85, 84.0, 48.75, 84.85, 56.65, 73.9, 24.5, 89.9, 78.55, 99.45, 25.2, 74.05, 55.0, 89.75, 85.15, 94.6, 94.25, 72.45, 105.2, 82.85, 103.85, 83.6, 108.45, 91.5, 80.85, 49.05, 105.3, 88.95, 110.85, 114.2, 36.5, 70.75, 40.15, 94.45, 97.35, 45.3, 94.95, 44.55, 23.45, 29.7, 101.1, 50.65, 55.4, 90.6, 86.85, 99.05, 111.25, 55.1, 102.25, 95.9, 95.05, 53.85, 104.2, 50.25, 99.5, 71.0, 93.2, 19.35, 19.85, 104.2, 79.25, 44.9, 70.1, 29.65, 77.85, 54.3, 95.15, 110.85, 80.5, 72.2, 51.5, 89.55, 77.5, 95.25, 85.0, 85.75, 98.25, 25.7, 69.0, 46.05, 76.4, 100.5, 40.05, 69.95, 93.3, 49.15, 35.05, 73.0, 83.85, 70.1, 70.65, 20.35, 99.0, 83.95, 116.2, 93.7, 19.7, 75.55, 77.65, 95.0, 105.4, 95.85, 79.15, 19.65, 79.0, 91.85, 106.0, 49.9, 100.35, 74.95, 49.25, 103.7, 89.75, 20.3, 101.95, 94.0, 89.3, 74.3, 44.65, 84.05, 80.7, 101.7, 20.4, 102.6, 106.0, 55.1, 106.5, 91.05, 77.5, 54.2, 69.25, 50.5, 74.4, 79.45, 104.65, 100.05, 100.55, 95.65, 94.4, 55.0, 101.45, 67.95, 69.5, 18.85, 69.2, 69.65, 101.85, 82.3, 79.25, 84.6, 93.4, 94.2, 25.05, 25.1, 48.95, 45.3, 85.85, 95.2, 100.6, 55.3, 74.85, 102.35, 115.65, 79.85, 76.0, 104.75, 74.65, 56.25, 106.1, 79.75, 68.65, 20.25, 100.2, 74.9, 70.85, 69.8, 54.55, 70.75, 94.1, 29.7, 110.9, 105.0, 25.3, 55.15, 20.1, 69.9, 101.4, 80.8, 62.05, 76.45, 91.3, 95.75, 62.65, 74.5, 102.1, 50.15, 86.5, 81.45, 69.15, 79.55, 95.5, 91.05, 79.2, 45.25, 74.3, 74.1, 100.6, 75.0, 70.2, 70.75, 50.25, 79.2, 96.0, 100.1, 56.5, 85.0, 59.3, 49.4, 55.0, 69.2, 62.3, 92.5, 69.65, 111.6, 48.55, 20.2, 39.65, 24.9, 56.7, 105.95, 74.95, 29.7, 50.35, 47.85, 70.3, 98.8, 99.85, 80.3, 80.45, 86.8, 84.2, 99.0, 100.75, 91.75, 79.55, 111.1, 89.1, 101.9, 91.65, 88.85, 25.3, 95.45, 91.1, 54.15, 69.7, 89.5, 35.15, 73.75, 101.35, 55.9, 105.9, 43.95, 45.1, 70.25, 55.35, 103.0, 103.75, 69.35, 73.85, 75.15, 69.6, 80.15, 30.5, 98.3, 107.4, 82.0, 75.4, 101.25, 89.5, 74.4, 20.5, 74.35, 99.75, 111.95, 39.0, 94.1, 95.65, 88.05, 66.5, 35.45, 49.9, 104.5, 75.05, 81.0, 74.45, 60.4, 70.65, 59.8, 44.2, 75.75, 102.95, 74.4]}, {\"name\": \"Monthly Charges of No Churn\", \"type\": \"histogram\", \"x\": [29.85, 56.95, 42.3, 89.1, 29.75, 56.15, 49.95, 18.95, 100.35, 105.5, 113.25, 20.65, 106.7, 90.05, 19.8, 59.9, 59.6, 55.3, 90.25, 96.35, 95.5, 66.15, 20.2, 45.25, 99.9, 74.8, 49.55, 69.2, 20.75, 79.85, 76.2, 84.5, 49.25, 79.75, 64.15, 90.25, 69.5, 74.85, 99.65, 108.45, 24.95, 107.5, 100.5, 89.9, 42.1, 54.4, 75.3, 78.9, 79.2, 20.15, 79.85, 49.05, 20.4, 111.6, 24.25, 64.5, 110.5, 55.65, 54.65, 74.75, 25.9, 50.55, 103.8, 20.15, 99.3, 62.15, 20.65, 19.95, 33.75, 82.05, 74.7, 84.0, 111.05, 100.9, 66.85, 21.0, 20.2, 19.45, 95.0, 45.55, 24.3, 104.15, 30.15, 94.35, 19.4, 57.95, 91.65, 54.6, 89.85, 31.05, 20.65, 85.2, 99.8, 20.7, 50.7, 20.85, 88.95, 23.55, 19.75, 56.45, 85.95, 50.55, 25.7, 75.0, 20.2, 19.6, 88.05, 101.05, 84.3, 23.95, 99.05, 19.6, 64.5, 69.5, 68.55, 108.15, 86.1, 19.7, 80.9, 84.15, 20.15, 64.25, 25.7, 56.0, 69.7, 73.9, 20.6, 19.9, 89.05, 20.4, 84.25, 81.95, 20.55, 24.7, 105.35, 20.55, 45.3, 74.75, 51.8, 30.4, 19.65, 56.6, 71.9, 19.75, 109.7, 19.3, 96.55, 24.1, 111.35, 112.25, 20.75, 101.9, 80.05, 105.55, 78.3, 68.85, 79.95, 55.45, 102.45, 25.25, 19.75, 20.0, 86.8, 58.75, 45.25, 56.6, 84.2, 80.0, 24.75, 20.2, 19.35, 50.6, 81.15, 89.9, 85.3, 108.0, 93.5, 20.25, 25.15, 29.6, 73.15, 19.75, 86.6, 109.2, 74.7, 94.4, 54.8, 75.35, 65.0, 99.0, 93.5, 83.7, 19.85, 59.55, 115.1, 114.35, 45.0, 89.85, 49.85, 113.3, 88.1, 24.9, 105.0, 24.25, 59.75, 107.05, 64.95, 55.0, 50.55, 55.15, 51.2, 25.4, 54.45, 76.0, 44.35, 61.2, 86.8, 89.35, 19.7, 20.25, 76.05, 100.8, 74.55, 64.9, 90.4, 60.3, 81.85, 24.8, 75.55, 101.15, 78.75, 19.25, 89.05, 115.05, 69.35, 110.05, 19.9, 91.5, 60.0, 44.8, 48.6, 60.05, 102.7, 82.9, 35.9, 82.65, 19.85, 19.2, 94.9, 88.15, 94.8, 54.65, 85.75, 67.45, 20.5, 20.25, 72.1, 19.45, 44.95, 97.0, 62.8, 44.6, 89.15, 54.1, 105.25, 97.1, 20.2, 98.8, 50.3, 20.55, 75.9, 59.95, 19.15, 98.65, 112.6, 20.6, 35.75, 99.75, 96.1, 85.1, 25.35, 89.65, 86.75, 86.2, 64.8, 108.1, 54.75, 90.4, 44.0, 95.6, 84.8, 44.3, 19.9, 95.05, 90.05, 109.9, 54.6, 20.05, 19.75, 20.05, 55.9, 19.7, 19.8, 95.4, 93.95, 24.45, 74.95, 87.35, 70.65, 73.25, 98.7, 24.8, 24.3, 69.85, 100.55, 25.7, 40.7, 51.65, 105.1, 85.95, 75.6, 58.25, 65.2, 53.45, 19.75, 44.45, 20.85, 114.05, 89.85, 55.05, 112.95, 101.55, 114.65, 64.8, 80.4, 25.05, 94.75, 105.5, 24.7, 69.75, 60.2, 24.4, 104.15, 92.9, 80.8, 20.0, 75.1, 19.65, 69.45, 116.05, 40.05, 102.1, 89.7, 19.9, 20.65, 70.05, 53.6, 76.05, 75.7, 19.65, 81.45, 108.5, 84.5, 100.15, 88.6, 52.55, 104.8, 59.0, 64.05, 20.4, 60.9, 19.8, 116.25, 80.7, 65.2, 84.05, 79.45, 78.0, 19.85, 94.3, 106.45, 105.45, 95.0, 54.3, 70.05, 20.05, 105.4, 51.6, 85.5, 91.25, 115.75, 94.7, 19.6, 99.9, 21.1, 20.05, 79.95, 107.15, 89.55, 81.55, 58.45, 95.65, 80.6, 113.1, 58.95, 19.55, 78.95, 105.05, 101.9, 19.75, 110.3, 115.6, 19.35, 25.6, 68.75, 19.9, 70.6, 70.2, 49.3, 107.25, 23.6, 69.7, 99.5, 64.3, 70.85, 101.9, 73.5, 40.4, 19.25, 59.6, 64.9, 110.85, 81.05, 98.05, 70.5, 94.55, 19.65, 19.0, 20.0, 63.25, 20.1, 99.15, 90.4, 111.9, 24.9, 83.5, 84.3, 45.6, 54.85, 65.55, 90.35, 20.4, 74.55, 19.95, 74.25, 108.65, 109.55, 86.65, 81.0, 47.85, 114.55, 20.55, 109.8, 69.5, 48.85, 25.25, 102.85, 87.55, 78.55, 34.55, 92.05, 85.05, 19.7, 95.15, 84.25, 104.6, 111.65, 55.0, 89.85, 20.35, 54.55, 99.45, 70.9, 25.4, 89.85, 25.25, 69.65, 70.0, 20.1, 24.8, 95.1, 88.85, 78.8, 20.35, 45.25, 20.05, 69.55, 19.5, 74.75, 30.2, 57.8, 19.85, 25.55, 24.85, 110.35, 24.55, 107.95, 81.4, 73.8, 64.4, 103.75, 71.1, 49.9, 24.6, 30.1, 83.4, 20.45, 75.25, 20.55, 20.05, 20.65, 85.15, 84.95, 66.5, 63.3, 83.15, 84.9, 20.55, 49.25, 59.6, 104.65, 75.3, 80.1, 81.0, 24.7, 86.0, 25.4, 89.15, 58.25, 85.65, 50.35, 80.35, 20.2, 20.55, 21.25, 26.25, 80.85, 74.2, 20.35, 75.5, 79.05, 90.15, 50.6, 110.45, 101.0, 79.35, 89.85, 65.0, 80.45, 24.1, 44.05, 110.8, 114.95, 75.05, 19.25, 90.05, 56.7, 80.15, 71.35, 20.25, 90.35, 19.7, 19.85, 90.35, 20.8, 66.85, 19.9, 35.8, 78.85, 20.4, 74.25, 64.8, 20.45, 110.65, 40.3, 107.0, 73.35, 44.8, 54.75, 40.6, 55.3, 60.85, 78.4, 69.65, 59.85, 76.9, 19.85, 67.65, 45.0, 64.2, 81.7, 25.55, 20.0, 75.65, 23.8, 64.2, 76.8, 55.2, 108.55, 25.25, 30.4, 20.05, 84.6, 103.7, 88.0, 106.35, 79.15, 103.1, 63.95, 25.8, 25.55, 44.85, 25.05, 74.1, 88.8, 93.25, 71.4, 79.2, 20.4, 105.0, 30.85, 20.55, 84.85, 33.15, 92.0, 89.8, 115.8, 85.15, 24.85, 64.35, 20.5, 86.05, 89.0, 64.8, 19.8, 93.4, 39.0, 20.5, 26.4, 98.2, 97.55, 19.95, 99.7, 34.8, 60.15, 64.75, 54.65, 19.3, 111.25, 35.8, 20.05, 84.35, 110.5, 91.2, 89.3, 81.1, 81.2, 94.3, 116.1, 105.55, 94.4, 19.5, 105.6, 81.35, 56.4, 65.35, 19.95, 111.25, 89.0, 106.1, 20.05, 25.2, 75.4, 65.55, 80.7, 104.55, 24.15, 20.45, 75.4, 79.7, 81.7, 76.3, 103.75, 86.45, 75.1, 80.6, 19.3, 33.6, 83.25, 80.85, 108.05, 19.9, 21.05, 30.15, 79.85, 65.5, 104.1, 20.5, 91.35, 20.5, 75.6, 55.1, 58.95, 95.1, 25.45, 56.75, 81.75, 86.1, 29.8, 20.5, 60.9, 73.25, 20.85, 77.35, 93.85, 70.1, 75.95, 75.25, 54.95, 19.5, 19.6, 47.85, 23.75, 43.8, 19.75, 19.15, 19.6, 80.3, 24.35, 25.25, 26.1, 20.0, 20.7, 70.3, 69.55, 19.85, 20.0, 95.85, 20.75, 50.15, 58.65, 95.9, 49.5, 80.1, 24.4, 40.05, 19.5, 51.05, 54.35, 84.7, 86.1, 70.35, 110.0, 94.9, 69.75, 71.6, 23.65, 81.85, 25.1, 114.7, 49.15, 80.9, 19.3, 70.2, 54.25, 99.3, 74.0, 50.25, 19.8, 35.5, 80.75, 19.55, 20.05, 112.4, 50.2, 62.25, 55.7, 19.65, 89.25, 54.0, 56.75, 21.1, 96.65, 24.5, 79.2, 69.55, 20.05, 98.85, 25.75, 19.6, 87.65, 74.75, 107.45, 75.35, 64.95, 100.45, 68.5, 80.55, 81.25, 89.55, 55.7, 24.8, 20.0, 105.2, 19.55, 79.75, 97.45, 24.25, 24.6, 50.15, 39.6, 89.85, 98.85, 53.85, 24.25, 89.45, 105.25, 59.5, 70.55, 82.5, 61.6, 49.05, 66.25, 19.4, 86.05, 19.15, 19.25, 81.95, 114.65, 20.0, 19.8, 65.15, 19.65, 88.95, 20.2, 75.2, 56.8, 75.5, 35.6, 60.25, 40.35, 18.85, 54.85, 64.3, 24.65, 76.1, 18.7, 97.95, 94.1, 95.1, 72.35, 82.7, 19.9, 53.8, 51.55, 19.65, 44.05, 114.0, 100.4, 54.25, 80.0, 79.2, 94.3, 49.8, 53.75, 93.45, 87.9, 61.05, 104.05, 99.25, 85.7, 104.85, 69.15, 74.45, 50.45, 60.0, 85.25, 19.45, 20.75, 78.9, 104.5, 49.4, 25.0, 25.55, 70.15, 69.4, 93.15, 69.55, 20.2, 20.4, 23.75, 70.45, 65.75, 24.6, 69.25, 24.65, 90.4, 100.85, 75.35, 87.2, 64.4, 24.7, 105.85, 98.3, 19.45, 58.7, 20.15, 64.5, 28.5, 90.45, 105.15, 83.15, 103.2, 19.45, 88.8, 85.9, 34.2, 20.15, 50.3, 80.15, 51.25, 95.2, 94.8, 80.25, 76.1, 115.55, 24.65, 53.6, 19.45, 88.2, 101.15, 56.8, 99.4, 20.1, 60.7, 20.95, 114.85, 19.25, 62.8, 105.5, 19.85, 74.1, 107.5, 19.55, 68.8, 84.45, 84.5, 111.2, 80.6, 80.7, 75.6, 57.6, 110.6, 58.2, 81.0, 19.7, 59.55, 75.55, 45.1, 70.95, 20.9, 19.95, 24.6, 66.7, 94.8, 65.85, 19.95, 24.65, 51.25, 54.25, 19.4, 56.25, 25.15, 23.95, 35.4, 75.35, 20.4, 56.05, 20.0, 73.05, 20.5, 100.75, 19.95, 49.65, 65.65, 20.45, 60.95, 20.35, 19.5, 75.2, 111.45, 70.15, 92.0, 85.5, 82.15, 84.4, 60.9, 20.25, 95.3, 19.85, 84.35, 19.85, 82.3, 66.8, 44.6, 98.45, 70.7, 24.95, 102.5, 86.55, 24.3, 58.35, 68.75, 85.8, 20.1, 20.35, 110.8, 82.85, 84.35, 19.55, 19.95, 66.25, 23.3, 25.3, 44.55, 104.1, 92.55, 101.45, 94.55, 95.5, 100.3, 55.5, 49.85, 89.55, 19.15, 99.8, 113.05, 19.95, 74.15, 92.0, 73.85, 24.45, 24.8, 64.85, 20.75, 68.95, 109.4, 49.0, 50.25, 19.9, 97.8, 100.3, 55.8, 111.15, 98.55, 50.05, 20.85, 19.5, 19.35, 69.5, 48.8, 94.5, 20.65, 106.05, 108.3, 99.65, 95.9, 20.0, 64.95, 74.6, 49.2, 73.75, 92.3, 19.2, 88.65, 95.95, 105.4, 20.25, 49.05, 35.55, 65.1, 96.85, 69.75, 99.2, 106.8, 51.25, 57.75, 70.85, 19.55, 79.5, 98.15, 20.25, 79.15, 94.25, 40.2, 19.95, 55.35, 102.15, 71.1, 54.1, 19.65, 88.45, 80.4, 19.25, 84.8, 19.5, 68.6, 92.6, 100.55, 20.55, 19.6, 67.45, 43.55, 109.85, 20.65, 95.4, 21.0, 56.2, 18.4, 25.75, 19.6, 19.8, 64.2, 75.75, 78.95, 50.3, 80.3, 19.85, 21.1, 50.0, 104.75, 19.85, 85.9, 80.8, 25.25, 80.55, 81.5, 20.9, 67.25, 20.35, 45.05, 34.65, 69.35, 81.55, 75.4, 67.8, 111.4, 46.3, 20.4, 20.05, 96.1, 19.65, 60.65, 71.7, 36.0, 65.2, 48.95, 53.5, 80.45, 109.05, 26.3, 106.8, 64.95, 19.35, 21.1, 18.85, 26.0, 70.35, 96.9, 19.55, 80.4, 94.65, 95.75, 19.55, 104.1, 20.1, 111.55, 60.5, 90.95, 19.7, 50.95, 20.05, 19.4, 59.45, 81.5, 29.05, 70.6, 97.2, 59.2, 75.9, 90.05, 70.95, 102.6, 43.8, 59.0, 69.95, 24.35, 29.45, 84.4, 20.65, 87.1, 19.85, 90.35, 65.5, 80.95, 56.15, 34.4, 20.75, 18.8, 90.8, 25.6, 70.8, 25.4, 108.8, 76.85, 20.25, 24.8, 115.65, 74.6, 103.15, 72.1, 25.1, 80.15, 25.4, 105.4, 45.75, 24.45, 25.0, 85.25, 19.6, 50.15, 70.55, 26.4, 20.15, 58.85, 97.55, 19.65, 25.25, 114.45, 70.7, 75.55, 84.8, 20.65, 20.45, 35.65, 90.45, 97.65, 73.85, 69.1, 82.75, 24.4, 55.25, 61.35, 76.75, 19.4, 54.75, 19.7, 19.9, 107.95, 83.8, 56.4, 20.1, 94.9, 94.2, 49.9, 71.05, 81.65, 89.45, 99.0, 19.05, 114.45, 44.25, 90.55, 20.4, 71.4, 24.85, 104.45, 19.8, 116.45, 20.05, 110.75, 89.7, 48.7, 96.6, 74.3, 54.3, 74.85, 79.95, 20.05, 19.4, 54.9, 24.45, 89.65, 45.4, 75.7, 110.65, 20.55, 115.15, 58.55, 93.25, 19.35, 48.75, 109.05, 25.0, 54.9, 24.75, 91.15, 104.35, 66.05, 92.2, 105.2, 19.6, 30.4, 61.5, 69.4, 24.75, 91.05, 73.65, 19.4, 26.2, 43.85, 69.7, 38.55, 53.1, 20.65, 64.45, 25.1, 95.15, 79.35, 96.65, 75.5, 19.7, 20.5, 19.2, 98.35, 74.35, 51.35, 45.65, 20.3, 54.2, 90.65, 50.9, 25.05, 74.85, 20.5, 63.55, 47.95, 45.0, 90.05, 25.3, 24.3, 75.95, 19.7, 66.4, 35.75, 18.8, 19.4, 19.3, 67.45, 20.1, 58.9, 19.45, 50.5, 25.1, 60.7, 99.0, 104.4, 44.05, 24.1, 45.55, 93.8, 19.7, 70.65, 86.45, 114.1, 95.2, 88.55, 20.75, 44.65, 60.2, 55.45, 70.3, 60.4, 55.8, 31.1, 50.95, 69.1, 43.95, 86.5, 69.95, 90.95, 19.9, 20.15, 90.6, 92.0, 24.85, 36.0, 78.5, 19.95, 20.65, 30.5, 106.1, 20.5, 95.5, 64.6, 51.1, 89.1, 54.95, 20.45, 85.95, 60.35, 19.8, 85.35, 72.1, 81.05, 20.5, 111.8, 20.2, 19.7, 19.85, 60.5, 19.55, 20.9, 21.05, 71.5, 54.65, 19.2, 49.8, 20.5, 90.4, 90.25, 104.6, 50.2, 95.5, 95.4, 101.3, 53.1, 84.85, 34.25, 88.6, 60.15, 99.95, 70.7, 54.8, 54.8, 100.3, 53.6, 19.35, 85.6, 80.8, 19.6, 90.7, 69.75, 20.0, 95.25, 19.95, 80.85, 93.3, 46.35, 78.75, 83.55, 19.6, 67.85, 105.65, 75.5, 20.15, 45.2, 79.25, 85.7, 20.1, 61.8, 49.9, 20.4, 75.4, 108.15, 86.25, 95.7, 116.85, 105.75, 20.15, 19.6, 60.95, 25.05, 60.3, 63.95, 74.3, 70.6, 90.55, 19.45, 64.45, 69.65, 19.5, 110.5, 24.7, 77.4, 85.4, 47.6, 19.4, 103.85, 108.45, 81.0, 86.65, 92.95, 90.35, 25.15, 76.4, 19.55, 85.35, 24.8, 103.15, 100.75, 94.1, 19.35, 19.9, 101.05, 59.1, 55.85, 106.05, 84.1, 75.3, 24.7, 55.8, 39.7, 29.5, 20.15, 79.55, 24.8, 19.65, 94.05, 90.75, 78.85, 99.5, 80.55, 70.2, 59.45, 93.35, 44.95, 26.1, 20.2, 21.25, 59.4, 95.0, 61.9, 118.65, 64.45, 80.15, 20.2, 21.0, 20.45, 75.85, 80.45, 75.5, 44.45, 74.55, 48.15, 19.65, 20.15, 106.6, 91.0, 25.4, 69.95, 66.85, 20.15, 64.85, 74.85, 50.5, 72.9, 115.05, 19.0, 19.55, 101.1, 84.1, 24.15, 50.1, 74.6, 19.75, 106.8, 84.5, 25.05, 83.7, 96.6, 101.1, 20.2, 94.05, 81.0, 60.25, 60.85, 43.95, 86.05, 20.25, 19.4, 102.65, 19.9, 19.55, 95.5, 84.15, 103.2, 50.2, 19.95, 116.25, 31.2, 24.45, 84.2, 85.65, 21.2, 25.55, 20.2, 63.85, 61.95, 25.75, 58.2, 85.85, 104.9, 99.85, 19.55, 104.0, 104.4, 19.5, 25.25, 49.85, 108.95, 89.9, 82.0, 89.95, 79.35, 64.05, 101.15, 39.1, 34.6, 19.55, 104.45, 70.5, 20.35, 19.45, 69.9, 59.7, 78.35, 71.45, 45.85, 95.85, 35.7, 89.55, 24.95, 24.85, 100.8, 105.35, 19.65, 54.45, 70.5, 20.1, 69.35, 19.8, 74.4, 93.05, 51.2, 65.6, 80.55, 52.7, 20.85, 52.15, 114.95, 104.45, 113.65, 20.6, 91.55, 49.85, 19.8, 104.15, 48.2, 25.1, 100.15, 55.9, 64.4, 85.3, 107.45, 91.3, 85.95, 45.2, 79.2, 55.5, 90.25, 91.25, 100.9, 97.7, 69.85, 65.6, 104.65, 90.45, 63.7, 104.5, 20.1, 104.3, 93.25, 73.45, 20.7, 25.25, 100.5, 90.6, 89.4, 95.45, 20.45, 19.95, 109.15, 85.7, 102.05, 94.7, 64.4, 26.8, 66.05, 65.2, 85.05, 55.8, 19.95, 45.0, 114.9, 106.4, 46.1, 39.7, 20.05, 95.75, 24.4, 33.6, 90.45, 84.0, 67.4, 19.7, 80.35, 19.6, 54.2, 45.2, 75.1, 19.7, 72.75, 20.05, 39.2, 44.75, 82.65, 93.9, 117.15, 99.25, 112.55, 25.7, 90.3, 49.4, 19.4, 109.7, 61.25, 55.3, 103.75, 19.5, 39.5, 26.05, 91.05, 29.65, 50.2, 105.3, 55.45, 85.45, 19.8, 59.25, 90.7, 79.05, 90.7, 95.0, 30.25, 49.85, 93.0, 54.55, 19.7, 84.8, 94.45, 20.85, 60.0, 80.45, 84.95, 49.65, 20.2, 100.5, 35.75, 86.45, 53.8, 38.55, 39.9, 70.05, 20.1, 20.3, 35.65, 82.95, 55.65, 25.2, 50.8, 19.65, 59.8, 73.55, 61.4, 19.9, 19.45, 81.5, 109.55, 74.4, 74.9, 59.65, 110.45, 74.45, 24.55, 24.55, 90.65, 105.05, 20.45, 19.55, 19.7, 70.45, 85.65, 20.55, 97.95, 20.0, 25.25, 70.9, 19.85, 106.35, 99.5, 84.7, 86.05, 44.55, 75.85, 25.0, 45.0, 20.5, 90.45, 60.45, 78.45, 100.55, 20.35, 90.75, 20.25, 20.05, 19.6, 53.8, 70.2, 75.5, 20.35, 26.05, 20.6, 20.1, 24.3, 24.5, 110.5, 25.25, 90.1, 68.75, 19.2, 115.1, 99.65, 91.45, 84.75, 78.75, 20.25, 19.9, 97.75, 100.4, 24.45, 101.1, 50.9, 107.2, 92.2, 113.4, 40.55, 26.0, 111.95, 53.8, 72.1, 78.85, 70.75, 76.15, 39.1, 69.95, 20.05, 20.05, 19.45, 26.9, 19.2, 50.0, 60.0, 84.55, 45.45, 20.05, 115.55, 99.0, 50.55, 25.0, 91.55, 19.35, 24.85, 100.4, 25.0, 19.25, 108.25, 20.15, 101.3, 20.0, 105.3, 69.85, 65.25, 19.8, 19.6, 20.05, 49.4, 88.4, 100.6, 19.45, 20.3, 107.65, 80.45, 58.85, 109.6, 75.15, 73.0, 70.1, 98.65, 111.45, 114.9, 100.55, 20.4, 104.35, 80.45, 91.35, 19.9, 68.35, 79.1, 51.0, 80.55, 66.7, 86.4, 50.05, 25.7, 83.4, 84.65, 64.75, 100.15, 25.25, 113.0, 40.65, 94.95, 59.9, 19.8, 81.8, 20.0, 59.6, 25.0, 84.35, 55.55, 75.35, 59.3, 66.1, 18.8, 86.45, 52.1, 47.4, 109.15, 94.95, 93.55, 79.5, 115.05, 95.15, 105.4, 20.1, 20.05, 20.7, 20.35, 19.7, 85.45, 40.4, 105.2, 100.65, 91.0, 116.75, 59.1, 49.8, 19.3, 19.65, 81.4, 38.9, 87.95, 19.85, 96.35, 24.15, 19.1, 44.0, 60.6, 25.65, 53.95, 20.4, 29.35, 20.45, 95.1, 25.25, 44.9, 92.65, 43.7, 72.6, 18.95, 20.5, 19.95, 24.5, 20.6, 61.05, 106.65, 108.25, 20.4, 55.3, 20.25, 72.95, 89.45, 104.65, 75.2, 101.15, 68.75, 111.05, 99.0, 21.0, 19.4, 77.2, 19.45, 24.85, 41.35, 19.6, 84.45, 20.25, 19.65, 20.65, 99.3, 81.05, 67.6, 70.15, 115.0, 84.8, 19.7, 63.15, 74.0, 29.1, 50.05, 20.0, 74.65, 44.4, 85.4, 94.1, 108.9, 56.2, 26.1, 85.45, 88.95, 74.35, 48.85, 80.1, 56.05, 89.8, 19.1, 20.35, 106.05, 19.65, 59.85, 86.1, 19.45, 97.1, 36.65, 103.9, 19.75, 24.55, 48.7, 109.55, 20.65, 55.2, 24.05, 20.45, 19.25, 26.35, 43.8, 50.15, 20.45, 61.4, 70.75, 61.15, 20.25, 63.85, 98.7, 20.0, 19.3, 84.4, 25.1, 48.25, 19.85, 94.2, 62.15, 79.3, 56.25, 20.3, 99.0, 90.6, 85.9, 79.2, 70.35, 19.35, 50.15, 63.8, 20.55, 88.55, 101.4, 44.6, 63.75, 109.25, 84.6, 20.45, 85.75, 107.95, 22.95, 19.45, 19.7, 87.0, 79.95, 64.0, 64.9, 25.75, 90.15, 116.1, 104.95, 50.0, 20.45, 19.85, 19.95, 26.45, 63.4, 53.95, 95.1, 74.1, 35.5, 79.2, 48.8, 55.45, 25.4, 93.5, 63.9, 64.85, 63.8, 44.45, 19.95, 43.35, 49.65, 85.1, 89.8, 103.05, 116.0, 69.9, 95.1, 40.25, 25.75, 105.35, 113.6, 24.0, 19.4, 86.1, 102.65, 92.85, 97.75, 97.95, 19.95, 24.6, 50.95, 75.6, 80.75, 90.4, 60.25, 20.2, 64.15, 20.25, 99.0, 80.3, 19.55, 100.75, 53.75, 25.6, 58.35, 46.35, 113.75, 90.4, 109.3, 90.3, 65.25, 60.65, 24.1, 19.5, 85.95, 53.5, 25.45, 20.5, 20.85, 89.9, 26.0, 113.2, 69.05, 20.1, 109.65, 19.2, 90.0, 34.0, 20.4, 38.6, 25.25, 60.6, 74.75, 20.6, 20.4, 81.7, 20.3, 20.0, 25.0, 80.45, 19.75, 65.65, 71.0, 89.2, 86.75, 61.5, 25.1, 34.05, 19.95, 19.95, 89.7, 20.4, 26.3, 20.7, 19.45, 110.8, 69.3, 49.35, 20.35, 105.6, 64.45, 49.9, 65.65, 103.3, 44.45, 89.9, 55.05, 104.1, 106.6, 70.5, 19.6, 24.05, 38.1, 34.25, 100.05, 68.65, 45.8, 75.75, 96.4, 20.55, 50.95, 90.5, 79.4, 58.75, 59.45, 105.7, 53.3, 24.3, 59.9, 23.95, 20.15, 95.65, 81.0, 82.45, 20.5, 54.4, 58.6, 84.8, 61.4, 20.4, 20.15, 94.45, 79.8, 74.05, 49.15, 19.4, 113.65, 106.0, 25.95, 19.1, 100.55, 95.4, 75.15, 89.15, 107.9, 19.5, 24.95, 19.5, 69.95, 82.85, 19.0, 38.85, 20.35, 95.0, 74.4, 78.45, 74.3, 51.05, 19.2, 109.1, 76.45, 72.8, 18.95, 101.75, 75.45, 64.1, 25.65, 75.1, 95.85, 54.4, 72.75, 19.85, 19.05, 94.85, 46.25, 19.35, 69.6, 90.7, 101.4, 20.25, 48.8, 74.35, 19.35, 68.75, 100.2, 20.85, 19.35, 45.0, 25.5, 48.9, 19.6, 20.0, 81.3, 95.2, 83.3, 20.3, 89.85, 19.8, 54.65, 29.35, 19.15, 19.1, 80.55, 20.25, 106.0, 25.5, 79.6, 55.25, 88.05, 20.4, 117.6, 20.0, 19.65, 70.55, 65.8, 20.05, 80.0, 35.4, 79.6, 80.25, 20.45, 79.6, 24.7, 77.3, 29.75, 44.9, 29.8, 71.95, 20.75, 56.3, 105.25, 19.55, 84.45, 53.65, 29.9, 19.7, 43.7, 55.3, 19.85, 19.65, 49.45, 20.45, 39.7, 54.5, 111.6, 55.55, 20.55, 62.1, 104.5, 101.8, 110.6, 84.9, 93.2, 24.4, 70.55, 85.0, 85.8, 91.1, 20.1, 20.05, 74.8, 24.8, 100.85, 105.1, 20.4, 20.0, 79.4, 57.2, 58.6, 94.8, 102.5, 20.35, 84.9, 69.2, 20.85, 88.5, 35.0, 55.15, 50.95, 64.0, 80.2, 49.3, 84.35, 20.05, 117.2, 20.1, 109.95, 94.75, 80.0, 79.65, 25.2, 19.9, 44.8, 20.3, 19.2, 80.05, 107.35, 47.85, 70.8, 59.1, 25.55, 20.25, 75.55, 95.3, 70.25, 50.3, 19.85, 19.35, 25.0, 20.3, 75.35, 88.0, 43.8, 62.05, 20.1, 101.35, 84.05, 20.9, 105.9, 85.05, 44.1, 90.2, 53.45, 19.95, 74.65, 57.5, 19.65, 93.8, 89.25, 94.15, 55.6, 48.7, 104.9, 19.9, 19.4, 25.05, 84.45, 19.3, 79.85, 25.55, 68.4, 20.65, 55.15, 70.6, 19.95, 19.0, 44.1, 107.6, 61.55, 90.7, 99.25, 91.7, 100.7, 78.45, 84.3, 19.55, 20.45, 55.6, 86.8, 20.95, 20.05, 113.65, 59.5, 87.8, 41.9, 69.85, 56.3, 109.55, 92.15, 69.5, 97.0, 58.35, 70.4, 94.3, 19.55, 95.95, 94.8, 107.75, 54.6, 71.3, 19.5, 56.3, 90.55, 60.8, 98.8, 98.15, 35.35, 103.15, 107.75, 81.4, 95.7, 104.8, 70.95, 44.95, 97.65, 35.65, 85.25, 19.5, 25.1, 100.05, 55.7, 91.15, 83.85, 45.9, 25.1, 19.7, 91.5, 51.3, 21.1, 104.75, 85.75, 20.3, 100.75, 74.15, 78.55, 19.85, 50.7, 45.0, 77.8, 83.45, 94.8, 20.1, 59.9, 90.1, 70.95, 29.2, 46.6, 74.3, 69.3, 94.3, 76.45, 54.0, 104.25, 19.95, 24.95, 84.75, 19.75, 113.65, 44.9, 75.25, 24.6, 25.0, 20.95, 110.6, 55.5, 19.45, 84.85, 19.6, 53.45, 19.8, 112.1, 74.2, 69.0, 19.35, 19.8, 109.2, 79.15, 53.65, 100.2, 108.65, 40.65, 55.35, 105.6, 95.7, 83.2, 90.05, 97.65, 68.05, 102.1, 23.4, 71.05, 19.45, 59.45, 92.2, 19.85, 43.9, 90.5, 90.45, 84.6, 99.15, 19.95, 20.5, 62.1, 79.5, 19.55, 20.35, 51.7, 23.3, 65.1, 81.2, 74.5, 80.5, 60.3, 75.0, 90.15, 69.05, 59.7, 19.85, 40.75, 95.7, 46.3, 81.3, 20.0, 66.15, 19.6, 49.8, 101.75, 55.15, 103.95, 99.65, 73.7, 50.05, 60.25, 87.3, 54.25, 85.3, 50.0, 90.95, 72.25, 96.1, 19.85, 55.3, 20.1, 69.5, 25.15, 79.65, 71.25, 113.8, 24.55, 19.7, 100.5, 74.45, 104.1, 19.05, 25.0, 19.05, 81.9, 69.7, 90.15, 25.35, 24.65, 19.55, 60.0, 89.9, 19.4, 49.8, 24.1, 54.25, 109.9, 35.5, 87.55, 88.4, 50.8, 99.0, 96.55, 59.75, 111.5, 24.25, 20.5, 70.4, 30.55, 84.9, 54.5, 75.35, 44.45, 98.05, 63.9, 69.15, 64.65, 98.85, 89.6, 83.25, 70.25, 24.5, 20.1, 73.0, 61.4, 84.3, 19.9, 20.4, 50.75, 20.45, 75.75, 65.4, 59.75, 78.5, 48.95, 99.65, 18.25, 54.55, 40.65, 20.45, 24.8, 88.8, 20.05, 69.8, 77.15, 35.05, 108.1, 84.05, 20.2, 49.2, 24.6, 71.65, 104.9, 106.5, 75.5, 58.5, 78.9, 79.2, 109.45, 59.2, 29.15, 20.05, 66.5, 49.55, 73.6, 82.65, 49.0, 25.2, 25.45, 110.9, 77.75, 26.2, 19.9, 80.85, 56.35, 19.3, 50.4, 55.25, 19.1, 84.05, 105.2, 75.75, 95.3, 19.85, 69.1, 20.25, 54.75, 81.45, 80.2, 100.3, 90.95, 20.0, 79.85, 73.55, 19.3, 20.15, 44.55, 54.45, 19.65, 105.0, 88.7, 75.15, 20.25, 109.1, 30.75, 112.9, 94.05, 78.85, 55.3, 19.35, 25.35, 20.45, 19.35, 78.65, 74.75, 19.9, 58.35, 100.5, 20.05, 25.65, 95.0, 45.45, 20.0, 49.2, 83.25, 19.25, 19.65, 72.8, 109.65, 65.0, 114.1, 20.65, 86.95, 94.75, 25.35, 105.45, 25.4, 102.55, 24.0, 25.6, 73.5, 98.25, 54.4, 103.1, 34.2, 43.75, 100.65, 116.05, 82.0, 65.15, 44.8, 79.8, 88.85, 74.95, 106.85, 19.3, 56.1, 19.7, 51.3, 118.6, 24.15, 20.3, 115.5, 25.05, 109.1, 19.65, 111.3, 29.9, 80.6, 20.8, 89.95, 116.05, 19.55, 115.25, 24.8, 19.9, 81.25, 69.95, 86.4, 66.3, 94.65, 72.1, 34.7, 95.95, 44.8, 109.4, 71.05, 19.7, 40.25, 19.85, 68.25, 20.15, 50.95, 25.15, 20.25, 44.0, 20.25, 55.8, 88.9, 57.65, 79.15, 108.05, 94.8, 45.9, 102.6, 61.35, 57.55, 29.25, 19.6, 111.75, 106.5, 107.7, 19.3, 20.05, 69.95, 63.7, 50.9, 60.4, 79.25, 110.1, 90.7, 25.3, 85.2, 24.35, 25.1, 54.55, 96.6, 81.15, 38.5, 92.9, 84.7, 66.0, 20.75, 61.45, 54.5, 99.75, 109.75, 80.85, 20.3, 67.8, 19.8, 25.7, 56.15, 86.7, 20.4, 19.65, 54.35, 108.1, 54.45, 45.35, 59.0, 69.45, 64.95, 18.85, 19.8, 25.05, 114.3, 109.2, 45.05, 51.0, 110.45, 84.65, 60.05, 44.65, 93.25, 20.25, 25.45, 20.6, 94.1, 34.8, 60.75, 51.35, 64.05, 84.8, 50.15, 94.6, 59.75, 100.25, 98.9, 97.7, 60.25, 56.25, 46.2, 24.9, 63.35, 50.1, 50.15, 64.65, 79.6, 19.5, 99.55, 74.0, 38.9, 79.55, 46.3, 99.35, 95.8, 78.15, 26.1, 40.35, 79.2, 20.9, 49.9, 68.9, 20.25, 76.0, 74.0, 82.3, 89.4, 99.15, 29.45, 19.8, 59.15, 44.75, 90.8, 49.55, 106.7, 94.45, 19.45, 67.95, 65.25, 99.45, 20.35, 19.95, 77.4, 19.7, 99.7, 74.8, 19.15, 78.95, 62.85, 71.55, 94.95, 86.1, 19.55, 24.8, 84.05, 36.25, 98.6, 103.65, 92.9, 19.9, 20.1, 80.5, 39.85, 60.5, 103.85, 67.8, 24.85, 19.35, 89.0, 55.0, 76.15, 20.3, 117.35, 19.75, 45.2, 25.2, 89.75, 75.0, 49.95, 65.7, 67.05, 110.9, 87.95, 19.8, 75.7, 62.15, 115.15, 19.5, 86.55, 20.4, 19.8, 45.65, 56.4, 73.3, 101.35, 33.6, 20.7, 104.05, 20.25, 73.7, 108.75, 20.15, 19.75, 25.95, 70.05, 24.05, 84.75, 23.05, 59.95, 19.55, 19.6, 20.05, 85.55, 78.6, 116.8, 43.55, 60.8, 54.9, 65.2, 108.2, 92.0, 75.1, 25.05, 75.15, 19.5, 19.3, 112.2, 70.3, 19.6, 20.25, 80.65, 115.75, 80.6, 59.55, 19.05, 95.65, 19.95, 19.4, 36.1, 19.75, 64.1, 19.75, 19.7, 110.2, 106.35, 90.55, 65.9, 104.5, 52.5, 56.1, 88.75, 26.0, 99.4, 73.15, 54.65, 115.55, 104.45, 91.15, 89.7, 92.4, 19.9, 18.85, 25.75, 20.95, 97.05, 25.4, 19.7, 35.0, 101.25, 70.2, 90.95, 73.85, 88.05, 20.1, 110.3, 85.15, 60.95, 73.55, 46.0, 58.55, 24.6, 19.75, 86.35, 25.5, 19.0, 19.55, 110.1, 69.75, 50.6, 65.6, 82.1, 79.1, 90.65, 20.55, 75.75, 110.0, 20.85, 80.35, 70.15, 67.45, 20.75, 69.9, 51.1, 25.55, 60.0, 90.55, 76.4, 84.95, 110.1, 99.65, 45.4, 69.0, 48.65, 59.85, 80.65, 20.55, 66.4, 100.2, 44.55, 20.35, 91.8, 20.2, 50.35, 18.8, 20.45, 64.75, 98.7, 89.45, 58.75, 20.7, 85.6, 80.3, 79.8, 79.85, 54.1, 80.9, 24.5, 20.15, 20.05, 19.6, 114.3, 100.3, 80.0, 20.85, 89.95, 20.0, 48.75, 80.0, 20.35, 20.25, 19.4, 100.4, 57.95, 59.5, 19.2, 59.55, 103.95, 68.95, 103.1, 24.7, 110.2, 62.45, 89.55, 78.9, 20.35, 71.45, 46.35, 94.65, 49.9, 25.45, 20.75, 66.1, 75.4, 21.05, 69.35, 88.85, 97.0, 66.4, 69.2, 79.5, 100.65, 79.7, 61.4, 69.8, 40.55, 75.65, 90.7, 80.5, 60.6, 101.15, 24.95, 20.3, 60.0, 20.25, 44.75, 98.0, 107.7, 104.7, 93.9, 86.45, 19.4, 24.95, 75.0, 78.2, 94.2, 50.05, 69.55, 90.1, 44.65, 80.35, 98.1, 53.35, 19.55, 48.95, 54.2, 24.45, 40.15, 25.6, 70.35, 91.7, 89.2, 24.1, 53.85, 115.6, 19.75, 24.05, 25.3, 84.3, 89.75, 97.95, 20.0, 103.9, 20.7, 20.15, 26.0, 77.35, 66.05, 19.9, 68.15, 80.85, 75.5, 80.6, 83.2, 87.55, 109.4, 45.55, 20.7, 75.3, 93.4, 73.75, 88.15, 49.2, 19.65, 105.15, 49.0, 49.8, 20.95, 79.3, 19.5, 44.15, 105.5, 92.7, 26.25, 96.95, 20.45, 115.8, 108.2, 20.2, 54.9, 20.15, 90.35, 55.75, 114.6, 66.8, 100.3, 105.35, 85.2, 18.95, 69.8, 106.15, 20.55, 105.75, 25.25, 19.75, 104.85, 60.95, 81.15, 19.1, 20.8, 90.15, 90.1, 74.1, 118.75, 85.9, 95.0, 20.15, 101.3, 21.2, 24.2, 20.3, 85.3, 89.6, 56.25, 50.95, 115.85, 103.65, 26.1, 35.1, 99.1, 67.25, 25.0, 59.55, 77.8, 55.1, 24.15, 45.25, 20.25, 64.75, 54.6, 20.7, 94.75, 115.8, 49.45, 83.8, 95.35, 74.05, 89.6, 116.6, 54.2, 19.3, 65.05, 24.05, 18.75, 20.15, 20.0, 71.0, 93.6, 24.4, 65.25, 50.55, 70.7, 45.25, 70.3, 108.95, 26.45, 19.65, 19.05, 74.75, 75.8, 25.1, 44.45, 104.3, 89.0, 20.15, 36.15, 19.2, 19.25, 61.2, 20.45, 35.05, 44.0, 50.35, 20.0, 86.4, 58.4, 94.1, 108.9, 107.4, 90.85, 19.9, 66.4, 100.7, 25.6, 19.85, 20.75, 95.8, 94.65, 106.65, 45.85, 104.35, 55.45, 61.15, 78.95, 109.2, 61.3, 96.85, 40.55, 19.8, 108.25, 105.05, 90.45, 86.4, 66.9, 110.7, 20.0, 102.1, 70.15, 80.05, 49.2, 20.5, 38.25, 54.95, 96.6, 19.9, 19.9, 84.6, 85.25, 81.25, 115.5, 79.0, 94.65, 20.8, 59.5, 20.05, 100.45, 20.6, 20.3, 39.55, 20.45, 25.25, 91.25, 72.45, 19.7, 75.1, 25.0, 69.15, 91.55, 35.8, 113.15, 19.85, 19.8, 19.9, 19.7, 59.1, 91.15, 68.95, 51.55, 24.4, 96.8, 70.05, 19.5, 78.75, 69.2, 19.55, 103.65, 54.7, 54.15, 84.85, 20.0, 99.25, 19.35, 94.75, 114.05, 74.9, 19.8, 80.85, 54.65, 91.7, 118.6, 24.55, 19.45, 116.15, 80.6, 20.3, 99.8, 75.0, 19.9, 80.3, 84.3, 54.05, 104.9, 97.25, 83.05, 41.1, 45.0, 74.55, 40.2, 70.5, 19.75, 24.65, 104.25, 78.35, 109.7, 33.45, 94.6, 20.2, 20.3, 39.4, 69.15, 51.35, 100.05, 20.3, 94.45, 46.4, 104.05, 24.9, 59.6, 108.5, 40.55, 58.95, 20.75, 113.15, 48.8, 63.05, 100.85, 80.55, 64.4, 75.2, 84.9, 19.3, 74.65, 59.05, 69.1, 20.55, 76.55, 62.5, 94.9, 111.65, 19.9, 20.45, 106.05, 113.45, 92.55, 65.6, 84.35, 71.1, 85.15, 49.7, 30.2, 56.35, 107.55, 19.85, 95.9, 23.85, 83.85, 84.8, 76.1, 74.55, 39.2, 79.55, 19.6, 19.55, 39.15, 20.1, 99.95, 59.8, 49.75, 108.5, 60.15, 19.05, 84.0, 44.55, 103.45, 80.65, 57.2, 110.75, 24.7, 97.05, 76.35, 18.9, 74.45, 84.4, 24.4, 20.05, 55.5, 84.3, 100.2, 19.4, 20.4, 94.75, 44.35, 74.55, 73.6, 50.1, 53.0, 19.85, 24.35, 19.55, 25.05, 93.8, 103.75, 56.75, 20.8, 24.45, 25.6, 59.65, 83.3, 79.55, 24.45, 19.2, 29.8, 45.5, 30.05, 65.65, 74.05, 110.75, 19.7, 49.5, 43.95, 111.15, 20.6, 19.65, 115.8, 88.65, 94.5, 20.1, 34.65, 52.3, 65.0, 35.45, 19.7, 95.6, 19.85, 81.85, 109.3, 25.4, 69.8, 20.0, 109.9, 50.3, 101.5, 89.15, 19.4, 29.9, 78.8, 19.3, 96.8, 20.65, 19.8, 104.6, 80.05, 45.15, 73.15, 99.1, 105.35, 45.65, 79.95, 54.45, 25.1, 84.7, 75.85, 48.8, 35.2, 76.25, 24.9, 54.3, 66.3, 20.9, 75.35, 104.45, 49.45, 19.45, 92.15, 19.85, 100.25, 95.7, 93.15, 69.7, 19.8, 71.35, 20.75, 40.6, 20.35, 19.75, 54.4, 20.45, 66.15, 89.85, 45.05, 86.85, 96.75, 106.65, 110.15, 82.85, 20.1, 59.45, 58.6, 49.7, 65.85, 73.5, 113.65, 83.4, 65.65, 61.35, 85.9, 75.65, 70.9, 49.85, 20.1, 103.05, 99.7, 81.9, 66.2, 19.75, 72.6, 116.5, 106.8, 24.95, 89.25, 19.25, 104.55, 87.2, 30.75, 25.7, 86.2, 30.1, 99.35, 19.2, 20.1, 20.35, 25.65, 94.55, 94.4, 56.1, 68.25, 24.75, 76.25, 74.35, 54.15, 19.45, 34.95, 53.65, 104.0, 70.35, 64.85, 19.65, 45.9, 20.0, 44.8, 20.35, 45.8, 108.95, 64.35, 90.8, 24.95, 84.7, 70.8, 104.4, 101.5, 54.3, 103.95, 91.1, 19.95, 26.45, 75.1, 108.1, 110.15, 111.5, 106.5, 19.9, 111.1, 70.7, 24.85, 91.2, 65.6, 59.45, 109.95, 38.5, 92.55, 24.5, 19.7, 20.6, 58.0, 107.45, 65.5, 25.45, 100.15, 104.45, 21.15, 96.2, 44.4, 94.35, 20.3, 105.75, 81.15, 89.55, 54.75, 53.75, 105.75, 105.85, 64.2, 87.7, 89.3, 20.15, 20.05, 67.2, 94.55, 107.5, 73.0, 114.75, 76.05, 77.9, 90.65, 110.45, 68.7, 44.85, 29.8, 88.9, 58.75, 19.85, 86.9, 59.65, 66.4, 20.15, 108.1, 56.9, 109.6, 25.15, 79.15, 66.75, 48.8, 80.7, 20.55, 115.1, 59.7, 86.45, 33.7, 80.1, 104.05, 108.75, 41.1, 20.35, 105.9, 65.5, 40.45, 70.45, 78.8, 90.1, 82.45, 20.25, 66.25, 89.7, 64.55, 93.65, 73.6, 109.75, 61.45, 106.4, 81.9, 105.2, 54.6, 20.55, 19.7, 66.05, 54.05, 58.9, 96.9, 19.1, 50.0, 45.4, 85.45, 84.1, 66.25, 76.9, 74.6, 116.95, 40.65, 114.35, 69.7, 98.65, 61.65, 89.35, 95.4, 35.4, 19.95, 19.25, 20.4, 24.75, 25.35, 20.0, 59.75, 82.5, 20.35, 90.8, 104.95, 105.25, 23.75, 61.3, 75.8, 98.0, 52.0, 64.4, 45.8, 30.5, 69.15, 49.25, 39.35, 105.1, 20.1, 19.75, 19.75, 70.4, 20.45, 20.35, 86.2, 95.65, 103.8, 97.2, 63.55, 24.95, 99.0, 85.55, 94.0, 50.3, 95.0, 61.4, 80.55, 78.5, 114.3, 20.05, 62.65, 92.7, 100.45, 75.2, 84.75, 79.5, 19.8, 100.9, 95.3, 90.95, 54.5, 49.6, 25.0, 45.45, 107.75, 89.1, 44.75, 101.6, 103.15, 95.65, 75.1, 61.35, 19.7, 51.0, 88.85, 20.05, 65.1, 70.15, 20.75, 56.05, 19.95, 98.6, 79.0, 89.45, 74.2, 81.0, 49.6, 84.6, 84.2, 106.3, 69.05, 45.4, 99.35, 50.75, 87.1, 20.15, 98.7, 25.2, 55.7, 65.35, 25.3, 24.25, 60.5, 25.1, 20.05, 30.25, 20.2, 59.9, 25.15, 101.3, 76.95, 55.3, 92.45, 48.45, 19.35, 86.7, 55.7, 84.25, 64.65, 69.2, 54.65, 24.75, 23.95, 105.0, 59.85, 20.05, 92.15, 44.8, 20.9, 95.4, 80.35, 85.1, 115.05, 19.95, 86.15, 78.85, 86.55, 42.4, 24.25, 20.5, 19.6, 20.25, 20.6, 19.8, 80.2, 116.4, 31.65, 94.15, 20.65, 76.85, 20.15, 55.25, 82.15, 103.0, 95.1, 95.15, 79.8, 74.8, 20.45, 78.35, 19.1, 20.0, 19.95, 24.0, 19.15, 91.3, 19.15, 19.75, 75.5, 83.75, 19.4, 26.5, 19.15, 40.9, 80.25, 70.8, 60.2, 55.2, 54.15, 100.4, 62.55, 70.45, 85.5, 54.5, 20.75, 20.35, 91.0, 104.8, 51.1, 89.8, 20.55, 64.05, 74.85, 25.0, 20.3, 26.35, 54.7, 90.25, 20.65, 25.45, 19.5, 66.15, 69.1, 39.1, 20.05, 59.8, 48.6, 105.35, 25.1, 49.75, 94.75, 93.0, 71.9, 77.55, 19.85, 95.25, 25.05, 53.15, 20.15, 101.25, 100.55, 25.3, 71.8, 19.7, 49.85, 69.6, 19.75, 80.8, 64.2, 35.0, 19.2, 90.65, 20.0, 74.65, 61.2, 19.95, 54.8, 73.45, 51.45, 80.45, 85.3, 79.3, 76.5, 25.4, 86.85, 19.65, 45.55, 78.1, 19.3, 110.5, 20.3, 81.35, 55.3, 56.55, 19.7, 104.05, 52.85, 80.65, 24.65, 21.3, 110.2, 51.05, 19.8, 19.9, 87.3, 19.85, 89.4, 20.0, 20.05, 83.25, 102.9, 39.1, 114.5, 20.2, 55.8, 24.2, 72.8, 99.85, 99.5, 20.25, 26.0, 19.9, 19.05, 96.5, 19.85, 25.7, 20.3, 91.55, 39.4, 105.7, 70.25, 93.75, 60.0, 59.8, 90.65, 109.0, 68.1, 20.4, 81.95, 60.55, 65.6, 82.5, 82.3, 68.15, 20.3, 20.2, 89.2, 74.8, 20.2, 84.4, 25.15, 19.8, 50.85, 102.4, 55.5, 109.75, 106.4, 60.0, 88.8, 80.05, 75.55, 49.55, 23.9, 66.4, 18.8, 108.4, 85.95, 80.9, 111.8, 20.6, 44.6, 105.1, 115.15, 59.8, 26.3, 70.55, 20.05, 79.85, 90.05, 24.45, 59.95, 25.35, 34.3, 105.05, 19.3, 19.15, 51.4, 71.85, 75.4, 49.7, 78.75, 81.6, 70.4, 76.1, 94.0, 103.95, 19.95, 110.8, 96.1, 48.8, 50.55, 44.65, 19.45, 89.3, 19.25, 70.5, 19.65, 20.85, 19.65, 19.35, 44.0, 94.4, 75.4, 71.0, 21.2, 61.05, 79.95, 19.7, 20.3, 24.35, 19.75, 50.3, 50.25, 85.35, 51.65, 24.0, 59.85, 25.45, 23.9, 24.15, 75.7, 50.85, 91.6, 98.9, 85.0, 44.3, 80.2, 60.9, 34.2, 87.15, 54.3, 19.1, 112.75, 19.95, 19.5, 65.55, 78.8, 78.2, 105.25, 89.25, 20.65, 68.7, 78.65, 24.75, 19.75, 89.1, 84.7, 59.9, 19.95, 108.9, 33.6, 85.85, 34.85, 95.3, 84.6, 44.95, 24.7, 100.3, 25.45, 50.7, 55.0, 68.4, 55.05, 19.8, 84.45, 35.9, 80.75, 78.65, 61.75, 63.7, 87.6, 89.15, 20.0, 104.4, 20.05, 34.3, 20.65, 84.25, 19.65, 79.85, 20.2, 19.8, 50.35, 74.6, 79.15, 20.35, 21.05, 94.7, 74.95, 111.95, 19.85, 89.75, 20.05, 108.95, 19.65, 24.9, 93.2, 84.8, 71.75, 30.35, 54.85, 19.5, 24.2, 19.35, 100.65, 94.1, 74.55, 56.15, 20.35, 80.55, 61.25, 20.45, 18.9, 19.6, 45.2, 19.45, 25.45, 94.9, 29.3, 20.25, 110.5, 109.4, 19.95, 19.6, 76.6, 19.6, 85.3, 65.85, 20.05, 99.4, 20.0, 78.45, 25.1, 55.0, 71.1, 61.55, 45.9, 40.3, 87.1, 49.5, 73.8, 19.2, 25.0, 35.3, 76.75, 81.0, 105.55, 18.8, 24.9, 64.9, 61.35, 113.95, 90.15, 54.1, 49.8, 24.4, 95.0, 69.9, 39.95, 103.25, 94.25, 47.05, 20.55, 19.65, 70.2, 81.0, 75.9, 24.7, 110.25, 85.0, 19.75, 23.9, 19.95, 25.15, 54.15, 59.8, 83.85, 104.9, 75.3, 66.65, 109.5, 73.85, 19.3, 118.2, 51.45, 59.45, 19.5, 19.55, 93.55, 59.3, 109.8, 78.1, 39.9, 64.9, 53.4, 24.9, 44.7, 114.0, 20.25, 83.85, 20.2, 19.95, 20.35, 90.0, 54.2, 99.1, 66.9, 25.85, 91.05, 20.95, 109.2, 85.8, 19.65, 20.5, 89.65, 74.35, 49.45, 89.1, 75.15, 70.65, 90.05, 19.4, 88.75, 91.0, 90.8, 18.95, 102.4, 99.9, 88.7, 54.3, 55.7, 103.95, 20.15, 20.05, 91.95, 55.65, 74.7, 104.15, 83.65, 110.05, 25.5, 19.5, 80.7, 105.1, 25.15, 95.65, 80.8, 24.85, 54.75, 50.75, 20.15, 20.05, 71.6, 81.45, 58.4, 53.7, 19.6, 89.4, 84.2, 106.1, 25.75, 64.95, 85.45, 20.05, 20.7, 25.3, 100.6, 74.0, 99.4, 107.45, 83.6, 99.05, 80.1, 65.3, 89.55, 60.8, 74.5, 99.15, 19.25, 39.45, 44.85, 97.2, 110.55, 19.9, 76.95, 35.4, 20.45, 96.75, 54.2, 100.1, 45.25, 20.85, 33.45, 20.2, 85.9, 61.0, 86.9, 69.4, 20.35, 104.3, 44.95, 49.45, 20.6, 19.55, 93.5, 54.55, 20.05, 79.45, 79.85, 100.0, 19.6, 20.2, 50.4, 113.35, 80.0, 80.95, 24.9, 54.9, 109.25, 116.3, 19.9, 70.35, 25.6, 44.45, 100.15, 73.85, 70.1, 25.25, 21.05, 24.95, 64.5, 105.95, 75.85, 43.6, 91.25, 89.75, 104.4, 90.15, 40.3, 105.25, 104.0, 69.65, 74.3, 100.9, 20.25, 96.9, 104.1, 20.1, 56.55, 68.6, 69.05, 19.7, 20.05, 94.4, 54.95, 93.7, 110.25, 98.9, 80.45, 79.4, 62.8, 74.9, 74.85, 25.85, 68.3, 48.4, 105.05, 25.15, 19.5, 92.95, 20.7, 19.35, 104.35, 19.55, 74.05, 40.1, 20.1, 83.55, 56.85, 19.55, 106.15, 78.95, 49.75, 92.4, 58.2, 91.95, 65.25, 73.1, 59.75, 59.8, 116.6, 109.3, 101.4, 50.65, 56.15, 19.2, 83.0, 70.1, 108.3, 25.25, 45.35, 43.9, 79.3, 84.9, 79.25, 71.05, 53.75, 24.25, 44.25, 50.05, 20.15, 69.35, 19.35, 19.15, 61.0, 20.5, 50.2, 79.6, 24.9, 106.9, 101.35, 55.35, 50.55, 19.5, 90.65, 89.85, 79.0, 19.55, 19.9, 116.25, 87.75, 81.3, 44.3, 70.35, 44.45, 49.15, 29.45, 85.3, 69.1, 70.35, 20.6, 74.15, 75.05, 44.6, 21.45, 43.45, 20.05, 94.15, 19.55, 75.9, 64.15, 109.55, 110.8, 53.45, 69.95, 97.0, 90.6, 73.55, 94.35, 19.4, 19.75, 54.6, 29.8, 103.05, 20.3, 35.1, 105.7, 56.25, 60.35, 59.8, 99.65, 50.65, 60.9, 59.65, 64.7, 54.85, 91.35, 25.1, 34.0, 45.9, 20.5, 20.35, 36.1, 65.8, 20.35, 105.8, 96.75, 24.4, 73.05, 64.35, 20.5, 54.75, 51.15, 41.95, 54.35, 96.0, 61.45, 19.65, 19.0, 100.0, 98.7, 19.8, 73.8, 20.05, 106.2, 116.55, 99.7, 19.7, 19.5, 29.15, 55.0, 90.8, 51.0, 90.1, 59.05, 20.3, 72.95, 73.55, 84.3, 78.0, 72.1, 106.75, 19.25, 20.55, 20.0, 24.65, 103.5, 23.85, 25.8, 59.45, 20.05, 82.55, 81.25, 74.3, 109.7, 96.35, 66.6, 44.5, 80.1, 69.05, 20.4, 19.7, 50.1, 83.45, 86.65, 20.15, 19.4, 60.05, 20.35, 94.05, 84.1, 78.75, 55.55, 20.1, 70.3, 53.65, 20.75, 103.4, 50.8, 79.0, 74.6, 96.5, 20.1, 19.4, 77.55, 20.05, 19.85, 20.2, 67.45, 18.55, 29.75, 24.2, 23.55, 20.45, 92.3, 53.65, 39.65, 54.65, 104.8, 29.3, 83.85, 103.65, 99.05, 73.35, 100.05, 20.35, 43.95, 23.5, 70.7, 94.3, 29.15, 20.85, 37.7, 92.45, 44.15, 36.05, 50.25, 109.75, 20.3, 112.35, 94.3, 41.15, 74.65, 48.25, 76.15, 71.1, 96.55, 79.3, 89.6, 20.5, 106.3, 100.35, 85.6, 106.15, 51.1, 19.9, 25.7, 99.4, 69.7, 98.35, 85.45, 95.9, 100.75, 89.2, 25.75, 84.1, 79.3, 107.05, 20.05, 19.5, 45.3, 115.15, 72.95, 19.65, 19.55, 89.55, 50.35, 87.25, 20.8, 109.25, 20.35, 55.9, 79.2, 24.0, 101.35, 35.45, 79.4, 35.2, 19.65, 49.85, 68.75, 61.9, 79.9, 89.75, 19.4, 93.65, 19.9, 72.9, 25.6, 19.75, 55.7, 117.5, 19.85, 78.9, 20.65, 19.65, 79.75, 79.95, 29.9, 19.75, 45.0, 44.8, 51.1, 53.15, 24.7, 109.95, 20.8, 25.6, 108.4, 19.55, 85.1, 69.05, 70.15, 111.15, 89.35, 89.1, 91.25, 90.35, 105.55, 19.1, 20.4, 100.45, 85.7, 94.0, 69.85, 25.85, 71.1, 93.35, 50.55, 81.3, 20.7, 79.05, 19.05, 19.6, 20.2, 20.9, 103.6, 38.8, 88.4, 79.7, 19.3, 55.75, 19.95, 89.65, 45.85, 55.95, 69.0, 83.55, 65.7, 94.9, 61.9, 20.0, 67.7, 25.15, 92.85, 111.3, 60.6, 65.5, 19.95, 74.6, 94.6, 81.15, 89.05, 49.2, 19.45, 104.3, 86.05, 25.2, 35.15, 99.65, 105.35, 24.3, 80.7, 89.85, 61.1, 29.05, 99.7, 46.0, 80.4, 100.05, 94.0, 68.95, 68.45, 69.0, 43.85, 44.5, 18.7, 53.55, 114.6, 20.1, 85.5, 108.75, 97.85, 19.55, 84.05, 89.4, 19.7, 79.85, 74.45, 74.1, 18.8, 64.4, 55.8, 20.05, 99.15, 56.75, 104.15, 110.8, 35.75, 69.9, 89.2, 55.65, 50.7, 20.0, 19.1, 45.55, 101.05, 103.7, 36.25, 49.4, 19.9, 19.8, 45.05, 64.55, 86.25, 19.75, 89.1, 95.55, 102.6, 56.3, 94.2, 43.05, 94.0, 98.85, 64.35, 72.0, 49.7, 80.7, 24.2, 65.45, 74.35, 83.2, 25.0, 40.2, 108.35, 69.5, 76.0, 93.6, 100.55, 24.45, 89.55, 76.1, 80.5, 20.55, 105.4, 35.75, 95.1, 19.3, 63.1, 84.95, 93.4, 89.2, 85.2, 49.95, 20.65, 20.15, 19.2, 104.95, 103.5, 84.8, 95.05, 73.35, 64.1, 44.4, 20.05, 60.0, 69.5, 78.7, 60.65, 21.15, 84.8, 103.2, 29.6, 105.65]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Tenure & Monthly Charges Distribution\"}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('7a934241-bb7b-434e-a7b0-5b0be08cef1b');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = go.Figure()\n",
    "fig.add_trace(go.Histogram(x=tlc['tenure'], name='Tenure'))\n",
    "fig.add_trace(go.Histogram(x=mon_churn, name='Monthly Charges of Churn'))\n",
    "fig.add_trace(go.Histogram(x=mon_no_churn, name='Monthly Charges of No Churn'))\n",
    "fig.update_layout(title_text='Tenure & Monthly Charges Distribution')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BLVgEIi1jTQu"
   },
   "source": [
    "> One Hot Encoding the features with more than two categories. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nD7OV0gFdBAa"
   },
   "outputs": [],
   "source": [
    "tlc['TotalCharges'] = pd.to_numeric(tlc['TotalCharges'], errors='coerce') # TotalCharges needed to be a float\n",
    "fts = tlc[[col for col in tlc.columns if tlc[col].dtype == int or tlc[col].dtype == float]]\n",
    "tlc_dum = ['MultipleLines', 'InternetService', 'Contract', 'PaymentMethod']\n",
    "dummies = pd.get_dummies(tlc, columns=tlc_dum, drop_first=True)\n",
    "dummies.drop('Churn', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HqE1GJcPj2Rs"
   },
   "source": [
    "> TotalCharges had 11 missing values. Of those missing values the tenure was 0 meaning that the customer was new to the company so the missing values were filled with 0 since they haven't had any charges yet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hdLQscaCiz3T"
   },
   "outputs": [],
   "source": [
    "feats = dummies.drop(columns='customerID', axis=1) # customerID was an unecessary feature \n",
    "feats['TotalCharges']= feats['TotalCharges'].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q23avKZLsg6I"
   },
   "outputs": [],
   "source": [
    "columns = [col for col in feats.columns] # creating columns variable for future manipulation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AH55q6_MlD6x"
   },
   "source": [
    "# Models\n",
    "> Problem: Predict if a customer has churned or not\n",
    "\n",
    "> Algorithm: Logistic Regression \n",
    "* Logistic Regression is best used for binary classification\n",
    "\n",
    "> Split of Data:\n",
    "* Train: 70%\n",
    "* Test: 30%\n",
    "\n",
    "> Thresholds:\n",
    "* Thersholds help with balancing how accurate and sensitivity of a model.\n",
    "* Thresholds were compared at .25, .30, .35, .40, .45, adn .50 but when running the models .45 and .50 showed the best metrics for the model.\n",
    "\n",
    "\n",
    "> Metrics:\n",
    "* Accuracy\n",
    "* Confusion Matrix\n",
    "* Precision\n",
    "* Recall \n",
    "* F1 Score\n",
    "* Mean Absolute Error (MAE)\n",
    "* ROC Curve \n",
    "* Area Under the Curve (AUC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GIBlv2t-8JqW"
   },
   "source": [
    "## Baseline Model\n",
    "> The baseline model is for comparing with other models to see if the changes that were being made were improving the model. The only data engineering that was completed for this model was One Hot Encoding and Binarizing the categorical features."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jr5ueg9a8lNO"
   },
   "source": [
    "### Training\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XiTNBmWqm8uF"
   },
   "outputs": [],
   "source": [
    "xtrain, xtest, ytrain, ytest = train_test_split(feats, target, random_state=0) \n",
    "\n",
    "model = Logit(C=1.0, fit_intercept=True, solver='liblinear')\n",
    "results = model.fit(xtrain, ytrain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_wDEGp_69FXI"
   },
   "source": [
    "> Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 765
    },
    "colab_type": "code",
    "id": "NxmEGYMUjum6",
    "outputId": "4162cf7d-9854-4b44-9274-ddebc1448e0f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n",
      "Accuracy:  0.8023475956077244\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[3376  500]\n",
      " [ 544  862]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.87      0.87      3876\n",
      "           1       0.63      0.61      0.62      1406\n",
      "\n",
      "    accuracy                           0.80      5282\n",
      "   macro avg       0.75      0.74      0.74      5282\n",
      "weighted avg       0.80      0.80      0.80      5282\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.19765240439227566\n",
      "_________________\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n",
      "Accuracy:  0.8082165846270352\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[3484  392]\n",
      " [ 621  785]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.90      0.87      3876\n",
      "           1       0.67      0.56      0.61      1406\n",
      "\n",
      "    accuracy                           0.81      5282\n",
      "   macro avg       0.76      0.73      0.74      5282\n",
      "weighted avg       0.80      0.81      0.80      5282\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.19178341537296478\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "thresholds = [.45, .50] # variable to calculate churn predictions at different thresholds\n",
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtrain, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  model_report(ytrain, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GHAlr-cu9PCp"
   },
   "source": [
    "> ROC Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "US2DGfUHlkpy",
    "outputId": "4d4d6bfc-7b23-44fa-cda7-f3eee24b2997"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"6bde2a70-9581-44fd-b4ac-3a4890dc8f03\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"6bde2a70-9581-44fd-b4ac-3a4890dc8f03\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '6bde2a70-9581-44fd-b4ac-3a4890dc8f03',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.12899896800825594, 1.0], \"y\": [0.0, 0.6130867709815079, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('6bde2a70-9581-44fd-b4ac-3a4890dc8f03');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.742043901486626\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"9ad858b2-1afa-4610-b4e9-a5dd0c21138d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"9ad858b2-1afa-4610-b4e9-a5dd0c21138d\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '9ad858b2-1afa-4610-b4e9-a5dd0c21138d',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.10113519091847266, 1.0], \"y\": [0.0, 0.5583214793741109, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('9ad858b2-1afa-4610-b4e9-a5dd0c21138d');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7285931442278191\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtrain, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  plot_roc_curve(ytrain, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RFDiUvRp8WtG"
   },
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pQXk-sm5gSJK"
   },
   "outputs": [],
   "source": [
    "xtrain, xtest, ytrain, ytest = train_test_split(feats, target, random_state=0)\n",
    "\n",
    "model = Logit(C=1.0, fit_intercept=True, solver='liblinear')\n",
    "results = model.fit(xtest, ytest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jYLYnMby81B7"
   },
   "source": [
    "> Model Summary\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 765
    },
    "colab_type": "code",
    "id": "VfVhGpeN7KKr",
    "outputId": "9ccb46a2-8928-4e87-e59c-bef3607e7131"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n",
      "Accuracy:  0.7955706984667802\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[1136  162]\n",
      " [ 198  265]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.88      0.86      1298\n",
      "           1       0.62      0.57      0.60       463\n",
      "\n",
      "    accuracy                           0.80      1761\n",
      "   macro avg       0.74      0.72      0.73      1761\n",
      "weighted avg       0.79      0.80      0.79      1761\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.20442930153321975\n",
      "_________________\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n",
      "Accuracy:  0.80465644520159\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[1175  123]\n",
      " [ 221  242]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.91      0.87      1298\n",
      "           1       0.66      0.52      0.58       463\n",
      "\n",
      "    accuracy                           0.80      1761\n",
      "   macro avg       0.75      0.71      0.73      1761\n",
      "weighted avg       0.79      0.80      0.80      1761\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.19534355479841\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtest, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  model_report(ytest, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3zlGIIeE9YaR"
   },
   "source": [
    "> ROC Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "vRcsO1Dw7YU7",
    "outputId": "ad154998-a297-418a-d73d-6cc6ae52953f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"de6cb428-ef09-4740-b056-6f7db092769d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"de6cb428-ef09-4740-b056-6f7db092769d\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        'de6cb428-ef09-4740-b056-6f7db092769d',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.12480739599383667, 1.0], \"y\": [0.0, 0.572354211663067, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('de6cb428-ef09-4740-b056-6f7db092769d');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7237734078346151\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"74245479-573a-4037-ba68-51bfab26c9d8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"74245479-573a-4037-ba68-51bfab26c9d8\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '74245479-573a-4037-ba68-51bfab26c9d8',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.09476117103235747, 1.0], \"y\": [0.0, 0.5226781857451404, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('74245479-573a-4037-ba68-51bfab26c9d8');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7139585073563914\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtest, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  plot_roc_curve(ytest, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "o_-tzxZTrVlg"
   },
   "source": [
    "### Reflection\n",
    "> When first running the model I was surprised to get such an high accuracy but then I remembered that class imbalance hadn't been taken care of."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8IpJMqS99cga"
   },
   "source": [
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ewennd1hph-V"
   },
   "source": [
    "# SMOTE\n",
    "> Synthetic Minority Oversampling Technique\n",
    "* Uses an over-sampling techinque but doesn't reuse existing observations but instead it generates new observations based on the previous observation\n",
    "* Fixes the problem of class imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "Q2EsUvGvpfiz",
    "outputId": "b8c55518-4073-4aed-c699-0c24fc814f31"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    0.5\n",
      "0    0.5\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "smote = SMOTE(random_state=0) # initializing the SMOTE object\n",
    "feats, target = smote.fit_sample(feats, target)\n",
    "print(pd.Series(target).value_counts(normalize=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Y0OxlMPHtKSP"
   },
   "source": [
    "> Pie Charts shows that SMOTE technique worked in creating a class balance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "colab_type": "code",
    "id": "CX3G6PuUrdin",
    "outputId": "b1ac1b2d-c937-4e0d-8a93-23055e142256"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"76f1a87c-01c4-4de8-8b25-130d4a020742\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"76f1a87c-01c4-4de8-8b25-130d4a020742\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '76f1a87c-01c4-4de8-8b25-130d4a020742',\n",
       "                        [{\"hole\": 0.5, \"labels\": [\"Non_churn\", \"Churn\"], \"type\": \"pie\", \"values\": [5174, 5174]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Class Embalance of the Target\"}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('76f1a87c-01c4-4de8-8b25-130d4a020742');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "trace = go.Pie(labels=['Non_churn', 'Churn'], values=pd.Series(target).value_counts(), hole=.5)\n",
    "layout = go.Layout(title='Class Embalance of the Target')\n",
    "fig = go.Figure(data=trace, layout=layout)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 241
    },
    "colab_type": "code",
    "id": "BBfAK3WfsDPJ",
    "outputId": "c28da084-1f41-4c77-fc83-85a1bb859813"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>Partner</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>PhoneService</th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>OnlineBackup</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "      <th>PaperlessBilling</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "      <th>TotalCharges</th>\n",
       "      <th>MultipleLines_No phone service</th>\n",
       "      <th>MultipleLines_Yes</th>\n",
       "      <th>InternetService_Fiber optic</th>\n",
       "      <th>InternetService_No</th>\n",
       "      <th>Contract_One year</th>\n",
       "      <th>Contract_Two year</th>\n",
       "      <th>PaymentMethod_Credit card (automatic)</th>\n",
       "      <th>PaymentMethod_Electronic check</th>\n",
       "      <th>PaymentMethod_Mailed check</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   gender  ...  PaymentMethod_Mailed check\n",
       "0     0.0  ...                         0.0\n",
       "1     0.0  ...                         1.0\n",
       "2     0.0  ...                         1.0\n",
       "3     0.0  ...                         0.0\n",
       "4     0.0  ...                         0.0\n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feats = pd.DataFrame(data=feats, columns=columns) # convert to a dataframe\n",
    "feats.head() # the SMOTE technique doesn't leave the data in a dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZeZWy8AI8r8W"
   },
   "source": [
    "## Scaled Model\n",
    "\n",
    "> Algorithm: Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PboMrmH_t90f"
   },
   "source": [
    "> The numerical features were scaled between 0 and 1 because the categorical features were being represented by 0s and 1s so intuitively the numerical features needed to be scaled between 0 and 1 creating normalization through out the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FUBF4f1L6Ubb"
   },
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler()\n",
    "feats[['tenure', 'MonthlyCharges', 'TotalCharges']] = scaler.fit_transform(feats[['tenure', 'MonthlyCharges', 'TotalCharges']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 241
    },
    "colab_type": "code",
    "id": "XdEv7i5I4aQr",
    "outputId": "b2dbc770-01df-4436-fcb7-94cb79f08625"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>Partner</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>PhoneService</th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>OnlineBackup</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "      <th>PaperlessBilling</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "      <th>TotalCharges</th>\n",
       "      <th>MultipleLines_No phone service</th>\n",
       "      <th>MultipleLines_Yes</th>\n",
       "      <th>InternetService_Fiber optic</th>\n",
       "      <th>InternetService_No</th>\n",
       "      <th>Contract_One year</th>\n",
       "      <th>Contract_Two year</th>\n",
       "      <th>PaymentMethod_Credit card (automatic)</th>\n",
       "      <th>PaymentMethod_Electronic check</th>\n",
       "      <th>PaymentMethod_Mailed check</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.013889</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.115423</td>\n",
       "      <td>0.003437</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.472222</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.385075</td>\n",
       "      <td>0.217564</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.027778</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.354229</td>\n",
       "      <td>0.012453</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.239303</td>\n",
       "      <td>0.211951</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.027778</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.521891</td>\n",
       "      <td>0.017462</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   gender  ...  PaymentMethod_Mailed check\n",
       "0     0.0  ...                         0.0\n",
       "1     0.0  ...                         1.0\n",
       "2     0.0  ...                         1.0\n",
       "3     0.0  ...                         0.0\n",
       "4     0.0  ...                         0.0\n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feats.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "O7-OxfOK9jjx"
   },
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yyYpVOkm7kPy"
   },
   "outputs": [],
   "source": [
    "xtrain, xtest, ytrain, ytest = train_test_split(feats, target, random_state=0)\n",
    "\n",
    "results = model.fit(xtrain, ytrain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NbHVLCcT9qep"
   },
   "source": [
    "> Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 765
    },
    "colab_type": "code",
    "id": "O6faZhf_70lm",
    "outputId": "13c98499-c434-4511-d850-855d0dc4c586"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n",
      "Accuracy:  0.7689730704806081\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[2682 1202]\n",
      " [ 591 3286]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.69      0.75      3884\n",
      "           1       0.73      0.85      0.79      3877\n",
      "\n",
      "    accuracy                           0.77      7761\n",
      "   macro avg       0.78      0.77      0.77      7761\n",
      "weighted avg       0.78      0.77      0.77      7761\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.23102692951939183\n",
      "_________________\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n",
      "Accuracy:  0.7697461667310913\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[2831 1053]\n",
      " [ 734 3143]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.73      0.76      3884\n",
      "           1       0.75      0.81      0.78      3877\n",
      "\n",
      "    accuracy                           0.77      7761\n",
      "   macro avg       0.77      0.77      0.77      7761\n",
      "weighted avg       0.77      0.77      0.77      7761\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.23025383326890864\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtrain, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  model_report(ytrain, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hmRisa3x9tz3"
   },
   "source": [
    "> ROC Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "CNV4ARVr765B",
    "outputId": "c6c3b0c7-ef86-42ea-d092-55757822903d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"76500d88-51a2-4c44-acb6-0a9fb7f573f2\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"76500d88-51a2-4c44-acb6-0a9fb7f573f2\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '76500d88-51a2-4c44-acb6-0a9fb7f573f2',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.3094747682801236, 1.0], \"y\": [0.0, 0.8475625483621356, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('76500d88-51a2-4c44-acb6-0a9fb7f573f2');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.769043890041006\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"3dcd1107-74bc-4e8d-b207-4350fca21caa\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"3dcd1107-74bc-4e8d-b207-4350fca21caa\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '3dcd1107-74bc-4e8d-b207-4350fca21caa',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.2711122554067971, 1.0], \"y\": [0.0, 0.810678359556358, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('3dcd1107-74bc-4e8d-b207-4350fca21caa');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7697830520747805\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtrain, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  plot_roc_curve(ytrain, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lCZKK_k795Oh"
   },
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "k0ioiJis92uT"
   },
   "outputs": [],
   "source": [
    "results = model.fit(xtest, ytest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "o0j9jY2M-HYL"
   },
   "source": [
    "> Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 765
    },
    "colab_type": "code",
    "id": "7CYSu81V-Ef1",
    "outputId": "887530e8-7417-491e-fb92-135796c6856b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n",
      "Accuracy:  0.7750289911093932\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[ 906  384]\n",
      " [ 198 1099]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.70      0.76      1290\n",
      "           1       0.74      0.85      0.79      1297\n",
      "\n",
      "    accuracy                           0.78      2587\n",
      "   macro avg       0.78      0.77      0.77      2587\n",
      "weighted avg       0.78      0.78      0.77      2587\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.2249710088906069\n",
      "_________________\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n",
      "Accuracy:  0.7738693467336684\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[ 950  340]\n",
      " [ 245 1052]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.74      0.76      1290\n",
      "           1       0.76      0.81      0.78      1297\n",
      "\n",
      "    accuracy                           0.77      2587\n",
      "   macro avg       0.78      0.77      0.77      2587\n",
      "weighted avg       0.78      0.77      0.77      2587\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.22613065326633167\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtest, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  model_report(ytest, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "C3RTluex-thR"
   },
   "source": [
    "> ROC Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "jQQ7Njkn-Mvc",
    "outputId": "1830dab4-aa3f-4275-eaef-9a6b6b7976b8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"2a726a41-0a50-4cc4-afec-4e3cf2f18426\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"2a726a41-0a50-4cc4-afec-4e3cf2f18426\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '2a726a41-0a50-4cc4-afec-4e3cf2f18426',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.29767441860465116, 1.0], \"y\": [0.0, 0.8473400154202004, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('2a726a41-0a50-4cc4-afec-4e3cf2f18426');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7748327984077746\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"0173cb0c-b0d5-41b2-8533-706a95e8f37a\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"0173cb0c-b0d5-41b2-8533-706a95e8f37a\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '0173cb0c-b0d5-41b2-8533-706a95e8f37a',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.26356589147286824, 1.0], \"y\": [0.0, 0.8111025443330764, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('0173cb0c-b0d5-41b2-8533-706a95e8f37a');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7737683264301041\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtest, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  plot_roc_curve(ytest, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GTzJHyOivc8l"
   },
   "source": [
    "### Reflection\n",
    "> "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fGmE3Lf-_RAr"
   },
   "source": [
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "e8zmyXgW_BCB"
   },
   "source": [
    "# Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qIXwag7b2XQJ"
   },
   "source": [
    "> Interactions\n",
    "* the feature tenure and total charges have a relationship with one each other so I want to see if adding an interaction feature of the two would be beneficial."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OnPC0F64p-N0"
   },
   "outputs": [],
   "source": [
    "feats['TotalChargesXtenure'] = np.multiply(feats['TotalCharges'], feats['tenure'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "KgT0iQ_r1Ugs",
    "outputId": "f485ac4b-8cae-4710-8e8f-697c6f19cb99"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        0.000048\n",
       "1        0.102739\n",
       "2        0.000346\n",
       "3        0.132469\n",
       "4        0.000485\n",
       "           ...   \n",
       "10343    0.000498\n",
       "10344    0.032796\n",
       "10345    0.029060\n",
       "10346    0.038059\n",
       "10347    0.000291\n",
       "Name: TotalChargesXtenure, Length: 10348, dtype: float64"
      ]
     },
     "execution_count": 37,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feats['TotalChargesXtenure']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uC5zECLP_S7B"
   },
   "source": [
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "O0Z3Jho4_IUC"
   },
   "source": [
    "## Feature Engineered Model\n",
    "> Algorithm: Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dSOGlwYT_Wbu"
   },
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "45AD_dTNwptc"
   },
   "outputs": [],
   "source": [
    "xtrain, xtest, ytrain, ytest = train_test_split(feats, target, random_state=0)\n",
    "\n",
    "results = model.fit(xtrain, ytrain)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mzrr6Wug3DMo"
   },
   "source": [
    "> Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 765
    },
    "colab_type": "code",
    "id": "S--9qn5A_PPY",
    "outputId": "798bf30c-d26a-4b5f-bedd-3dd42fe896bf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n",
      "Accuracy:  0.7693596186058498\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[2686 1198]\n",
      " [ 592 3285]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.69      0.75      3884\n",
      "           1       0.73      0.85      0.79      3877\n",
      "\n",
      "    accuracy                           0.77      7761\n",
      "   macro avg       0.78      0.77      0.77      7761\n",
      "weighted avg       0.78      0.77      0.77      7761\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.23064038139415025\n",
      "_________________\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n",
      "Accuracy:  0.7723231542327019\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[2843 1041]\n",
      " [ 726 3151]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.73      0.76      3884\n",
      "           1       0.75      0.81      0.78      3877\n",
      "\n",
      "    accuracy                           0.77      7761\n",
      "   macro avg       0.77      0.77      0.77      7761\n",
      "weighted avg       0.77      0.77      0.77      7761\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.22767684576729802\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtrain, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  model_report(ytrain, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "IxbWrgGT2_-g",
    "outputId": "91ad328a-e1cc-4c92-c959-b3a28a458eeb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"2d3fab1b-d9ea-4922-9717-e80a3ca8b474\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"2d3fab1b-d9ea-4922-9717-e80a3ca8b474\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '2d3fab1b-d9ea-4922-9717-e80a3ca8b474',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.30844490216271886, 1.0], \"y\": [0.0, 0.8473046169718855, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('2d3fab1b-d9ea-4922-9717-e80a3ca8b474');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7694298574045833\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"e025ac7e-0769-46a7-a1a6-8d78f9678cbd\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"e025ac7e-0769-46a7-a1a6-8d78f9678cbd\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        'e025ac7e-0769-46a7-a1a6-8d78f9678cbd',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.2680226570545829, 1.0], \"y\": [0.0, 0.8127418106783596, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('e025ac7e-0769-46a7-a1a6-8d78f9678cbd');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7723595768118883\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtrain, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  plot_roc_curve(ytrain, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1uZY0mCI_db0"
   },
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "i129SyHh3Puy"
   },
   "source": [
    "> Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 765
    },
    "colab_type": "code",
    "id": "CcDU74ye_fPC",
    "outputId": "6aadaef7-f4de-4ba6-d651-3703e51e040c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n",
      "Accuracy:  0.7676845767298028\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[ 890  400]\n",
      " [ 201 1096]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.69      0.75      1290\n",
      "           1       0.73      0.85      0.78      1297\n",
      "\n",
      "    accuracy                           0.77      2587\n",
      "   macro avg       0.77      0.77      0.77      2587\n",
      "weighted avg       0.77      0.77      0.77      2587\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.23231542327019714\n",
      "_________________\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n",
      "Accuracy:  0.7750289911093932\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[ 947  343]\n",
      " [ 239 1058]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.73      0.76      1290\n",
      "           1       0.76      0.82      0.78      1297\n",
      "\n",
      "    accuracy                           0.78      2587\n",
      "   macro avg       0.78      0.77      0.77      2587\n",
      "weighted avg       0.78      0.78      0.77      2587\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.2249710088906069\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtest, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  model_report(ytest, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "lq1AO5hT3T_9",
    "outputId": "5887aed5-c077-47dd-fb73-6c2b21c7763d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"4ebb0f93-f5ac-45dd-b6b6-e1d58a90986b\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"4ebb0f93-f5ac-45dd-b6b6-e1d58a90986b\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '4ebb0f93-f5ac-45dd-b6b6-e1d58a90986b',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.31007751937984496, 1.0], \"y\": [0.0, 0.8450269853508096, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('4ebb0f93-f5ac-45dd-b6b6-e1d58a90986b');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7674747329854824\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"07caaa97-2369-4a08-b32c-a6507d17720f\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"07caaa97-2369-4a08-b32c-a6507d17720f\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '07caaa97-2369-4a08-b32c-a6507d17720f',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.26589147286821707, 1.0], \"y\": [0.0, 0.8157286044718581, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('07caaa97-2369-4a08-b32c-a6507d17720f');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7749185658018205\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(model, xtest, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  plot_roc_curve(ytest, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nYO67HIWxsKY"
   },
   "source": [
    "### Reflection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5drSZAGc3ekU"
   },
   "source": [
    "> No metrics were improved so adding the interaction feature with tenure and total charges didn't seem to help."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Bu34MvaPBAsB"
   },
   "source": [
    "## Hyperparameter Tuning Model\n",
    "> Algorithm: Logistic Regression\n",
    "\n",
    "> To find the best hyperparameters I used a grid search which is an exhaustive iteravtive approach to find the best model with the combination of hyperparameters."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hp69KLD5BIDw"
   },
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nlIKWYORzX6P"
   },
   "outputs": [],
   "source": [
    "C = np.arange(.000001, 2, .01) # interval of learning rates for regularization\n",
    "params = {'solver': ['liblinear'],\n",
    "          'penalty': ['l1','l2'], # lasso or ridge regularization\n",
    "          'C':C,\n",
    "          'class_weight': ['balanced'],\n",
    "          }\n",
    "\n",
    "gsm = GridSearchCV(estimator=Logit(), param_grid=params, cv=5)\n",
    "\n",
    "best_model = gsm.fit(xtrain, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 765
    },
    "colab_type": "code",
    "id": "rpv6UpiE5IDR",
    "outputId": "6e13be01-c4a8-41e4-e54e-c4961d3bdeb8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n",
      "Accuracy:  0.7691019198556887\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[2684 1200]\n",
      " [ 592 3285]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.82      0.69      0.75      3884\n",
      "           1       0.73      0.85      0.79      3877\n",
      "\n",
      "    accuracy                           0.77      7761\n",
      "   macro avg       0.78      0.77      0.77      7761\n",
      "weighted avg       0.78      0.77      0.77      7761\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.2308980801443113\n",
      "_________________\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n",
      "Accuracy:  0.7720654554825409\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[2841 1043]\n",
      " [ 726 3151]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.73      0.76      3884\n",
      "           1       0.75      0.81      0.78      3877\n",
      "\n",
      "    accuracy                           0.77      7761\n",
      "   macro avg       0.77      0.77      0.77      7761\n",
      "weighted avg       0.77      0.77      0.77      7761\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.2279345445174591\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(best_model, xtrain, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  model_report(ytrain, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "nsxJLiJB792S",
    "outputId": "4cc6c909-1a1a-4d0d-a7c6-036f95bb4118"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"24ababf0-e678-4930-aaef-75ef8547f6af\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"24ababf0-e678-4930-aaef-75ef8547f6af\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '24ababf0-e678-4930-aaef-75ef8547f6af',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.30895983522142123, 1.0], \"y\": [0.0, 0.8473046169718855, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('24ababf0-e678-4930-aaef-75ef8547f6af');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7691723908752321\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"5d0825af-9306-4e04-8e5b-4469277e0d1c\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"5d0825af-9306-4e04-8e5b-4469277e0d1c\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '5d0825af-9306-4e04-8e5b-4469277e0d1c',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.26853759011328526, 1.0], \"y\": [0.0, 0.8127418106783596, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('5d0825af-9306-4e04-8e5b-4469277e0d1c');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7721021102825372\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(best_model, xtrain, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  plot_roc_curve(ytrain, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hUJ4ZUy2BOo_"
   },
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3WzQoZAG92uj"
   },
   "outputs": [],
   "source": [
    "best_model = gsm.fit(xtest, ytest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 765
    },
    "colab_type": "code",
    "id": "VlVA2ZOp-nFO",
    "outputId": "b91b3b53-b501-4968-fc56-4f7d1c5bec9a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n",
      "Accuracy:  0.7645921917278701\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[ 893  397]\n",
      " [ 212 1085]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.69      0.75      1290\n",
      "           1       0.73      0.84      0.78      1297\n",
      "\n",
      "    accuracy                           0.76      2587\n",
      "   macro avg       0.77      0.76      0.76      2587\n",
      "weighted avg       0.77      0.76      0.76      2587\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.23540780827212987\n",
      "_________________\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n",
      "Accuracy:  0.7754155392346347\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[ 968  322]\n",
      " [ 259 1038]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.75      0.77      1290\n",
      "           1       0.76      0.80      0.78      1297\n",
      "\n",
      "    accuracy                           0.78      2587\n",
      "   macro avg       0.78      0.78      0.78      2587\n",
      "weighted avg       0.78      0.78      0.78      2587\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.22458446076536528\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(best_model, xtest, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  model_report(ytest, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "ShNsiMTJSZAZ",
    "outputId": "f2d6dba0-6aef-461c-ab3f-693042863299"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Thershold @  0.45 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"0c6086f0-c085-43d9-b4c8-6228c7894af9\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"0c6086f0-c085-43d9-b4c8-6228c7894af9\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '0c6086f0-c085-43d9-b4c8-6228c7894af9',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.3077519379844961, 1.0], \"y\": [0.0, 0.8365458750963762, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('0c6086f0-c085-43d9-b4c8-6228c7894af9');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.76439696855594\n",
      "\n",
      "Thershold @  0.5 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"4fc7908d-40de-4f3f-bfb6-5242ae324b09\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"4fc7908d-40de-4f3f-bfb6-5242ae324b09\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '4fc7908d-40de-4f3f-bfb6-5242ae324b09',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.2496124031007752, 1.0], \"y\": [0.0, 0.8003084040092521, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('4fc7908d-40de-4f3f-bfb6-5242ae324b09');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.7753480004542385\n"
     ]
    }
   ],
   "source": [
    "for x in thresholds:\n",
    "  pred = thresh_pred(best_model, xtest, threshold=x)\n",
    "  print('\\nThershold @ ', x, '\\n')\n",
    "  plot_roc_curve(ytest, pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5B_faEniyTQF"
   },
   "source": [
    "### Reflection\n",
    "> Since there was no huge change in accuracy from the previous model I can safely say that the model isn't overfitting. If the model was overfitting the accuracy of the previous model would have been really high for the training data and low for the testing data and the accuracy of hyperparameters tuned model would've changed significantlly from the previous model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Fzp9UEUV_kqK"
   },
   "source": [
    "# Logistic Regression using Deep Learning\n",
    "> First Attempt at modeling with neural networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pL0EPFyENVQ0"
   },
   "source": [
    "> TensorBoard Configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ASLRFpqyNUtc"
   },
   "outputs": [],
   "source": [
    "log_dir=\"logs/fit/\" + datetime.datetime.now().strftime(\"%Y.%m.%d\")\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vufChtSbT4Qx"
   },
   "source": [
    "> Creating the Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 88
    },
    "colab_type": "code",
    "id": "hN1avRRJ_ucC",
    "outputId": "63a712cb-29fb-4a15-bb66-f1e0b75c597f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    }
   ],
   "source": [
    "l1 = tf.keras.regularizers.l1(.001)\n",
    "lrnn = Sequential()\n",
    "lrnn.add(Dense(16, input_dim=xtrain.shape[1], activation='relu', kernel_regularizer=l1))\n",
    "lrnn.add(Dense(8, activation='relu'))\n",
    "lrnn.add(Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fTkzoLNktN-C"
   },
   "source": [
    "> Training the Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 105
    },
    "colab_type": "code",
    "id": "jIuhRHMeZY9P",
    "outputId": "378e706e-3b28-4608-b905-c24990acc234"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/nn_impl.py:183: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f6433f2f7f0>"
      ]
     },
     "execution_count": 51,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrnn.compile(loss='binary_crossentropy', \n",
    "              optimizer='adam', \n",
    "              metrics=['binary_accuracy',])\n",
    "\n",
    "lrnn.fit(xtrain, \n",
    "         ytrain, \n",
    "         epochs=150, \n",
    "         batch_size=10,\n",
    "         callbacks=[tensorboard_callback],\n",
    "         verbose=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 17
    },
    "colab_type": "code",
    "id": "60c2vIPdEsAU",
    "outputId": "f16f08a6-cb83-4f80-8eaf-b356c1604126"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div id=\"root\"></div>\n",
       "    <script>\n",
       "      (function() {\n",
       "        window.TENSORBOARD_ENV = window.TENSORBOARD_ENV || {};\n",
       "        window.TENSORBOARD_ENV[\"IN_COLAB\"] = true;\n",
       "        document.querySelector(\"base\").href = \"https://localhost:6006\";\n",
       "        function fixUpTensorboard(root) {\n",
       "          const tftb = root.querySelector(\"tf-tensorboard\");\n",
       "          // Disable the fragment manipulation behavior in Colab. Not\n",
       "          // only is the behavior not useful (as the iframe's location\n",
       "          // is not visible to the user), it causes TensorBoard's usage\n",
       "          // of `window.replace` to navigate away from the page and to\n",
       "          // the `localhost:<port>` URL specified by the base URI, which\n",
       "          // in turn causes the frame to (likely) crash.\n",
       "          tftb.removeAttribute(\"use-hash\");\n",
       "        }\n",
       "        function executeAllScripts(root) {\n",
       "          // When `script` elements are inserted into the DOM by\n",
       "          // assigning to an element's `innerHTML`, the scripts are not\n",
       "          // executed. Thus, we manually re-insert these scripts so that\n",
       "          // TensorBoard can initialize itself.\n",
       "          for (const script of root.querySelectorAll(\"script\")) {\n",
       "            const newScript = document.createElement(\"script\");\n",
       "            newScript.type = script.type;\n",
       "            newScript.textContent = script.textContent;\n",
       "            root.appendChild(newScript);\n",
       "            script.remove();\n",
       "          }\n",
       "        }\n",
       "        function setHeight(root, height) {\n",
       "          // We set the height dynamically after the TensorBoard UI has\n",
       "          // been initialized. This avoids an intermediate state in\n",
       "          // which the container plus the UI become taller than the\n",
       "          // final width and cause the Colab output frame to be\n",
       "          // permanently resized, eventually leading to an empty\n",
       "          // vertical gap below the TensorBoard UI. It's not clear\n",
       "          // exactly what causes this problematic intermediate state,\n",
       "          // but setting the height late seems to fix it.\n",
       "          root.style.height = `${height}px`;\n",
       "        }\n",
       "        const root = document.getElementById(\"root\");\n",
       "        fetch(\".\")\n",
       "          .then((x) => x.text())\n",
       "          .then((html) => void (root.innerHTML = html))\n",
       "          .then(() => fixUpTensorboard(root))\n",
       "          .then(() => executeAllScripts(root))\n",
       "          .then(() => setHeight(root, 800));\n",
       "      })();\n",
       "    </script>\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%tensorboard --logdir logs/fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    },
    "colab_type": "code",
    "id": "KyuTLSF8qet2",
    "outputId": "7b33f3b2-17bd-4008-f9f6-0cdb070d3e09"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8406133230253833\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[3215  669]\n",
      " [ 568 3309]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.83      0.84      3884\n",
      "           1       0.83      0.85      0.84      3877\n",
      "\n",
      "    accuracy                           0.84      7761\n",
      "   macro avg       0.84      0.84      0.84      7761\n",
      "weighted avg       0.84      0.84      0.84      7761\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.15938667697461667\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "pred = lrnn.predict_classes(xtrain)\n",
    "model_report(ytrain, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 559
    },
    "colab_type": "code",
    "id": "J9iFTSaRtrse",
    "outputId": "45ec9a99-3fba-4fe6-a225-d4322562eb83"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"3af927d0-6e3b-49f6-968c-71ede25ac654\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"3af927d0-6e3b-49f6-968c-71ede25ac654\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '3af927d0-6e3b-49f6-968c-71ede25ac654',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.17224510813594232, 1.0], \"y\": [0.0, 0.8534949703378901, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('3af927d0-6e3b-49f6-968c-71ede25ac654');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.8406249311009738\n"
     ]
    }
   ],
   "source": [
    "plot_roc_curve(ytrain, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 340
    },
    "colab_type": "code",
    "id": "bBLA26YrYg3O",
    "outputId": "a4d2a026-c794-477e-83ef-f17c46138773"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.8144568998840356\n",
      "-----------------\n",
      "Confusion Matrix: \n",
      " [[1017  273]\n",
      " [ 207 1090]]\n",
      "-----------------\n",
      "Classification Report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.79      0.81      1290\n",
      "           1       0.80      0.84      0.82      1297\n",
      "\n",
      "    accuracy                           0.81      2587\n",
      "   macro avg       0.82      0.81      0.81      2587\n",
      "weighted avg       0.82      0.81      0.81      2587\n",
      "\n",
      "-----------------\n",
      "MAE Score:  0.18554310011596445\n",
      "_________________\n"
     ]
    }
   ],
   "source": [
    "pred = lrnn.predict_classes(xtest)\n",
    "model_report(ytest, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 559
    },
    "colab_type": "code",
    "id": "dzlw9Li9YtDu",
    "outputId": "39ae7e0d-093d-4e98-a850-a6bc2dd1fa74"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"65d0b23c-fb6f-46e0-94c2-8ba0472918b4\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"65d0b23c-fb6f-46e0-94c2-8ba0472918b4\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '65d0b23c-fb6f-46e0-94c2-8ba0472918b4',\n",
       "                        [{\"line\": {\"color\": \"red\", \"dash\": \"solid\", \"width\": 2}, \"name\": \"ROC Curve\", \"type\": \"scatter\", \"x\": [0.0, 0.2116279069767442, 1.0], \"y\": [0.0, 0.8404009252120278, 1.0]}, {\"line\": {\"color\": \"black\", \"dash\": \"dash\", \"width\": 2}, \"name\": \"No name\", \"type\": \"scatter\", \"x\": [0, 1], \"y\": [0, 1]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"ROC Curve Churn Classificaiton\"}, \"xaxis\": {\"title\": {\"text\": \"False Positive Rate\"}}, \"yaxis\": {\"title\": {\"text\": \"True Positive Rate\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('65d0b23c-fb6f-46e0-94c2-8ba0472918b4');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC Score:  0.8143865091176419\n"
     ]
    }
   ],
   "source": [
    "plot_roc_curve(ytest, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wLtE97TAY9x2"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "WGq2sUbVARYp",
    "lGAR9aYaAYy-",
    "GIBlv2t-8JqW",
    "jr5ueg9a8lNO",
    "Ewennd1hph-V",
    "ZeZWy8AI8r8W",
    "O0Z3Jho4_IUC",
    "dSOGlwYT_Wbu",
    "Bu34MvaPBAsB",
    "Fzp9UEUV_kqK"
   ],
   "name": "index.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "learn-env",
   "language": "python",
   "name": "learn-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}