Data preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>color</th>\n",
       "      <th>size</th>\n",
       "      <th>price</th>\n",
       "      <th>classlabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>green</td>\n",
       "      <td>M</td>\n",
       "      <td>10.1</td>\n",
       "      <td>class1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>red</td>\n",
       "      <td>L</td>\n",
       "      <td>13.5</td>\n",
       "      <td>class2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blue</td>\n",
       "      <td>XL</td>\n",
       "      <td>15.3</td>\n",
       "      <td>class1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   color size price classlabel\n",
       "0  green    M  10.1     class1\n",
       "1    red    L  13.5     class2\n",
       "2   blue   XL  15.3     class1"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame([\n",
    "    ['green', 'M', '10.1', 'class1'],\n",
    "    ['red', 'L', '13.5', 'class2'],\n",
    "    ['blue', 'XL', '15.3', 'class1']])\n",
    "df.columns = ['color', 'size', 'price', 'classlabel']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>color</th>\n",
       "      <th>size</th>\n",
       "      <th>price</th>\n",
       "      <th>classlabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>green</td>\n",
       "      <td>1</td>\n",
       "      <td>10.1</td>\n",
       "      <td>class1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>red</td>\n",
       "      <td>2</td>\n",
       "      <td>13.5</td>\n",
       "      <td>class2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blue</td>\n",
       "      <td>3</td>\n",
       "      <td>15.3</td>\n",
       "      <td>class1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   color  size price classlabel\n",
       "0  green     1  10.1     class1\n",
       "1    red     2  13.5     class2\n",
       "2   blue     3  15.3     class1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "size_mapping = {\n",
    "    'XL': 3,\n",
    "    'L': 2,\n",
    "    'M': 1}\n",
    "df['size'] = df['size'].map(size_mapping)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     M\n",
       "1     L\n",
       "2    XL\n",
       "Name: size, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inv_size_mapping = {v:k for k,v in size_mapping.items()}\n",
    "df['size'].map(inv_size_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>color</th>\n",
       "      <th>size</th>\n",
       "      <th>price</th>\n",
       "      <th>classlabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>green</td>\n",
       "      <td>1</td>\n",
       "      <td>10.1</td>\n",
       "      <td>class1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>red</td>\n",
       "      <td>2</td>\n",
       "      <td>13.5</td>\n",
       "      <td>class2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blue</td>\n",
       "      <td>3</td>\n",
       "      <td>15.3</td>\n",
       "      <td>class1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   color  size price classlabel\n",
       "0  green     1  10.1     class1\n",
       "1    red     2  13.5     class2\n",
       "2   blue     3  15.3     class1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'class1': 0, 'class2': 1}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "class_mapping = {label:idx for idx, label in\n",
    "                enumerate(np.unique(df['classlabel']))}\n",
    "class_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>color</th>\n",
       "      <th>size</th>\n",
       "      <th>price</th>\n",
       "      <th>classlabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>green</td>\n",
       "      <td>1</td>\n",
       "      <td>10.1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>red</td>\n",
       "      <td>2</td>\n",
       "      <td>13.5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blue</td>\n",
       "      <td>3</td>\n",
       "      <td>15.3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   color  size price  classlabel\n",
       "0  green     1  10.1           0\n",
       "1    red     2  13.5           1\n",
       "2   blue     3  15.3           0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['classlabel'] = df['classlabel'].map(class_mapping)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>color</th>\n",
       "      <th>size</th>\n",
       "      <th>price</th>\n",
       "      <th>classlabel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>green</td>\n",
       "      <td>1</td>\n",
       "      <td>10.1</td>\n",
       "      <td>class1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>red</td>\n",
       "      <td>2</td>\n",
       "      <td>13.5</td>\n",
       "      <td>class2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>blue</td>\n",
       "      <td>3</td>\n",
       "      <td>15.3</td>\n",
       "      <td>class1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   color  size price classlabel\n",
       "0  green     1  10.1     class1\n",
       "1    red     2  13.5     class2\n",
       "2   blue     3  15.3     class1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inv_class_mapping = {v: k for k, v in class_mapping.items()}\n",
    "df['classlabel'] = df['classlabel'].map(inv_class_mapping)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0], dtype=int64)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "class_le = LabelEncoder()\n",
    "y = class_le.fit_transform(df['classlabel'].values)\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\zhuowu\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['class1', 'class2', 'class1'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_le.inverse_transform(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 1, '10.1'],\n",
       "       [2, 2, '13.5'],\n",
       "       [0, 3, '15.3']], dtype=object)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df[['color', 'size', 'price']].values\n",
    "color_le = LabelEncoder()\n",
    "X[:, 0] = color_le.fit_transform(X[:, 0])\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0. ,  1. ,  0. ,  1. , 10.1],\n",
       "       [ 0. ,  0. ,  1. ,  2. , 13.5],\n",
       "       [ 1. ,  0. ,  0. ,  3. , 15.3]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "ohe = OneHotEncoder(categorical_features=[0])\n",
    "ohe.fit_transform(X).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>size</th>\n",
       "      <th>price_10.1</th>\n",
       "      <th>price_13.5</th>\n",
       "      <th>price_15.3</th>\n",
       "      <th>color_blue</th>\n",
       "      <th>color_green</th>\n",
       "      <th>color_red</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   size  price_10.1  price_13.5  price_15.3  color_blue  color_green  \\\n",
       "0     1           1           0           0           0            1   \n",
       "1     2           0           1           0           0            0   \n",
       "2     3           0           0           1           1            0   \n",
       "\n",
       "   color_red  \n",
       "0          0  \n",
       "1          1  \n",
       "2          0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.get_dummies(df[['price', 'color', 'size']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class labels [1 2 3]\n"
     ]
    }
   ],
   "source": [
    "df_wine = pd.read_csv('wine_data.csv')\n",
    "df_wine.columns = ['Class label', 'Alcohol', \n",
    "                  'Malic acid', 'Ash',\n",
    "                  'Alcalinity of ash', 'Magnesium',\n",
    "                  'Total phenols', 'Flavanoids',\n",
    "                  'Nonflavanoid phenols', 'Proanthocyanins',\n",
    "                  'Color intensity', 'Hue',\n",
    "                  'OD280/OD315 of diluted wines',\n",
    "                  'Proline']\n",
    "print('Class labels', np.unique(df_wine['Class label']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.head of      Class label  Alcohol  Malic acid   Ash  Alcalinity of ash  Magnesium  \\\n",
       "0              1    13.20        1.78  2.14               11.2        100   \n",
       "1              1    13.16        2.36  2.67               18.6        101   \n",
       "2              1    14.37        1.95  2.50               16.8        113   \n",
       "3              1    13.24        2.59  2.87               21.0        118   \n",
       "4              1    14.20        1.76  2.45               15.2        112   \n",
       "5              1    14.39        1.87  2.45               14.6         96   \n",
       "6              1    14.06        2.15  2.61               17.6        121   \n",
       "7              1    14.83        1.64  2.17               14.0         97   \n",
       "8              1    13.86        1.35  2.27               16.0         98   \n",
       "9              1    14.10        2.16  2.30               18.0        105   \n",
       "10             1    14.12        1.48  2.32               16.8         95   \n",
       "11             1    13.75        1.73  2.41               16.0         89   \n",
       "12             1    14.75        1.73  2.39               11.4         91   \n",
       "13             1    14.38        1.87  2.38               12.0        102   \n",
       "14             1    13.63        1.81  2.70               17.2        112   \n",
       "15             1    14.30        1.92  2.72               20.0        120   \n",
       "16             1    13.83        1.57  2.62               20.0        115   \n",
       "17             1    14.19        1.59  2.48               16.5        108   \n",
       "18             1    13.64        3.10  2.56               15.2        116   \n",
       "19             1    14.06        1.63  2.28               16.0        126   \n",
       "20             1    12.93        3.80  2.65               18.6        102   \n",
       "21             1    13.71        1.86  2.36               16.6        101   \n",
       "22             1    12.85        1.60  2.52               17.8         95   \n",
       "23             1    13.50        1.81  2.61               20.0         96   \n",
       "24             1    13.05        2.05  3.22               25.0        124   \n",
       "25             1    13.39        1.77  2.62               16.1         93   \n",
       "26             1    13.30        1.72  2.14               17.0         94   \n",
       "27             1    13.87        1.90  2.80               19.4        107   \n",
       "28             1    14.02        1.68  2.21               16.0         96   \n",
       "29             1    13.73        1.50  2.70               22.5        101   \n",
       "..           ...      ...         ...   ...                ...        ...   \n",
       "147            3    13.32        3.24  2.38               21.5         92   \n",
       "148            3    13.08        3.90  2.36               21.5        113   \n",
       "149            3    13.50        3.12  2.62               24.0        123   \n",
       "150            3    12.79        2.67  2.48               22.0        112   \n",
       "151            3    13.11        1.90  2.75               25.5        116   \n",
       "152            3    13.23        3.30  2.28               18.5         98   \n",
       "153            3    12.58        1.29  2.10               20.0        103   \n",
       "154            3    13.17        5.19  2.32               22.0         93   \n",
       "155            3    13.84        4.12  2.38               19.5         89   \n",
       "156            3    12.45        3.03  2.64               27.0         97   \n",
       "157            3    14.34        1.68  2.70               25.0         98   \n",
       "158            3    13.48        1.67  2.64               22.5         89   \n",
       "159            3    12.36        3.83  2.38               21.0         88   \n",
       "160            3    13.69        3.26  2.54               20.0        107   \n",
       "161            3    12.85        3.27  2.58               22.0        106   \n",
       "162            3    12.96        3.45  2.35               18.5        106   \n",
       "163            3    13.78        2.76  2.30               22.0         90   \n",
       "164            3    13.73        4.36  2.26               22.5         88   \n",
       "165            3    13.45        3.70  2.60               23.0        111   \n",
       "166            3    12.82        3.37  2.30               19.5         88   \n",
       "167            3    13.58        2.58  2.69               24.5        105   \n",
       "168            3    13.40        4.60  2.86               25.0        112   \n",
       "169            3    12.20        3.03  2.32               19.0         96   \n",
       "170            3    12.77        2.39  2.28               19.5         86   \n",
       "171            3    14.16        2.51  2.48               20.0         91   \n",
       "172            3    13.71        5.65  2.45               20.5         95   \n",
       "173            3    13.40        3.91  2.48               23.0        102   \n",
       "174            3    13.27        4.28  2.26               20.0        120   \n",
       "175            3    13.17        2.59  2.37               20.0        120   \n",
       "176            3    14.13        4.10  2.74               24.5         96   \n",
       "\n",
       "     Total phenols  Flavanoids  Nonflavanoid phenols  Proanthocyanins  \\\n",
       "0             2.65        2.76                  0.26             1.28   \n",
       "1             2.80        3.24                  0.30             2.81   \n",
       "2             3.85        3.49                  0.24             2.18   \n",
       "3             2.80        2.69                  0.39             1.82   \n",
       "4             3.27        3.39                  0.34             1.97   \n",
       "5             2.50        2.52                  0.30             1.98   \n",
       "6             2.60        2.51                  0.31             1.25   \n",
       "7             2.80        2.98                  0.29             1.98   \n",
       "8             2.98        3.15                  0.22             1.85   \n",
       "9             2.95        3.32                  0.22             2.38   \n",
       "10            2.20        2.43                  0.26             1.57   \n",
       "11            2.60        2.76                  0.29             1.81   \n",
       "12            3.10        3.69                  0.43             2.81   \n",
       "13            3.30        3.64                  0.29             2.96   \n",
       "14            2.85        2.91                  0.30             1.46   \n",
       "15            2.80        3.14                  0.33             1.97   \n",
       "16            2.95        3.40                  0.40             1.72   \n",
       "17            3.30        3.93                  0.32             1.86   \n",
       "18            2.70        3.03                  0.17             1.66   \n",
       "19            3.00        3.17                  0.24             2.10   \n",
       "20            2.41        2.41                  0.25             1.98   \n",
       "21            2.61        2.88                  0.27             1.69   \n",
       "22            2.48        2.37                  0.26             1.46   \n",
       "23            2.53        2.61                  0.28             1.66   \n",
       "24            2.63        2.68                  0.47             1.92   \n",
       "25            2.85        2.94                  0.34             1.45   \n",
       "26            2.40        2.19                  0.27             1.35   \n",
       "27            2.95        2.97                  0.37             1.76   \n",
       "28            2.65        2.33                  0.26             1.98   \n",
       "29            3.00        3.25                  0.29             2.38   \n",
       "..             ...         ...                   ...              ...   \n",
       "147           1.93        0.76                  0.45             1.25   \n",
       "148           1.41        1.39                  0.34             1.14   \n",
       "149           1.40        1.57                  0.22             1.25   \n",
       "150           1.48        1.36                  0.24             1.26   \n",
       "151           2.20        1.28                  0.26             1.56   \n",
       "152           1.80        0.83                  0.61             1.87   \n",
       "153           1.48        0.58                  0.53             1.40   \n",
       "154           1.74        0.63                  0.61             1.55   \n",
       "155           1.80        0.83                  0.48             1.56   \n",
       "156           1.90        0.58                  0.63             1.14   \n",
       "157           2.80        1.31                  0.53             2.70   \n",
       "158           2.60        1.10                  0.52             2.29   \n",
       "159           2.30        0.92                  0.50             1.04   \n",
       "160           1.83        0.56                  0.50             0.80   \n",
       "161           1.65        0.60                  0.60             0.96   \n",
       "162           1.39        0.70                  0.40             0.94   \n",
       "163           1.35        0.68                  0.41             1.03   \n",
       "164           1.28        0.47                  0.52             1.15   \n",
       "165           1.70        0.92                  0.43             1.46   \n",
       "166           1.48        0.66                  0.40             0.97   \n",
       "167           1.55        0.84                  0.39             1.54   \n",
       "168           1.98        0.96                  0.27             1.11   \n",
       "169           1.25        0.49                  0.40             0.73   \n",
       "170           1.39        0.51                  0.48             0.64   \n",
       "171           1.68        0.70                  0.44             1.24   \n",
       "172           1.68        0.61                  0.52             1.06   \n",
       "173           1.80        0.75                  0.43             1.41   \n",
       "174           1.59        0.69                  0.43             1.35   \n",
       "175           1.65        0.68                  0.53             1.46   \n",
       "176           2.05        0.76                  0.56             1.35   \n",
       "\n",
       "     Color intensity   Hue  OD280/OD315 of diluted wines  Proline  \n",
       "0           4.380000  1.05                          3.40     1050  \n",
       "1           5.680000  1.03                          3.17     1185  \n",
       "2           7.800000  0.86                          3.45     1480  \n",
       "3           4.320000  1.04                          2.93      735  \n",
       "4           6.750000  1.05                          2.85     1450  \n",
       "5           5.250000  1.02                          3.58     1290  \n",
       "6           5.050000  1.06                          3.58     1295  \n",
       "7           5.200000  1.08                          2.85     1045  \n",
       "8           7.220000  1.01                          3.55     1045  \n",
       "9           5.750000  1.25                          3.17     1510  \n",
       "10          5.000000  1.17                          2.82     1280  \n",
       "11          5.600000  1.15                          2.90     1320  \n",
       "12          5.400000  1.25                          2.73     1150  \n",
       "13          7.500000  1.20                          3.00     1547  \n",
       "14          7.300000  1.28                          2.88     1310  \n",
       "15          6.200000  1.07                          2.65     1280  \n",
       "16          6.600000  1.13                          2.57     1130  \n",
       "17          8.700000  1.23                          2.82     1680  \n",
       "18          5.100000  0.96                          3.36      845  \n",
       "19          5.650000  1.09                          3.71      780  \n",
       "20          4.500000  1.03                          3.52      770  \n",
       "21          3.800000  1.11                          4.00     1035  \n",
       "22          3.930000  1.09                          3.63     1015  \n",
       "23          3.520000  1.12                          3.82      845  \n",
       "24          3.580000  1.13                          3.20      830  \n",
       "25          4.800000  0.92                          3.22     1195  \n",
       "26          3.950000  1.02                          2.77     1285  \n",
       "27          4.500000  1.25                          3.40      915  \n",
       "28          4.700000  1.04                          3.59     1035  \n",
       "29          5.700000  1.19                          2.71     1285  \n",
       "..               ...   ...                           ...      ...  \n",
       "147         8.420000  0.55                          1.62      650  \n",
       "148         9.400000  0.57                          1.33      550  \n",
       "149         8.600000  0.59                          1.30      500  \n",
       "150        10.800000  0.48                          1.47      480  \n",
       "151         7.100000  0.61                          1.33      425  \n",
       "152        10.520000  0.56                          1.51      675  \n",
       "153         7.600000  0.58                          1.55      640  \n",
       "154         7.900000  0.60                          1.48      725  \n",
       "155         9.010000  0.57                          1.64      480  \n",
       "156         7.500000  0.67                          1.73      880  \n",
       "157        13.000000  0.57                          1.96      660  \n",
       "158        11.750000  0.57                          1.78      620  \n",
       "159         7.650000  0.56                          1.58      520  \n",
       "160         5.880000  0.96                          1.82      680  \n",
       "161         5.580000  0.87                          2.11      570  \n",
       "162         5.280000  0.68                          1.75      675  \n",
       "163         9.580000  0.70                          1.68      615  \n",
       "164         6.620000  0.78                          1.75      520  \n",
       "165        10.680000  0.85                          1.56      695  \n",
       "166        10.260000  0.72                          1.75      685  \n",
       "167         8.660000  0.74                          1.80      750  \n",
       "168         8.500000  0.67                          1.92      630  \n",
       "169         5.500000  0.66                          1.83      510  \n",
       "170         9.899999  0.57                          1.63      470  \n",
       "171         9.700000  0.62                          1.71      660  \n",
       "172         7.700000  0.64                          1.74      740  \n",
       "173         7.300000  0.70                          1.56      750  \n",
       "174        10.200000  0.59                          1.56      835  \n",
       "175         9.300000  0.60                          1.62      840  \n",
       "176         9.200000  0.61                          1.60      560  \n",
       "\n",
       "[177 rows x 14 columns]>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_wine.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X, y = df_wine.iloc[:, 1:].values, df_wine.iloc[:, 0].values\n",
    "X_train, X_test, y_train, y_test= train_test_split(X, y, test_size=0.3, random_state=0, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Standardized:  [-1.46385011 -0.87831007 -0.29277002  0.29277002  0.87831007  1.46385011]\n",
      "('Normalized:', array([0. , 0.2, 0.4, 0.6, 0.8, 1. ]))\n"
     ]
    }
   ],
   "source": [
    "ex = np.array([0, 1, 2, 3, 4, 5])\n",
    "print('Standardized: ', (ex - ex.mean())/ex.std())\n",
    "print(('Normalized:', (ex - ex.min())/(ex.max()-ex.min())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "stdsc = StandardScaler()\n",
    "X_train_std = stdsc.fit_transform(X_train)\n",
    "X_test_std = stdsc.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy:  1.0\n",
      "Test accuracy:  0.9814814814814815\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression(penalty='l1', C=1.0)\n",
    "lr.fit(X_train_std, y_train)\n",
    "print(\"Training accuracy: \" , lr.score(X_train_std, y_train))\n",
    "print(\"Test accuracy: \" , lr.score(X_test_std, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.27159003, -1.46052354, -2.24848949])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.05160174,  0.30367127,  0.17398168, -0.64330424,  0.        ,\n",
       "         0.        ,  1.17108649,  0.        ,  0.        ,  0.        ,\n",
       "         0.        ,  0.69748592,  3.13099643],\n",
       "       [-1.08367173, -0.57823561, -0.45603117,  0.08442115, -0.13671913,\n",
       "         0.05484033,  0.57864928,  0.        ,  0.13457285, -1.9877204 ,\n",
       "         0.91255318,  0.        , -2.99165172],\n",
       "       [ 0.16292729,  0.11432326,  0.34184394,  0.        ,  0.        ,\n",
       "         0.        , -2.55587708,  0.        ,  0.        ,  1.5801097 ,\n",
       "        -0.90065178, -0.39879688,  0.        ]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'list' object cannot be interpreted as an integer",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-31-2a19b82f99f6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[0mweights\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mweights\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m \u001b[1;32mfor\u001b[0m \u001b[0mcolumn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolor\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mweights\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m     \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolumn\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdf_wine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcolumn\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcolor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maxhline\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'black'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlinestyle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'--'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlinewidth\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: 'list' object cannot be interpreted as an integer"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAADYBJREFUeJzt3HGI33d9x/Hny8ROprWO5QRJou1YuhrKoO7oOoRZ0Y20fyT/FEmguEppwK0OZhE6HCr1rylDELJptolT0Fr9Qw+J5A9X6RAjudJZmpTALTpzROhZu/5TtGZ774/fT++4XHLf3v3uLt77+YDA7/v7fX6/e+fD3TO/fH/3+6WqkCRtf6/a6gEkSZvD4EtSEwZfkpow+JLUhMGXpCYMviQ1sWrwk3wuyXNJnrnC7Uny6SRzSZ5O8rbJjylJWq8hz/A/Dxy4yu13AfvGf44C/7T+sSRJk7Zq8KvqCeBnV1lyCPhCjZwC3pDkTZMaUJI0GTsn8Bi7gQtLjufH1/1k+cIkRxn9L4DXvva1f3TLLbdM4MtLUh9PPvnkT6tqai33nUTws8J1K35eQ1UdB44DTE9P1+zs7AS+vCT1keS/13rfSfyWzjywd8nxHuDiBB5XkjRBkwj+DPDe8W/r3AG8WFWXnc6RJG2tVU/pJPkycCewK8k88FHg1QBV9RngBHA3MAe8BLxvo4aVJK3dqsGvqiOr3F7AX01sIknShvCdtpLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDUxKPhJDiQ5l2QuycMr3P7mJI8neSrJ00nunvyokqT1WDX4SXYAx4C7gP3AkST7ly37O+CxqroNOAz846QHlSStz5Bn+LcDc1V1vqpeBh4FDi1bU8Drx5dvAC5ObkRJ0iQMCf5u4MKS4/nxdUt9DLg3yTxwAvjASg+U5GiS2SSzCwsLaxhXkrRWQ4KfFa6rZcdHgM9X1R7gbuCLSS577Ko6XlXTVTU9NTX1yqeVJK3ZkODPA3uXHO/h8lM29wOPAVTV94DXALsmMaAkaTKGBP80sC/JTUmuY/Si7MyyNT8G3gWQ5K2Mgu85G0m6hqwa/Kq6BDwInASeZfTbOGeSPJLk4HjZQ8ADSX4AfBm4r6qWn/aRJG2hnUMWVdUJRi/GLr3uI0sunwXePtnRJEmT5DttJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNDAp+kgNJziWZS/LwFda8J8nZJGeSfGmyY0qS1mvnaguS7ACOAX8GzAOnk8xU1dkla/YBfwu8vapeSPLGjRpYkrQ2Q57h3w7MVdX5qnoZeBQ4tGzNA8CxqnoBoKqem+yYkqT1GhL83cCFJcfz4+uWuhm4Ocl3k5xKcmClB0pyNMlsktmFhYW1TSxJWpMhwc8K19Wy453APuBO4AjwL0necNmdqo5X1XRVTU9NTb3SWSVJ6zAk+PPA3iXHe4CLK6z5RlX9sqp+CJxj9A+AJOkaMST4p4F9SW5Kch1wGJhZtubrwDsBkuxidIrn/CQHlSStz6rBr6pLwIPASeBZ4LGqOpPkkSQHx8tOAs8nOQs8Dnyoqp7fqKElSa9cqpafjt8c09PTNTs7uyVfW5J+UyV5sqqm13Jf32krSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSE4OCn+RAknNJ5pI8fJV19ySpJNOTG1GSNAmrBj/JDuAYcBewHziSZP8K664H/hr4/qSHlCSt35Bn+LcDc1V1vqpeBh4FDq2w7uPAJ4CfT3A+SdKEDAn+buDCkuP58XW/luQ2YG9VffNqD5TkaJLZJLMLCwuveFhJ0toNCX5WuK5+fWPyKuBTwEOrPVBVHa+q6aqanpqaGj6lJGndhgR/Hti75HgPcHHJ8fXArcB3kvwIuAOY8YVbSbq2DAn+aWBfkpuSXAccBmZ+dWNVvVhVu6rqxqq6ETgFHKyq2Q2ZWJK0JqsGv6ouAQ8CJ4Fngceq6kySR5Ic3OgBJUmTsXPIoqo6AZxYdt1HrrD2zvWPJUmaNN9pK0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqYlDwkxxIci7JXJKHV7j9g0nOJnk6ybeTvGXyo0qS1mPV4CfZARwD7gL2A0eS7F+27Clguqr+EPga8IlJDypJWp8hz/BvB+aq6nxVvQw8ChxauqCqHq+ql8aHp4A9kx1TkrReQ4K/G7iw5Hh+fN2V3A98a6UbkhxNMptkdmFhYfiUkqR1GxL8rHBdrbgwuReYBj650u1VdbyqpqtqempqaviUkqR12zlgzTywd8nxHuDi8kVJ3g18GHhHVf1iMuNJkiZlyDP808C+JDcluQ44DMwsXZDkNuCzwMGqem7yY0qS1mvV4FfVJeBB4CTwLPBYVZ1J8kiSg+NlnwReB3w1yX8mmbnCw0mStsiQUzpU1QngxLLrPrLk8rsnPJckacJ8p60kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNGHxJasLgS1ITBl+SmjD4ktSEwZekJgy+JDVh8CWpCYMvSU0YfElqwuBLUhMGX5KaMPiS1ITBl6QmDL4kNWHwJakJgy9JTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ1YfAlqQmDL0lNDAp+kgNJziWZS/LwCrf/VpKvjG//fpIbJz2oJGl9Vg1+kh3AMeAuYD9wJMn+ZcvuB16oqt8HPgX8/aQHlSStz5Bn+LcDc1V1vqpeBh4FDi1bcwj4t/HlrwHvSpLJjSlJWq+dA9bsBi4sOZ4H/vhKa6rqUpIXgd8Ffrp0UZKjwNHx4S+SPLOWobehXSzbq8bci0XuxSL3YtEfrPWOQ4K/0jP1WsMaquo4cBwgyWxVTQ/4+tuee7HIvVjkXixyLxYlmV3rfYec0pkH9i453gNcvNKaJDuBG4CfrXUoSdLkDQn+aWBfkpuSXAccBmaWrZkB/mJ8+R7g36vqsmf4kqSts+opnfE5+QeBk8AO4HNVdSbJI8BsVc0A/wp8Mckco2f2hwd87ePrmHu7cS8WuReL3ItF7sWiNe9FfCIuST34TltJasLgS1ITGx58P5Zh0YC9+GCSs0meTvLtJG/Zijk3w2p7sWTdPUkqybb9lbwhe5HkPePvjTNJvrTZM26WAT8jb07yeJKnxj8nd2/FnBstyeeSPHel9ypl5NPjfXo6ydsGPXBVbdgfRi/y/hfwe8B1wA+A/cvW/CXwmfHlw8BXNnKmrfozcC/eCfz2+PL7O+/FeN31wBPAKWB6q+fewu+LfcBTwO+Mj9+41XNv4V4cB94/vrwf+NFWz71Be/GnwNuAZ65w+93Atxi9B+oO4PtDHnejn+H7sQyLVt2Lqnq8ql4aH55i9J6H7WjI9wXAx4FPAD/fzOE22ZC9eAA4VlUvAFTVc5s842YZshcFvH58+QYuf0/QtlBVT3D19zIdAr5QI6eANyR502qPu9HBX+ljGXZfaU1VXQJ+9bEM282QvVjqfkb/gm9Hq+5FktuAvVX1zc0cbAsM+b64Gbg5yXeTnEpyYNOm21xD9uJjwL1J5oETwAc2Z7RrzivtCTDsoxXWY2Ify7ANDP57JrkXmAbesaETbZ2r7kWSVzH61NX7NmugLTTk+2Ino9M6dzL6X99/JLm1qv5ng2fbbEP24gjw+ar6hyR/wuj9P7dW1f9t/HjXlDV1c6Of4fuxDIuG7AVJ3g18GDhYVb/YpNk222p7cT1wK/CdJD9idI5yZpu+cDv0Z+QbVfXLqvohcI7RPwDbzZC9uB94DKCqvge8htEHq3UzqCfLbXTw/ViGRavuxfg0xmcZxX67nqeFVfaiql6sql1VdWNV3cjo9YyDVbXmD426hg35Gfk6oxf0SbKL0Sme85s65eYYshc/Bt4FkOStjIK/sKlTXhtmgPeOf1vnDuDFqvrJanfa0FM6tXEfy/AbZ+BefBJ4HfDV8evWP66qg1s29AYZuBctDNyLk8CfJzkL/C/woap6fuum3hgD9+Ih4J+T/A2jUxj3bccniEm+zOgU3q7x6xUfBV4NUFWfYfT6xd3AHPAS8L5Bj7sN90qStALfaStJTRh8SWrC4EtSEwZfkpow+JLUhMGXpCYMviQ18f+GmWq6NWLIwgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1378dba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig = plt.figure()\n",
    "ax = plt.subplot(111)\n",
    "\n",
    "colors = ['blue', 'green', 'red', 'cyan','magenta', 'yellow',\n",
    "         'black', 'pink', 'lightgreen', 'lightblue', 'gray', 'indigo', 'orange']\n",
    "weights, params =[], []\n",
    "for c in np.arange(-4., 6.):\n",
    "    lr = LogisticRegression(penalty='l1', C=10.**c, random_state=0)\n",
    "    lr.fit(X_train_std, y_train)\n",
    "    weights.append(lr.coef_[1])\n",
    "    params.append(10**c)\n",
    "\n",
    "weights = np.array(weights)\n",
    "\n",
    "for column, color in zip(range(weights.shape[1], colors)):\n",
    "    plt.plot(params, weights[:, column], label=df_wine.columns[column+1], color=color)\n",
    "plt.axhline(0, color='black', linestyle='--', linewidth=3)\n",
    "plt.xlim([10**(-5), 10**5])\n",
    "plt.ylabel('weight coefficient')\n",
    "plt.xlabel('C')\n",
    "plt.xscale('log')\n",
    "plt.legend(loc='upper left')\n",
    "ax.legend(loc='upper center', bbox_to_anchor=(1.38, 1.03), ncol=1, fancybox=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import clone\n",
    "from itertools import combinations\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "class SBS():\n",
    "    \n",
    "    def __init__(self, estimator, k_features, scoring=accuracy_score, test_size=0.25, random_state=1):\n",
    "        self.scoring = scoring\n",
    "        self.estimator = clone(estimator)\n",
    "        self.k_features = k_features\n",
    "        self.test_size = test_size\n",
    "        self.random_state = random_state\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=self.test_size, random_state=self.random_state)\n",
    "        dim = X_train.shape[1]\n",
    "        self.indices_ = tuple(range(dim))\n",
    "        self.subsets_ = [self.indices_]\n",
    "        score = self._calc_score(X_train, X_test, y_train, y_test, self.indices_)\n",
    "        self.scores_ = [score]\n",
    "        \n",
    "        while dim > self.k_features:\n",
    "            scores = []\n",
    "            subsets =[]\n",
    "            \n",
    "            for p in combinations(self.indices_, r=dim -1):\n",
    "                score = self._calc_score(X_train, X_test, y_train, y_test, p)\n",
    "                scores.append(score)\n",
    "                subsets.append(p)\n",
    "                \n",
    "            best = np.argmax(scores)\n",
    "            self.indices_ = subsets[best]\n",
    "            self.subsets_.append(self.indices_)\n",
    "            dim -=1\n",
    "            \n",
    "            self.scores_.append(scores[best])\n",
    "        self.k_score_ = self.scores_[-1]\n",
    "        \n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        return X[:, self.indices_]\n",
    "    \n",
    "    def _calc_score(self, X_train, X_test, y_train, y_test, indices):\n",
    "        self.estimator.fit(X_train[:, indices], y_train)\n",
    "        y_pred = self.estimator.predict(X_test[:, indices])\n",
    "        score = self.scoring(y_test, y_pred)\n",
    "        return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.SBS at 0x1388be48>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "knn = KNeighborsClassifier(n_neighbors=5)\n",
    "sbs = SBS(knn, k_features=1)\n",
    "sbs.fit(X_train_std, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xl81PW1+P/XyQZhDWuAsCuyiYKJIGBrtCq2boAbtHq11lLXtrZyK7/2qrVfq7e0vb23dW3FrVZERKQWjajEVhbNhEU2I4uEJKxCAgQSsp3fH/OJHULCTMJ88pnlPB+PeTCfbT7nzWTmzGd5n7eoKsYYY8zJJHgdgDHGmMhnycIYY0xQliyMMcYEZcnCGGNMUJYsjDHGBGXJwhhjTFCWLIwxxgRlycIYY0xQliyMMcYEleR1AOHSvXt3HThwoNdhnNSRI0do376912GERay0JVbaAdaWSBXpbcnPz/9SVXsEWy9mksXAgQPx+Xxeh3FSubm5ZGdnex1GWMRKW2KlHWBtiVSR3hYRKQxlPTsNZYwxJihLFsYYY4KyZGGMMSYoSxbGGGOCsmRhjDEmKEsWxhhjgrJkYYwxJihLFsYYY4JyLVmIyBwR2Ssi65tYLiLyfyKyRUQ+FZFzApbdLCKbncfNbsVojDEmNG724H4e+BPwYhPLvwkMcR7jgCeBcSLSFXgQyAIUyBeRRapa6mKsJkQLV5cwO6eAkrIKMlZ+wMxJQ5k8JsPrsEwrqH/vd5ZV0Cct1ZX3vjX+vlqjHbHItWShqv8UkYEnWeVq4EVVVWCliKSJSG8gG1iiqgcARGQJcBnwiluxmtAsXF3CrAXrqKiuBaCkrIJZC9YB2IctxrXGex8r+4hVXtaGygCKAqaLnXlNzTcem51T8NWHrF5FdS2zcwrsgxbjmnrvZ85fy18+2haWfRTsPkx1rXqyD/sbDs7LZCGNzNOTzD/xBURmADMA0tPTyc3NDVtwbigvL4/4GE+mpKyiyfnR2q5of08CudmWpt776lolsepIWPbR8Eu8Nffh5t9wrPyNeZksioF+AdN9gZ3O/OwG83MbewFVfQZ4BiArK0sjubIjRH71yWAyVn7Q6JdGn85to7Zd0f6eBHKrLXM/2QGsa3RZRloqb/70orDsZ+Jjjf99tcY+urVPce3vIFb+xry8dXYR8B/OXVHnAQdVdReQA1wqIl1EpAtwqTPPeOyuC09rdH5yorC//FgrR2PcVlen/Oadz7h/wTqG9upI2+Tjvy5SkxOZOWlo2PY3c9JQUpMTW30fIrD/SBUvrtgetv3EIjdvnX0FWAEMFZFiEfmeiNwuIrc7qywGtgFbgD8DdwI4F7Z/BeQ5j4frL3Ybb9XU+Q/he3RsA/h/8X13wgB2HzrGlCeWs3VfuZfhmTCqrK7lh3NX80TuVqaP7c9b95zPY1PPIiMtFcH/3j86dVRYz/NPHpPBo1NHkZGWCq2wj/p2PDZlFBcP78kDb27gV29tpLau8VNV8c7Nu6GmB1muwF1NLJsDzHEjLtNyr+YVMbJPJ/7xw68dd2h91egMbnvBx9QnlvP0TZmcN7ibt4GaU3LgSBUzXvThKyzl/m8O4wdfH4yIMHlMhusXgev34eapm8bacW1WP3711kae/egLikuP8ocbxpCaktjEK8Qn68FtQrK+5CAbdh7i+qx+Jywb078Lb9w5ke4dUrjp2Y95Y3WxBxGacPjiyyNMfWIZn5Yc5PFvn8PtF5yGSGP3nMSWxAThoatG8sAVI3h34x6mPbOCfYft1GogSxYmJK/5ikhJSmDy6MZ/Wfbv1o4Fd0wkc0AX7n11Lf/3/mb8B48mWuRtP8DUJ5ZxqLKGV74/jsvP6u11SK3u1vMH8fSNmRTsOcyUJ5axec9hr0OKGJYsTFCV1bUsXLOTy0b2onO75CbX69wumRdvHcfUMRn8fsnnzJz/KVU1da0YqWmpRWt38p0/f0yXdim8cecEMgd09Tokz1w6shevzhhPZXUdU59czvItX3odUkSwZGGCytmwm4MV1dxw7omnoBpKSUrgd9efzY8vHsL8/GJuee4TDlZUt0KUpiVUlceXbuGHr6xmdL80Xr9jAgO6tfc6LM+d3S+NN+6cQK9ObfmPOZ8wP99OrVqyMEHN8xXRt0sq40O8cC0i/PjiM/jddWeTt/0A1zy5nKIDR12O0jRXdW0d97++jtk5BVw9ug8v3TaWLu1TvA4rYvTr2o75d0xg3OCu3PfaWn7/bkFcn1q1ZGFOqujAUZZt2c91mf1ISGjehc5rMvvy4q3j2HuokilPLGNNUZlLUZrmOlRZza3P5/Gqr4h7LjqdP9wwmjZJdvdPQ51Tk3nulrFcl9mX//tgCz+Zt5ZjNbXBN4xBlizMSb2WX4wIXJvVt0Xbjz+tGwvunEDb5ESmPbOCnA27wxyhaa6Ssgque3IFK7bu5zfXnsVPLx0aF3c8tVRKUgK/ufYs7rv0DN5YXcJNz35C2dEqr8NqdZYsTJNq65T5viLOP737Vx2lWuL0nh15486JDO3Vidv/ms+zH30R14fzXlpXfJDJjy9jZ1kFL9w6ttFboc2JRIS7LxrC/04bzZodZUx9cjk79sfXqVVLFqZJH235kp0HK0O6sB1Mj45tmPv985g0ohe/emsjDy3aQE2t3SnVmt7buIfrn15BSmICr985gYmnd/c6pKhz9egMXvreWPaXVzHliWWs2hE/w+xYsjBNmucroku7ZC4ZkR6W10tNSeSJ75zD9782iBdWFPKDl/I5cqwmLK9tTu6F5duZ8ZKPIekdeOOuCZyR3tHrkKLWuMH+U6vt2yQx/ZmVLF63y+uQWoUlC9Oo0iNVLNmwh8ljMsJ64TMhQfj55SP41dUjWVqwl+ufXsGeQ5Vhe31zvNo65eG/b+TBRRv4xvB05s44j54d23odVtQ7rUcH3rhzAiP7dOLOl1fx9IdbY/7UqiUL06g3VpdQVVvn2jntm8YP5C83Z/HFl0eY8vgyPtt9yJX9xLOjVTXc/td85iz7glsnDuKpGzNpl+LlqASxpVuHNvzt++dx+ajePPr2Z/xi4fqYPrVqycKcQFWZ5yvirL6dGd67k2v7uWhYOvN+MJ5aVa59cgX//Hyfa/uKN3sPVzLtmZW8v2kPD105ggeuHEFiM299NsG1TU7kj9PHcPsFp/Hyxzu47UUf5TF6atWShTnBupKDfLb7cKvcKXNmRmcW3jWRvl1S+e7zebzyyQ7X9xnrNu85zJTHl7N5TzlP35TFLRMHeR1STEtIEO7/5jB+PWUU/9r8Jdc9tYJdBxsfWTCa2TGpOcGreUW0SUrgqtF9WmV/vTun8trt47nrb6uZtWAd723aw6Zdh9hVVkmftFRmThoateMjL1xdwuycAnaWVbjWlvp9lJRV0P2jJRyurKZTagrzfjCeUX07h3VfpmnfHtefjC6p3PXyKiY/voybxg/glY+LKCmrIGPlB1H9dwx2ZGEaqKiqZdGanXxrVG86tW26aGC4dWybzLM3ZzF+cFfe37SXnWWVKP4OZLMWrGPh6pJWiyVcFq4uYdaCdZSUVbjWlsB9AHxZXkVVjXLHBYMtUXjggjN68Nrt46msruW3OZ9/9b5E899xPTuyMMd5e/0uDh+r8aSzVnJiAjsaqSFVUV3L7JyCqPtVNjungIrq40tDVFTX8tN5a5mdUxCWfew+WEltg7twFHj2o+3cev7gsOzDNM/w3p1ITU7iYMXx1y6i9e+4niULc5x5viIGdGvHeYO9KVG9s6zx22h3lkXfOeCmYq5VZfxp4RlNsKlqqNH4/xVLmrodPJrfF0sW5iuF+4+wctsBZk7yrlZQn7TUrw7dA3XrEF3VUE82BkJGWiq/ve7ssOxnxdb9jf5/9TmF8izm1DX1dxzN74tdszBfmecrIkHgmnNaVjQwHGZOGkpq8vGdAAV/J8E310TH+d75+cX8x5xP6NmxDW2Sjv+IpSYnMnPS0LDtq7H/r3DvwzRfY+9L26SEqH5fLFkYAGpq65ifX8wFZ/SgV2fvevhOHpPBo1NHkZGWiuD/Ff7IlDPJHNCVH81dw+NLt0RsT1lV5fdLPue+19YybnBX3v3JBfz3NWcd15ZHp44K6znrwP8vXNqHab6G7wvAmRmdovp9sdNQBoB/bf6SPYeO8curvK9COnlMxgkfqmsy+/Kf8z9ldk4BhfuP8MiUUSQnRs5vnWM1tdz/+jreWF3CdZl9eWTKKP+Y5Y20Jdzq95Gbm0t2drar+zKhC3xfPqnsxRO5W1m1o5Rz+nfxOrQWiZxPm/HUq3lFdGufwkXDwlM0MNzaJCXyhxtG88OLTmeer5jvPpfHocrIGK617GgVNz37CW+sLuG+S8/gN9eeRUqSfbTMv9154en07NiGX/59I3V1kXlkHIz9RRu+LD/Ge5v2MGVMRkR/yYkIP7l0KL+59ixWbtvPtU8up7jU2zEFduw/ytQnl7NmRxn/O200d180xAYSMifo0CaJ+785jLVFZbwRpX0tIvebwbSaN1aVUFOnYRm3ojVcn9WPF24dy66DlUx5Yjnrig96EseqHaVMeWIZB45U8dL3xnL16Og9H23cN3l0BqP7pfHYO59FZf0oV5OFiFwmIgUiskVE7m9k+QAReV9EPhWRXBHpG7CsVkTWOI9FbsYZz+qLBo7pn8aQKBrjYOLp3VlwxwRSEhO4/ukVLNm4p1X3//a6XUx/ZiUd2iax4I4JjBscnn4TJnYlJAgPXjmCfYeP8fjSLV6H02yuJQsRSQQeB74JjACmi8iIBqv9FnhRVc8CHgYeDVhWoaqjncdVbsUZ71YXlbF5bzk3ROHwmkPSO/LGXRMYkt6BGS/5eG7ZF67vU1V55p9bufNvqxjZpxML7pjA4B4dXN+viQ1j+ndh6jkZPPuvLyjcf8TrcJrFzSOLscAWVd2mqlXAXODqBuuMAN53ni9tZLlx2by8IlKTE7n8rN5eh9IiPTu2Ze6M87h4eDq//PtGfvn3DdS6dAGxpraOXyxcz68Xf8a3zuzN375/Ht06tHFlXyZ2/eyyYSQlCo/8Y5PXoTSLm8kiAygKmC525gVaC1zjPJ8CdBSR+uP5tiLiE5GVIjLZxTjj1pFjNfx97U4uP6s3HVuxaGC4tUtJ4qkbM7l14iCeW7adH7yUz9Gq8J4TLj9Ww20v+nj54x3cfsFp/HH6GNomh28EQRM/0ju15a4LT+fdjXv4aHPTPf0jjbjVwUlErgMmqeptzvRNwFhVvSdgnT7An4BBwD/xJ46RqnpQRPqo6k4RGQx8AHxDVbc22McMYAZAenp65ty5c11pS7iUl5fToUPknLL4V3E1z66v4v8b15YzujTviy/S2lLvvcJqXt5UxYBOCfw4sw1pbU7+eyiUdpRW1vH7/GOUlNfxHyNSyO4XmYk1Ut+Tloj1tlTVKj//qIKURHh4QqqnA1NdeOGF+aqaFXRFVXXlAYwHcgKmZwGzTrJ+B6C4iWXPA9eebH+ZmZka6ZYuXep1CMe59slleuHspVpXV9fsbSOtLYGWbNitw37xtk549H0t2H3opOsGa8eGkoM67pH3dOQD72huwd4wRhl+kfyeNFc8tOXtdbt0wM/e0heWf9Gq8TQE+DSE73Q3T0PlAUNEZJCIpADTgOPuahKR7iJSH8MsYI4zv4uItKlfB5gIbHQx1rizdV85edtLuS6rX8z1C7h4hH+41qraOq55YnmLD/WXFuzluqeWIwKv3T6eC87oEeZITTybNDKdCad143fvfk7pkSqvwwnKtWShqjXA3UAOsAmYp6obRORhEam/uykbKBCRz4F04BFn/nDAJyJr8V/4fkxVLVmE0Wu+YhIThGsyY7NvwKi+/uFa+6SlcstznzDPVxR8owB/XVnIbS/4GNi9PW/cOdHVschNfBIRHrxyJIcrq/nDe597HU5QrtaGUtXFwOIG8x4IeD4fmN/IdsuBUW7GFs9qaut4fVUxFw7tSc+O3hUNdFtGWiqv3TGeu15exX/O/5Qd+4/y00vPOOmRVF2d8t/vfMbT/9zGRcN68sfpY2jfxkqoGXcM7dWRG88bwF8/3sG3xw1gaK/I7etkPbjj0NKCfew7fCxqemyfik5tk5lzy7lMO7cff1q6hR/NXcOxmtpG162sruXuV1bx9D+3cdN5A3jmpkxLFMZ19158Bh3aJPHLv2+I2IrKYMkiLr2aV0T3Dm3IHhof5+CTExN4dOooZk4ayqK1O7npL5+ccI54f/kxpv95JW+v380vLh/Ow1ePJCmCqtqa2NWlfQo/ueQMlm/dz7utXImgOexnU5zZe7iSpQV7ue1rgyKqxLfbRIS7Ljydfl3bcd9ra7nkfz4kMUHYc+gYPZe9R01dHUerannyO+dw2ZnR2UHRRK/vjOvPyx8X8sg/NnHBGT0isg9P/HxbGAAWrCqhtk65PgrLe4TDVWf34fYLBvNleRV7Dh0DYO/hYxw4Us0d2adZojCeSEpM4IErRrLjwFHmtELZmpawZBFHVJV5eUWcO7ALp8VxPaPX8xsvET0vr7iVIzHm384f0p1LRqTzpw+2sOdQpdfhnMCSRRzxFZay7csjXBenRxX1dpZVNGu+Ma3l598aTk2t8pt3CrwO5QSWLOLIvLwi2qckcvmo+D7V0idgXORQ5hvTWgZ2b8+t5w/i9VXFrCkq8zqc41iyiBPlx2r4x7pdXHl2n7i/HXTmpKGkNriAmJqcyMxJQz2KyJh/u/ui0+nRsQ0PLdoQUUOwWrKIE2+t3cnRqlquj4O+FcFMHpPBo1NHkeEcSWSkpfLo1FFMHhObvdlNdOnQJomfXTaMNUVlvLk2coZgje+fmHHkVV8Rp/fswJh+aV6HEhEmj8lg8pgMcnNzyc7O9jocY44zdUwGL63YzmNvf8alI3pFxNkAO7KIA5v3HGb1jjJuiMGigcbEooQE4YErR7Ln0DGeyI2MIVgtWcSBeb4ikhKEKefYaRZjokXmgC5MGZPBn//1BTv2H/U6HEsWsa6qpo4Fq0q4eHg63W0IUGOiys8uG0aiCL9e7P0QrJYsYtwHn+1h/5Eqrj+3r9ehGGOaqVfnttx14Wm8s2E3y7d4OwSrJYsYN89XTHqnNnx9SHwUDTQm1tz2tcH07ZLKw29tpKa2zrM4LFnEsN0HK8kt2Mu1mX2tgqoxUaptciI//9ZwPtt9mFfymjeIVzjZN0gMe31VMXUK12Va3wpjotllZ/Zi/OBu/P7dAsqOejMEqyWLGFVXp8zzFTFuUFcGdm/vdTjGmFMgIjxw5QgOVlTzh/c2exKDJYsY9cn2AxTuPxoXo+EZEw+G9+7Et8f156WVhXy+53Cr79+SRYyal1dExzZJfNPGZzAmZvzkkqG0T0nkV29tbPUhWC1ZxKBDldUsXr+Lq0b3ITUl8kbcMsa0TNf2Kdx7yRn8a/OXvLdpb6vu25JFDFq0ZieV1XVxOxqeMbHsxvMGcHrPDvy/f2zkWE1tq+3XkkUMes1XxLBeHTmrb2evQzHGhFlyYgL/dcUICvcf5bll21ttv5YsYsxnuw+xtvgg11vRQGNi1gVn9ODi4T354/ub2Xu4dYZg9b7urQmLhatLmJ1TQIkzNGibJPsdYEws+/nlI/jG73LJnp1LRVUtfdJSmTlpqGvjsrj6jSIil4lIgYhsEZH7G1k+QETeF5FPRSRXRPoGLLtZRDY7j5vdjDPaLVxdwqwF675KFAD/7x+bWLg6cgZOMcaE19qiMhJEOFpViwIlZRXMWrDOtc+9a8lCRBKBx4FvAiOA6SIyosFqvwVeVNWzgIeBR51tuwIPAuOAscCDItLFrVij3eycAiqqj7/QVVFdy+ycyBv03RgTHrNzCqhpMOyqm597N48sxgJbVHWbqlYBc4GrG6wzAnjfeb40YPkkYImqHlDVUmAJcJmLsUa1nQFHFKHMN8ZEv9b+3LuZLDKAwKpXxc68QGuBa5znU4COItItxG0NUHqkiuQmigT2ccaYNsbEnqY+32597t28wN3YrTgNuxzeB/xJRG4B/gmUADUhbouIzABmAKSnp5Obm3sK4bqvvLw8rDHuOVLH7/Mrqa5VkgRqAv6HUhLg8v61rv2fhLstXomVdoC1JVK51ZbL+9fy/CGoCqha7ubn3s1kUQwE9grrC+wMXEFVdwJTAUSkA3CNqh4UkWIgu8G2uQ13oKrPAM8AZGVlaXZ2dsNVIkpubi7hijG/8AD3vuBDSWLe7VmUlFYwO6eAnWUVrt8VAeFti5dipR1gbYlUbrUlGxjh3AXZGp97N5NFHjBERAbhP2KYBnw7cAUR6Q4cUNU6YBYwx1mUA/w64KL2pc5yA7z16U5+Mm8tfTq35bnvjmVQ9/acOxBXk4MxJvJMHpPRap97165ZqGoNcDf+L/5NwDxV3SAiD4vIVc5q2UCBiHwOpAOPONseAH6FP+HkAQ878+KaqvJk7lbu/ttqzsrozII7JzLIyo8bY1qBq53yVHUxsLjBvAcCns8H5jex7Rz+faQR96pr63jgzfW88kkRV57dh9nXnkXbZCsSaIxpHdaDOwocrqzmzpdX8a/NX3Jn9mncd+lQEhKslIcxpvVYsohwO8squPX5PDbvLeexqaOYNra/1yEZY+JQSNcsROR1EblcRKzgUCtaX3KQyY8vo6S0gue/e64lCmOMZ0L98n8S/51Mm0XkMREZ5mJMBvjgsz1c//QKkhKE1+4Yz9eG9PA6JGNMHAspWajqe6r6HeAcYDuwRESWi8h3RSTZzQDj0UsrtnPbCz4G92jPG3dNZFivTl6HZIyJcyFfs3DKcNwI3ASsBl4Gzgdu5vgOdKaFauuURxdv4i8ffcHFw3vyv9PG0L6NXVYyxngvpG8iEVkADANeAq5U1V3OoldFxOdWcPGkoqqWH7+6mpwNe7hlwkD+64oRJNodT8aYCBHqz9Y/qeoHjS1Q1awwxhOX9h0+xm0v+vi0uIwHrhjBrecP8jokY4w5TqgXuIeLSFr9hIh0EZE7XYoprmzZe5gpTyyjYPchnrox0xKFMSYihZosvq+qZfUTzhgT33cnpPixfOuXTH1iOZXVdbw6YzyTRvbyOiRjjGlUqKehEkREVFXhq1HwUtwLK/bNzy9m1oJPGditPXNuOZd+Xdt5HZIxxjQp1GSRA8wTkafwjytxO/COa1HFMFXlf97bzP+9v5kJp3XjyRsz6Zxqdx8bYyJbqMniZ8APgDvwD0z0LvAXt4KKNQudmvMlZRWkvp9DRXUt12b25ddTRpGSZJ3ijTGRL6Rk4Yw38aTzMM2wcHUJsxaso6K6FvAPqJ6UIEw8rZslCmNM1Ai1NtQQEZkvIhtFZFv9w+3gYsHsnIKvEkW9mjrlt+9+7lFExhjTfKH+tH0O/1FFDXAh8CL+DnomiJ1lFc2ab4wxkSjUZJGqqu8DoqqFqvoQcJF7YcWOPmmpzZpvjDGRKNRkUemUJ98sIneLyBSgp4txxYyZk4aSnHh82Y7U5ERmThrqUUTGGNN8oSaLHwPtgB8CmfgLCt7sVlCxZPKYDM7u25n6Mk8Zaak8OnVUqw2ybowx4RD0biinA971qjoTKAe+63pUMWb3oWNMGtmLG/oeJjs72+twjDGm2YIeWahqLZApIlYCtQX2HKqkuLSCzAFdvA7FGGNaLNROeauBN0XkNeBI/UxVXeBKVDHEt70UgKyBXSnbusPjaIwxpmVCTRZdgf0cfweUApYsgvAVHqBtcgIj+3Ri2VavozHGmJYJtQe3XadoId/2Ukb3SyM50XprG2OiV6gj5T2H/0jiOKp6a9gjiiFHjtWwcdch7rjgNK9DMcaYUxLqz923gH84j/eBTvjvjDopEblMRApEZIuI3N/I8v4islREVovIpyLyLWf+QBGpEJE1zuOp0JsUOdYWlVFbp2QOtIvbxpjoFuppqNcDp0XkFeC9k23j3HL7OHAJUAzkicgiVd0YsNovgHmq+qSIjAAWAwOdZVtVdXRIrYhQvsJSROCc/pYsjDHRraUn0ocA/YOsMxbYoqrbVLUKmAtc3WAdxX+UAtAZ2NnCeCJS3vYDDE3vaONVGGOiXqhVZw+LyKH6B/B3/GNcnEwGUBQwXezMC/QQcKOIFOM/qrgnYNkg5/TUhyLytVDijCS1dcrqHWXWv8IYExNCPQ3VsQWv3VgnvoYXyacDz6vq70RkPPCSiJwJ7AL6q+p+EckEForISFU9dNwORGYAMwDS09PJzc1tQZju2HGolvJjNbSv2ENu7n4AysvLIyrGUxErbYmVdoC1JVLFTFtUNegDmAJ0DphOAyYH2WY8kBMwPQuY1WCdDUC/gOltQM9GXisXyDrZ/jIzMzWSvLj8Cx3ws7d0x/4jX81bunSpdwGFWay0JVbaoWptiVSR3hbApyHkgVCvWTyoqgcDEkwZ8GCQbfKAISIySERSgGnAogbr7AC+ASAiw4G2wD4R6eFcIEdEBuO/RhJVgy3lbS8lvVMb+naxUuTGmOgXag/uxpLKSbdV1RoRuRvIARKBOaq6QUQexp/JFgE/Bf4sIvfiP0V1i6qqiHwdeFhEaoBa4HZVPRBirBEhv7CUrAFdsZJaxphYEGqy8InI7/HfCqv4L0TnB9tIVRfjv3AdOO+BgOcbgYmNbPc68HrD+dFi18EKSsoq+N75g7wOxRhjwiLU01D3AFXAq8A8oAK4y62got2/iwfanVDGmNgQ6t1QR4ATemCbxuUXltIuJZERvTsFX9kYY6JAqP0slohIWsB0FxHJcS+s6Ja3/QCj+6WRZMUDjTExItRvs+7OHVAAqGopNgZ3o8qP1bBp1yGyrDOeMSaGhJos6kTkq/IeIjKQRqrQGlizo4w6hcyBXb0OxRhjwibUu6F+DnwkIh8601/H6TltjucrPIAIjOmfFnxlY4yJEqFe4H5HRLLwJ4g1wJv474gyDfi2lzKsVyc6tbXigcaY2BHq4Ee3AT8C+uJPFucBKzh+mNW4V1Nbx+odpUw9p6/XoRhjTFiFes3iR8C5QKGqXgiMAfa5FlWU+mz3YY5U1Vr/CmNMzAk1WVSqaiWAiLRR1c+Aoe6FFZ3yC/2d8awsuTEm1oR6gbvY6WexEFgiIqXE2EBF4ZC3/QASgvEHAAAQ9UlEQVS9O7clI82KBxpjYkuoF7inOE8fEpGl+Ee1e8e1qKJUfmEpmQO6WPFAY0zMCfXI4iuq+mHwteJPSVkFuw5WWmc8Y0xMsnoUYeLb7q+gnmWd8YwxMciSRZj4tpfSPiWRYb1aMgKtMcZENksWYeIrLGVM/y5WPNAYE5Psmy0MDldWU7D7kN0ya4yJWZYswmC1UzzQOuMZY2KVJYsw8G0/QILAmP6WLIwxscmSRRj4CksZ3rsTHdo0+05kY4yJCpYsTlFNbR1risqsf4UxJqZZsjhFm3Yd5mhVrQ12ZIyJaZYsTlGe0xnvXLu4bYyJYZYsTlF+YSkZaan07mzFA40xscuSxSlQVXyFB6x/hTEm5rmaLETkMhEpEJEtInJ/I8v7i8hSEVktIp+KyLcCls1ytisQkUluxtlSxaUV7Dl0zPpXGGNinmv3eopIIvA4cAlQDOSJyCJV3Riw2i+Aear6pIiMABYDA53n04CRQB/gPRE5Q1Vr3Yq3JXyFTvHAAXZx2xgT29w8shgLbFHVbapaBcwFrm6wjgKdnOed+feASlcDc1X1mKp+AWxxXi+i+LaX0rFNEkOteKAxJsa5mSwygKKA6WJnXqCHgBtFpBj/UcU9zdjWc/mFpYzun0Zigg12ZIyJbW52OW7sG1QbTE8HnlfV34nIeOAlETkzxG0RkRnADID09HRyc3NPLeJmOFKtFOw+yvAOlSHvt7y8vFVjdFOstCVW2gHWlkgVK21xM1kUA/0Cpvty4rjd3wMuA1DVFSLSFuge4rao6jPAMwBZWVmanZ0drtiDyi3Yi5LHddnnMOH07qFtk5tLa8boplhpS6y0A6wtkSpW2uLmaag8YIiIDBKRFPwXrBc1WGcH8A0AERkOtAX2OetNE5E2IjIIGAJ84mKszebbXkpigjC6f5rXoRhjjOtcO7JQ1RoRuRvIARKBOaq6QUQeBnyqugj4KfBnEbkX/2mmW1RVgQ0iMg/YCNQAd0XinVAjeneiXYoVDzTGxD5Xv+lUdTH+C9eB8x4IeL4RmNjEto8Aj7gZX0tVO8UDp53b3+tQjDGmVVgP7hbYuPMQldV1nGvFA40xccKSRQvUFw+0ntvGmHhhyaIF8gtL6dsllfRObb0OxRhjWoUli2byFw8stcGOjDFxxZJFMxUdqGDf4WM22JExJq5YsmgmG+zIGBOPLFk0k6+wlI5tkzijpxUPNMbED0sWzZRfeIBz+nchwYoHGmPiiCWLZjh4tJrP95TbxW1jTNyxZNEM+Tvq+1fYxW1jTHyxZNEMvu2lJCUIo/tZ8UBjTHyxZNEMvsJSRvbpRGpKotehGGNMq7JkEaKqmjrWFpWRaeNtG2PikCWLEK3feZBjNXXWv8IYE5csWYQof3spAJmWLIwxcciSRYh8hQfo37UdPTta8UBjTPyxZBECVSXfigcaY+KYJYsQbN9/lC/Lq6x/hTEmblmyCIHPBjsyxsQ5SxYhyC8spVPbJE7v0cHrUIwxxhOWLELgKywlc4AVDzTGxC9LFkGUHqliy95yu15hjIlrliyCyC/096+wO6GMMfHMkkUQvsJSkhOFs614oDEmjlmyCCK/8AAj+3SmbbIVDzTGxC9Xk4WIXCYiBSKyRUTub2T5/4jIGufxuYiUBSyrDVi2yM04m3Ksppa1xQetHpQxJu4lufXCIpIIPA5cAhQDeSKySFU31q+jqvcGrH8PMCbgJSpUdbRb8YVifclBqmrqrNKsMSbuuXlkMRbYoqrbVLUKmAtcfZL1pwOvuBhPs/nqiwfaxW1jTJxzM1lkAEUB08XOvBOIyABgEPBBwOy2IuITkZUiMtm9MJvmKyxlYLd29OjYxovdG2NMxHDtNBTQWA82bWLdacB8Va0NmNdfVXeKyGDgAxFZp6pbj9uByAxgBkB6ejq5ublhCNsJVJWVm49yds+ksL1ueXl5WGP0Uqy0JVbaAdaWSBUrbXEzWRQD/QKm+wI7m1h3GnBX4AxV3en8u01EcvFfz9jaYJ1ngGcAsrKyNDs7OxxxA7B1XzmHcz7kinHDyR7bPyyvmZubSzhj9FKstCVW2gHWlkgVK21x8zRUHjBERAaJSAr+hHDCXU0iMhToAqwImNdFRNo4z7sDE4GNDbd1U/1gR1Y80BhjXDyyUNUaEbkbyAESgTmqukFEHgZ8qlqfOKYDc1U18BTVcOBpEanDn9AeC7yLqjX4Cg+Q1i6Zwd2teKAxxrh5GgpVXQwsbjDvgQbTDzWy3XJglJuxBeNzBjuy4oHGGGM9uBu1v/wY2/Ydsf4VxhjjsGTRiK+KB9r1CmOMASxZNCq/sJSUxARGZXT2OhRjjIkIliwa4Sss5cyMTlY80BhjHJYsGqisrmVd8UHOtcGOjDHmK5YsGlhXcpCq2jqrB2WMMQEsWTRgxQONMeZEliwayC88wODu7enWwYoHGmNMPUsWAerq1N8Zz26ZNcaY41iyCLDty3LKjlaTZZ3xjDHmOJYsAnx1vcKOLIwx5jiWLAL4Ckvp2j6Fwd3bex2KMcZEFEsWAXzbD5A5oAsiVjzQGGMCWbJw7Dt8jO37j5Jlt8waY8wJLFk4rHigMcY0zZKFI7/wAClJCZxpxQONMeYEliwcedtLObtvZ9okWfFAY4xpyJIF/uKBG3YetMGOjDGmCZYsgLVFZVTXql3cNsaYJsR9sli4uoTvv+gD4L/eXM/C1SUeR2SMMZEnyesAvLRwdQmzFqyjoroWgF0HK5m1YB0Ak8dkeBmaMcZElLg+spidU/BVoqhXUV3L7JwCjyIyxpjIFNfJYmdZRbPmG2NMvIrrZNEnLbVZ840xJl7FdbKYOWkoqcnH96tITU5k5qShHkVkjDGRydVkISKXiUiBiGwRkfsbWf4/IrLGeXwuImUBy24Wkc3O42Y34ps8JoNHp44iIy0VATLSUnl06ii7uG2MMQ24djeUiCQCjwOXAMVAnogsUtWN9euo6r0B698DjHGedwUeBLIABfKdbUvDHefkMRmWHIwxJgg3jyzGAltUdZuqVgFzgatPsv504BXn+SRgiaoecBLEEuAyF2M1xhhzEm4miwygKGC62Jl3AhEZAAwCPmjutsYYY9znZqe8xkYQ0ibWnQbMV9X6Tg8hbSsiM4AZzmS5iER6B4nuwJdeBxEmsdKWWGkHWFsiVaS3ZUAoK7mZLIqBfgHTfYGdTaw7DbirwbbZDbbNbbiRqj4DPHMqQbYmEfGpapbXcYRDrLQlVtoB1pZIFSttcfM0VB4wREQGiUgK/oSwqOFKIjIU6AKsCJidA1wqIl1EpAtwqTPPGGOMB1w7slDVGhG5G/+XfCIwR1U3iMjDgE9V6xPHdGCuqmrAtgdE5Ff4Ew7Aw6p6wK1YjTHGnJyrhQRVdTGwuMG8BxpMP9TEtnOAOa4F542oOWUWglhpS6y0A6wtkSom2iIBP+iNMcaYRsV1uQ9jjDGhsWTRCkSkn4gsFZFNIrJBRH7kdUynQkQSRWS1iLzldSynQkTSRGS+iHzmvDfjvY6ppUTkXudva72IvCIibb2OKVQiMkdE9orI+oB5XUVkiVPuZ4lzo0tEa6Ids52/r09F5A0RSfMyxlNhyaJ11AA/VdXhwHnAXSIywuOYTsWPgE1eBxEG/wu8o6rDgLOJ0jaJSAbwQyBLVc/Ef0PJNG+japbnObFCw/3A+6o6BHjfmY50z3NiO5YAZ6rqWcDnwKzWDipcLFm0AlXdpaqrnOeH8X8pRWWPdBHpC1wO/MXrWE6FiHQCvg48C6CqVapadvKtIloSkCoiSUA7mu7TFHFU9Z9Aw7sdrwZecJ6/AExu1aBaoLF2qOq7qlrjTK7E32csKlmyaGUiMhB/wcSPvY2kxf4A/CdQ53Ugp2gwsA94zjml9hcRae91UC2hqiXAb4EdwC7goKq+621UpyxdVXeB/8cW0NPjeMLhVuBtr4NoKUsWrUhEOgCvAz9W1UNex9NcInIFsFdV872OJQySgHOAJ1V1DHCE6DjVcQLnfP7V+Our9QHai8iN3kZlAonIz/Gfjn7Z61haypJFKxGRZPyJ4mVVXeB1PC00EbhKRLbjryJ8kYj81duQWqwYKFbV+iO8+fiTRzS6GPhCVfepajWwAJjgcUynao+I9AZw/t3rcTwt5ozHcwXwHY3ivgqWLFqBiAj+c+ObVPX3XsfTUqo6S1X7qupA/BdQP1DVqPwFq6q7gSKn3AzAN4CNJ9kkku0AzhORds7f2jeI0ov1ARYB9YOe3Qy86WEsLSYilwE/A65S1aNex3MqLFm0jonATfh/idePDPgtr4My3AO8LCKfAqOBX3scT4s4R0fzgVXAOvyf66jpNSwir+CvDTdURIpF5HvAY8AlIrIZ/wBqj3kZYyiaaMefgI7AEudz/5SnQZ4C68FtjDEmKDuyMMYYE5QlC2OMMUFZsjDGGBOUJQtjjDFBWbIwxhgTlCULExVEREXkdwHT94nIQ2F67edF5NpwvFaQ/VznVLdd2siy2U7V2NkteN3Rdiu2cZslCxMtjgFTRaS714EEEpHEZqz+PeBOVb2wkWU/AM5R1ZktCGM00KxkIX72+Tchsz8WEy1q8Hc0u7fhgoZHBiJS7vybLSIfisg8EflcRB4Tke+IyCcisk5ETgt4mYtF5F/Oelc42yc6v/jznPEIfhDwuktF5G/4O8E1jGe68/rrReS/nXkPAOcDTzU8ehCRRUB74GMRuUFEeojI685+80RkorPeWBFZ7hQ+XC4iQ0UkBXgYuMHp9HWDiDwkIvcFvP56ERnoPDaJyBP4O/D1E5FLRWSFiKwSkdec+mU4/1cbnXb/trlvlolBqmoPe0T8AygHOgHbgc7AfcBDzrLngWsD13X+zQbKgN5AG6AE+KWz7EfAHwK2fwf/j6ch+OtGtQVmAL9w1mkD+PAX68vGX3hwUCNx9sFffqMH/mKFHwCTnWW5+MecaLR9Ac//BpzvPO+Pv0wMTvuTnOcXA687z28B/hSw/UPAfQHT64GBzqMOOM+Z3x34J9Demf4Z8ADQFSjg351207x+/+3h/SMpeDoxJjKo6iEReRH/QD8VIW6Wp06paxHZCtSX7l4HBJ4OmqeqdcBmEdkGDAMuBc4KOGrpjD+ZVAGfqOoXjezvXCBXVfc5+3wZ/7gZC0OMF/yJYIS/zBMAnUSko7P/F0RkCKBAcjNes16hqq50np8HjACWOftKwV+u4hBQCfxFRP4BRPWIiCY8LFmYaPMH/KdQnguYV4NzStUppJcSsOxYwPO6gOk6jv/7b1j3RgEB7lHVnMAFIpKN/8iiMdLE/OZIAMar6nEJUUT+CCxV1SniHxclt4ntv/r/cAQOsRoYtwBLVHV6wxcQkbH4CxJOA+4GLmpeE0yssWsWJqqo6gFgHv6LxfW2A5nO86tp2S/u60QkwbmOMRj/aZgc4A6nvDwicoYEHyDpY+ACEenuXPyeDnzYzFjexf8FjbPf0c7TzvhPpYH/1FO9w/iL1dXbjlNuXUTOwX/qrDErgYkicrqzbjunjR2Azqq6GPgx/gvoJs5ZsjDR6Hf4z7fX+zP+L+hPgHE0/av/ZArwf6m/DdyuqpX4h47dCKwSkfXA0wQ5GndOec0ClgJrgVWq2tzy2j8EspyLyxuB2535vwEeFZFl+MfZrrcU/2mrNSJyA/5xU7qKyBrgDvxjPzcW6z78SecV8VfeXYn/9FtH4C1n3oc0clOBiT9WddYYY0xQdmRhjDEmKEsWxhhjgrJkYYwxJihLFsYYY4KyZGGMMSYoSxbGGGOCsmRhjDEmKEsWxhhjgvr/Afs06x9hRCMsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1389cd68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "k_feat = [len(k) for k in sbs.subsets_]\n",
    "\n",
    "plt.plot(k_feat, sbs.scores_, marker='o')\n",
    "plt.ylim([0.7, 1.02])\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('Number of features')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-46-3875ecb1486b>, line 11)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-46-3875ecb1486b>\"\u001b[1;36m, line \u001b[1;32m11\u001b[0m\n\u001b[1;33m    print(\"%2d\") %-*s %f\" % (f+1, 30, feat_labels[indices[f]], importances[indices[f]]))\u001b[0m\n\u001b[1;37m                   ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "feat_labels = df_wine.columns[1:]\n",
    "\n",
    "forest = RandomForestClassifier(n_estimators=500, random_state=1)\n",
    "forest.fit(X_train, y_train)\n",
    "importances = forest.feature_importances_\n",
    "indices = np.argsort(importances)[::-1]\n",
    "\n",
    "for f in range(X_train.shape[1]):\n",
    "    print(\"%2d\") %-*s %f\" % (f+1, 30, feat_labels[indices[f]], importances[indices[f]]))\n",
    "\n",
    "plt.title('Feature Importance')\n",
    "plt.bar(range(X_train.shape[1]), importances[indices], align='center')\n",
    "plt.xticks(range(X_train.shape[1]), feat_labels, rotation=90)\n",
    "plt.xlim([-1, X_train.shape[1]])\n",
    "plt.tight_layout\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}