subpopulation_attack_paperimplementation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "operating-hello",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code was given with paper except for the last section which we wrote the print the feature and cluster with highest\n",
    "# Target metric value\n",
    "\n",
    "# importing necessary packages\n",
    "\n",
    "import pickle\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import neural_network, linear_model, cluster\n",
    "\n",
    "np.random.seed(0) # seeding 0 for random number generation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "small-recycling",
   "metadata": {},
   "source": [
    "### Loading the Adult Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "distant-series",
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing the adult dataset, it is split into train and test by default\n",
    "a = pd.read_csv('adult/adult.data', header=None,names=['age', 'workclass', 'fnlwgt', 'education',\n",
    "                       'education-num', 'marital-status', 'occupation', 'relationship',\n",
    "                       'race', 'sex', 'capital-gain', 'capital-loss', 'hours-per-week',\n",
    "                       'native-country', 'income']) # dim: (32561 x 15)\n",
    "\n",
    "b = pd.read_csv('adult/adult.test', header=None,\n",
    "                names=['age', 'workclass', 'fnlwgt', 'education',\n",
    "                       'education-num', 'marital-status', 'occupation', 'relationship',\n",
    "                       'race', 'sex', 'capital-gain', 'capital-loss', 'hours-per-week',\n",
    "                      'native-country', 'income'])# dim: (16282, 15)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "geological-scenario",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(48843, 60)\n",
      "['age', 'capital-gain', 'capital-loss', 'hours-per-week', 'workclass_ Federal-gov', 'workclass_ Local-gov', 'workclass_ Never-worked', 'workclass_ Private', 'workclass_ Self-emp-inc', 'workclass_ Self-emp-not-inc', 'workclass_ State-gov', 'workclass_ Without-pay', 'education_2.0', 'education_3.0', 'education_4.0', 'education_5.0', 'education_6.0', 'education_7.0', 'education_8.0', 'education_9.0', 'education_10.0', 'education_11.0', 'education_12.0', 'education_13.0', 'education_14.0', 'education_15.0', 'education_16.0', 'marital_ Married-AF-spouse', 'marital_ Married-civ-spouse', 'marital_ Married-spouse-absent', 'marital_ Never-married', 'marital_ Separated', 'marital_ Widowed', 'occupation_ Adm-clerical', 'occupation_ Armed-Forces', 'occupation_ Craft-repair', 'occupation_ Exec-managerial', 'occupation_ Farming-fishing', 'occupation_ Handlers-cleaners', 'occupation_ Machine-op-inspct', 'occupation_ Other-service', 'occupation_ Priv-house-serv', 'occupation_ Prof-specialty', 'occupation_ Protective-serv', 'occupation_ Sales', 'occupation_ Tech-support', 'occupation_ Transport-moving', 'relationship_ Not-in-family', 'relationship_ Other-relative', 'relationship_ Own-child', 'relationship_ Unmarried', 'relationship_ Wife', 'race_ Asian-Pac-Islander', 'race_ Black', 'race_ Other', 'race_ White', 'sex_ Male', 'income_ <=50K.', 'income_ >50K', 'income_ >50K.']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>capital-gain</th>\n",
       "      <th>capital-loss</th>\n",
       "      <th>hours-per-week</th>\n",
       "      <th>workclass_ Federal-gov</th>\n",
       "      <th>workclass_ Local-gov</th>\n",
       "      <th>workclass_ Never-worked</th>\n",
       "      <th>workclass_ Private</th>\n",
       "      <th>workclass_ Self-emp-inc</th>\n",
       "      <th>workclass_ Self-emp-not-inc</th>\n",
       "      <th>...</th>\n",
       "      <th>relationship_ Unmarried</th>\n",
       "      <th>relationship_ Wife</th>\n",
       "      <th>race_ Asian-Pac-Islander</th>\n",
       "      <th>race_ Black</th>\n",
       "      <th>race_ Other</th>\n",
       "      <th>race_ White</th>\n",
       "      <th>sex_ Male</th>\n",
       "      <th>income_ &lt;=50K.</th>\n",
       "      <th>income_ &gt;50K</th>\n",
       "      <th>income_ &gt;50K.</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16277</th>\n",
       "      <td>39</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16278</th>\n",
       "      <td>64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16279</th>\n",
       "      <td>38</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16280</th>\n",
       "      <td>44</td>\n",
       "      <td>5455.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16281</th>\n",
       "      <td>35</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      age  capital-gain  capital-loss  hours-per-week  workclass_ Federal-gov  \\\n",
       "16277  39           0.0           0.0            36.0                       0   \n",
       "16278  64           0.0           0.0            40.0                       0   \n",
       "16279  38           0.0           0.0            50.0                       0   \n",
       "16280  44        5455.0           0.0            40.0                       0   \n",
       "16281  35           0.0           0.0            60.0                       0   \n",
       "\n",
       "       workclass_ Local-gov  workclass_ Never-worked  workclass_ Private  \\\n",
       "16277                     0                        0                   1   \n",
       "16278                     0                        0                   0   \n",
       "16279                     0                        0                   1   \n",
       "16280                     0                        0                   1   \n",
       "16281                     0                        0                   0   \n",
       "\n",
       "       workclass_ Self-emp-inc  workclass_ Self-emp-not-inc  ...  \\\n",
       "16277                        0                            0  ...   \n",
       "16278                        0                            0  ...   \n",
       "16279                        0                            0  ...   \n",
       "16280                        0                            0  ...   \n",
       "16281                        1                            0  ...   \n",
       "\n",
       "       relationship_ Unmarried  relationship_ Wife  race_ Asian-Pac-Islander  \\\n",
       "16277                        0                   0                         0   \n",
       "16278                        0                   0                         0   \n",
       "16279                        0                   0                         0   \n",
       "16280                        0                   0                         1   \n",
       "16281                        0                   0                         0   \n",
       "\n",
       "       race_ Black  race_ Other  race_ White  sex_ Male  income_ <=50K.  \\\n",
       "16277            0            0            1          0               1   \n",
       "16278            1            0            0          1               1   \n",
       "16279            0            0            1          1               1   \n",
       "16280            0            0            0          1               1   \n",
       "16281            0            0            1          1               0   \n",
       "\n",
       "       income_ >50K  income_ >50K.  \n",
       "16277             0              0  \n",
       "16278             0              0  \n",
       "16279             0              0  \n",
       "16280             0              0  \n",
       "16281             0              1  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concatenate the data to preprocess the whole dataset according to our need\n",
    "\n",
    "total = pd.concat([a, b], axis=0) # dim: (488843, 15)\n",
    "\n",
    "# dropping education, native country, fnlwgt as mentioned in the paper ( three columns basically)\n",
    "total = total.drop('education', axis=1)\n",
    "total = total.drop('native-country', axis=1)\n",
    "total = total.drop('fnlwgt', axis=1)\n",
    "\n",
    "# dim: (488843, 12)\n",
    "# print(total['income'])\n",
    "\n",
    "# adding new columns for columns with categorical variables\n",
    "for col in ['workclass', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex', 'income']:\n",
    "    if '-' in col:\n",
    "        prefix_col = col.split('-')[0]\n",
    "    else:\n",
    "        prefix_col = col\n",
    "    \n",
    "    # converting categorical varaibles into indicator variables\n",
    "    total = pd.concat([total, pd.get_dummies(total[col], prefix=prefix_col, drop_first=True)], axis=1)\n",
    "    total = total.drop(col, axis=1)\n",
    "\n",
    "# dim: (488843, 60)\n",
    "\n",
    "print(total.shape)\n",
    "print(list(total.columns))\n",
    "total.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "weighted-radiation",
   "metadata": {},
   "source": [
    "### Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "metric-coordinator",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((7841, 57), (7841,), (7841, 57), (7841, 57), (7692, 57), (7692,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# converting the pandas dataframe to numpy array\n",
    "total_np = total.to_numpy()\n",
    "\n",
    "# D has 7841 samples, D_aux has 7841 samples, and D_test has 7692 samples\n",
    "\n",
    "# taking '>50K' income as y values\n",
    "y = (total_np[:, -1] + total_np[:, -2]).astype(np.float32) # last two columns are duplicate, so add them \n",
    "y = np.delete(y, 32561, axis=0)                            # delete the 32561th row value for y as it has NaN values\n",
    "\n",
    "# taking rest of the data as x values, after deleting the last three columns i.e columns\n",
    "x = np.delete(total_np, [total_np.shape[1]-1, total_np.shape[1]-2, total_np.shape[1]-3], axis=1)\n",
    "x = np.delete(x, 32561, axis=0).astype(np.float32)         # delete the 32561th row value for x as it has NaN vlaues\n",
    "\n",
    "\n",
    "#separating the dataset into training and testing, training [0-32560] row, testing [32561-rest]\n",
    "train_x, train_y = x[:32561], y[:32561]     # dim: (32561 x 57)\n",
    "test_x, test_y = x[32561:], y[32561:]       # dim: (16281 x 57)\n",
    "\n",
    "# saving the index of samples where y is true(1) and false(0)\n",
    "train_zero_inds = np.where(train_y==0)[0]   # dim: (24720 x 1)\n",
    "train_one_inds = np.where(train_y==1)[0]    # dim: (7841 x 1)\n",
    "test_zero_inds = np.where(test_y==0)[0]     # dim: (12435 x 1)\n",
    "test_one_inds = np.where(test_y==1)[0]      # dim: (3846 x 1)\n",
    "\n",
    "# creating an array of random numbers, range[0-24720(train_zero_inds.shape[0])], of dimension (7841(train_one_inds.shape[0]), 1) \n",
    "train_zeros = np.random.choice(train_zero_inds.shape[0], train_one_inds.shape[0], replace=False) # dim:(7841 x 1)\n",
    "\n",
    "# creating an array of random numbers, range[0-12435(test_zero_inds.shape[0])], of dimension (3846(test_one_inds.shape[0]), 1)\n",
    "test_zeros = np.random.choice(test_zero_inds.shape[0], test_one_inds.shape[0], replace=False) # dim: (3846 x 1)\n",
    "\n",
    "\n",
    "# concatenating random choices of zero indexed example with one indexed example to build a dataset with \n",
    "# equal number of zero and one indexed samples\n",
    "train_x = np.concatenate((train_x[train_zeros], train_x[train_one_inds]), axis=0) # dim: (15682 x 57)\n",
    "train_y = np.concatenate((train_y[train_zeros], train_y[train_one_inds]), axis=0) # dim: (15682 x 1)\n",
    "\n",
    "\n",
    "test_x = np.concatenate((test_x[test_zeros], test_x[test_one_inds]), axis=0)      # dim: (7692 x 57)\n",
    "test_y = np.concatenate((test_y[test_zeros], test_y[test_one_inds]), axis=0)      # dim: (7692 x 1)\n",
    "\n",
    "\n",
    "# shuffle training data(row wise) by shuffling the index and getting the data from shuffled index\n",
    "train_shuffle = np.random.choice(train_x.shape[0], train_x.shape[0], replace=False) # dim: (15682 x 1)\n",
    "train_x, train_y = train_x[train_shuffle], train_y[train_shuffle]                   # dim: (15682 x 57) and (15682 x 1)\n",
    "\n",
    "train_size = train_x.shape[0]//2\n",
    "\n",
    "train_x, train_y, ho_x, ho_y = train_x[:train_size], train_y[:train_size], train_x[train_size:], train_y[train_size:]\n",
    "\n",
    "pois_rates = [0.5, 1, 2]     # poison rates\n",
    "#pois_rates = [0.5, 1, 1.5, 2, 2.5, 3]\n",
    "\n",
    "# D: (train_x, train_y)\n",
    "# D_aux : (ho_x, ho_y)\n",
    "# D_test: (test_x, test_y)\n",
    "train_x.shape, train_y.shape, ho_x.shape, ho_x.shape, test_x.shape, test_y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lyric-treasure",
   "metadata": {},
   "source": [
    "### Training Clean LR and NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "martial-decrease",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8364534581383255"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training and testing the linear regression model\n",
    "lm = linear_model.LogisticRegression(max_iter = 5000)\n",
    "lm.fit(train_x, train_y)\n",
    "lm.score(test_x, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "brief-asset",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the trained model as a pickel file\n",
    "save_path = 'subpopulation/'\n",
    "\n",
    "with open(os.path.join(save_path,\"lm.pickle\"), \"wb\") as f:\n",
    "    pickle.dump(lm,f)\n",
    "\n",
    "# # loading the trained model \n",
    "# with open(os.path.join(save_path,\"lm.pickle\"), \"rb\") as f:\n",
    "#     test = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "recovered-williams",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.97870038 0.97758587 0.05441383 ... 0.94849411 0.95495388 0.67978749]\n",
      "[0. 1. 0. ... 0. 1. 0.]\n"
     ]
    }
   ],
   "source": [
    "# print(lm.classes_) # two classes '0' & '1'\n",
    "\n",
    "# predict the output probability for ho_x dataset. predict_proba gives probability for each class i.e a tuple \n",
    "# in this case [0.1 0.9] probaibility for this class. np.eye(2) gets [1 0] when y = 0 and [0 1] when y = 1\n",
    "# which when muliplied singles out the probability\n",
    "lm_preds = np.multiply(lm.predict_proba(ho_x), np.eye(2)[ho_y.astype(np.int)]).sum(axis=1) # dim: (7841, 1)\n",
    "\n",
    "# could have simply done predict\n",
    "lm_pred_class = lm.predict(ho_x)                    # dim: (7841, 1)\n",
    "\n",
    "print(lm_preds) # print the probabilites but has no information on which class it belongs to\n",
    "print(lm_pred_class)  # print the predictions, gives classes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "present-patient",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8337233489339574\n"
     ]
    }
   ],
   "source": [
    "# training and test the neural network with 10 hidden units\n",
    "nn = neural_network.MLPClassifier(hidden_layer_sizes=(10,))\n",
    "nn.fit(train_x, train_y)\n",
    "clean_test_acc = nn.score(test_x, test_y)\n",
    "print(clean_test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "played-insight",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.97376835 0.9968766  0.08229731 ... 0.96648937 0.99939501 0.75527519]\n",
      "[0. 1. 0. ... 0. 1. 0.]\n"
     ]
    }
   ],
   "source": [
    "# prediction for the nn\n",
    "\n",
    "# probability of belonging to a clas\n",
    "\n",
    "nn_preds = np.multiply(nn.predict_proba(ho_x), np.eye(2)[ho_y.astype(np.int)]).sum(axis=1) \n",
    "nn_pred_class = nn.predict(ho_x) # prediction class\n",
    "\n",
    "print(nn_preds)\n",
    "print(nn_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "induced-marathon",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7632116561694878\n"
     ]
    }
   ],
   "source": [
    "pred_mean = np.multiply(nn.predict_proba(test_x), np.eye(2)[test_y.astype(int)]).mean()*2\n",
    "print(pred_mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "documentary-technique",
   "metadata": {},
   "source": [
    "# FeatureMatch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cooked-anthony",
   "metadata": {},
   "source": [
    "### Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "theoretical-thirty",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FeatureMatch Data Preprocessing\n",
    "\n",
    "np.random.seed(0)\n",
    "# concatenating education level and race for D_aux data\n",
    "protected = np.concatenate((ho_x[:, 12:27], ho_x[:, 52:57]), axis=1) # dim: (7841, 20)\n",
    "\n",
    "# concatenating education level and race for D_test data\n",
    "test_prot = np.concatenate((test_x[:, 12:27], test_x[:, 52:57]), axis=1) # dim: (7692, 20)\n",
    "\n",
    "# concatenating education level and race for D_train data\n",
    "train_prot = np.concatenate((train_x[:, 12:27], train_x[:, 52:57]), axis=1) # dim: (7841, 20)\n",
    "\n",
    "all_cols = list(total.columns)                # getting names of all columns\n",
    "prot_cols = all_cols[12:27] + all_cols[52:57] # getting names of protected columns\n",
    "\n",
    "subclasses, counts = np.unique(protected, axis=0, return_counts=True)# 122 unique examples in the protected data \n",
    "# dim: (122 x 20), (122,1)\n",
    "# print(tuple(zip(subclasses, counts)))\n",
    "# print(subclasses[0:5], counts[0:5])\n",
    "\n",
    "hd_sbcl_conf = []                             # empty array\n",
    "hd_used = []                                  # empty array\n",
    "\n",
    "# declaring arrays for storing errors in the subclasses, i.e [subclasses, (clean_acc, collat, target), pois_ind]\n",
    "\n",
    "#  dim: (122 x 3 x len: pois rates) for a class (3 x pois len) dimensional array\n",
    "hd_lr_errs = np.zeros((len(subclasses), 3, len(pois_rates)))\n",
    "#  dim: (122 x 3 x len: pois rates) for a class (3 x 3 pois len) dimensional array\n",
    "hd_nn_errs = np.zeros((len(subclasses), 3, len(pois_rates)))  \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "similar-northeast",
   "metadata": {},
   "source": [
    "### FeatureMatch: Poisoning the data and calculating error rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "architectural-thanksgiving",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Subclass Index: 7, Subclass Count: 32 \n",
      "Sc_lr_pred: 0.797437, Sc_nn_pred: 0.744064 \n",
      "['education_16.0', 'race_ White'] 33 16 20\n",
      "poison fraction: 0.48484848484848486\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.826\n",
      "lr clean sbc 0.800\n",
      "lr poisoned sbc 0.800\n",
      "nn clean  sbc 0.800\n",
      "nn poisoned sbc 0.800\n",
      "['education_16.0', 'race_ White'] 33 33 20\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.800\n",
      "lr poisoned sbc 0.700\n",
      "nn clean  sbc 0.800\n",
      "nn poisoned sbc 0.817\n",
      "['education_16.0', 'race_ White'] 33 66 20\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.800\n",
      "lr poisoned sbc 0.750\n",
      "nn clean  sbc 0.800\n",
      "nn poisoned sbc 0.817\n",
      "\n",
      "\n",
      "Subclass Index: 12, Subclass Count: 14 \n",
      "Sc_lr_pred: 0.896783, Sc_nn_pred: 0.891565 \n",
      "['education_16.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 12 6 18\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.835\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_16.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 12 12 18\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.835\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_16.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 12 24 18\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 0.944\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "\n",
      "\n",
      "Subclass Index: 13, Subclass Count: 33 \n",
      "Sc_lr_pred: 0.807605, Sc_nn_pred: 0.833843 \n",
      "['education_15.0', 'race_ White'] 22 11 22\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 0.955\n",
      "nn poisoned sbc 1.000\n",
      "['education_15.0', 'race_ White'] 22 22 22\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.829\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 0.955\n",
      "nn poisoned sbc 0.939\n",
      "['education_15.0', 'race_ White'] 22 44 22\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 0.864\n",
      "nn clean  sbc 0.955\n",
      "nn poisoned sbc 0.909\n",
      "\n",
      "\n",
      "Subclass Index: 18, Subclass Count: 18 \n",
      "Sc_lr_pred: 0.929539, Sc_nn_pred: 0.906774 \n",
      "['education_15.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 12 6 11\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.835\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_15.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 12 12 11\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_15.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 12 24 11\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.821\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 0.970\n",
      "\n",
      "\n",
      "Subclass Index: 24, Subclass Count: 12 \n",
      "Sc_lr_pred: 0.842777, Sc_nn_pred: 0.862745 \n",
      "['education_14.0', 'race_ Black'] 9 4 15\n",
      "poison fraction: 0.4444444444444444\n",
      "Poisoned lr test acc 0.838\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.667\n",
      "lr poisoned sbc 0.733\n",
      "nn clean  sbc 0.800\n",
      "nn poisoned sbc 0.800\n",
      "['education_14.0', 'race_ Black'] 9 9 15\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.667\n",
      "lr poisoned sbc 0.867\n",
      "nn clean  sbc 0.800\n",
      "nn poisoned sbc 0.800\n",
      "['education_14.0', 'race_ Black'] 9 18 15\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.838\n",
      "Poisoned nn test acc 0.835\n",
      "lr clean sbc 0.667\n",
      "lr poisoned sbc 0.733\n",
      "nn clean  sbc 0.800\n",
      "nn poisoned sbc 0.800\n",
      "\n",
      "\n",
      "Subclass Index: 25, Subclass Count: 22 \n",
      "Sc_lr_pred: 0.822951, Sc_nn_pred: 0.813337 \n",
      "['education_14.0', 'race_ Black', 'sex_ Male'] 16 8 19\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.947\n",
      "lr poisoned sbc 0.947\n",
      "nn clean  sbc 0.947\n",
      "nn poisoned sbc 0.947\n",
      "['education_14.0', 'race_ Black', 'sex_ Male'] 16 16 19\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.824\n",
      "Poisoned nn test acc 0.836\n",
      "lr clean sbc 0.947\n",
      "lr poisoned sbc 0.947\n",
      "nn clean  sbc 0.947\n",
      "nn poisoned sbc 0.912\n",
      "['education_14.0', 'race_ Black', 'sex_ Male'] 16 32 19\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 0.947\n",
      "lr poisoned sbc 0.947\n",
      "nn clean  sbc 0.947\n",
      "nn poisoned sbc 0.930\n",
      "\n",
      "\n",
      "Subclass Index: 27, Subclass Count: 17 \n",
      "Sc_lr_pred: 0.826912, Sc_nn_pred: 0.825429 \n",
      "['education_14.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 35 17 29\n",
      "poison fraction: 0.4857142857142857\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.816\n",
      "lr clean sbc 0.759\n",
      "lr poisoned sbc 0.759\n",
      "nn clean  sbc 0.793\n",
      "nn poisoned sbc 0.701\n",
      "['education_14.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 35 35 29\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.759\n",
      "lr poisoned sbc 0.759\n",
      "nn clean  sbc 0.793\n",
      "nn poisoned sbc 0.690\n",
      "['education_14.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 35 70 29\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.820\n",
      "Poisoned nn test acc 0.828\n",
      "lr clean sbc 0.759\n",
      "lr poisoned sbc 0.690\n",
      "nn clean  sbc 0.793\n",
      "nn poisoned sbc 0.667\n",
      "\n",
      "\n",
      "Subclass Index: 34, Subclass Count: 25 \n",
      "Sc_lr_pred: 0.718887, Sc_nn_pred: 0.735360 \n",
      "['education_13.0', 'race_ Black'] 37 18 39\n",
      "poison fraction: 0.4864864864864865\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.846\n",
      "lr poisoned sbc 0.846\n",
      "nn clean  sbc 0.872\n",
      "nn poisoned sbc 0.863\n",
      "['education_13.0', 'race_ Black'] 37 37 39\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.846\n",
      "lr poisoned sbc 0.846\n",
      "nn clean  sbc 0.872\n",
      "nn poisoned sbc 0.838\n",
      "['education_13.0', 'race_ Black'] 37 74 39\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.846\n",
      "lr poisoned sbc 0.846\n",
      "nn clean  sbc 0.872\n",
      "nn poisoned sbc 0.872\n",
      "\n",
      "\n",
      "Subclass Index: 35, Subclass Count: 60 \n",
      "Sc_lr_pred: 0.766246, Sc_nn_pred: 0.768005 \n",
      "['education_13.0', 'race_ Black', 'sex_ Male'] 60 30 54\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.802\n",
      "lr clean sbc 0.944\n",
      "lr poisoned sbc 0.944\n",
      "nn clean  sbc 0.963\n",
      "nn poisoned sbc 0.877\n",
      "['education_13.0', 'race_ Black', 'sex_ Male'] 60 60 54\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.820\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.944\n",
      "lr poisoned sbc 0.944\n",
      "nn clean  sbc 0.963\n",
      "nn poisoned sbc 0.926\n",
      "['education_13.0', 'race_ Black', 'sex_ Male'] 60 120 54\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.820\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.944\n",
      "lr poisoned sbc 0.963\n",
      "nn clean  sbc 0.963\n",
      "nn poisoned sbc 0.932\n",
      "\n",
      "\n",
      "Subclass Index: 36, Subclass Count: 18 \n",
      "Sc_lr_pred: 0.707752, Sc_nn_pred: 0.724150 \n",
      "['education_13.0', 'race_ Asian-Pac-Islander'] 18 9 23\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.826\n",
      "lr poisoned sbc 0.826\n",
      "nn clean  sbc 0.826\n",
      "nn poisoned sbc 0.812\n",
      "['education_13.0', 'race_ Asian-Pac-Islander'] 18 18 23\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.819\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.826\n",
      "lr poisoned sbc 0.826\n",
      "nn clean  sbc 0.826\n",
      "nn poisoned sbc 0.826\n",
      "['education_13.0', 'race_ Asian-Pac-Islander'] 18 36 23\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 0.826\n",
      "lr poisoned sbc 0.826\n",
      "nn clean  sbc 0.826\n",
      "nn poisoned sbc 0.768\n",
      "\n",
      "\n",
      "Subclass Index: 37, Subclass Count: 66 \n",
      "Sc_lr_pred: 0.740910, Sc_nn_pred: 0.737496 \n",
      "['education_13.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 58 29 51\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.826\n",
      "lr clean sbc 0.882\n",
      "lr poisoned sbc 0.882\n",
      "nn clean  sbc 0.882\n",
      "nn poisoned sbc 0.863\n",
      "['education_13.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 58 58 51\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.882\n",
      "lr poisoned sbc 0.882\n",
      "nn clean  sbc 0.882\n",
      "nn poisoned sbc 0.876\n",
      "['education_13.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 58 116 51\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.882\n",
      "lr poisoned sbc 0.882\n",
      "nn clean  sbc 0.882\n",
      "nn poisoned sbc 0.895\n",
      "\n",
      "\n",
      "Subclass Index: 38, Subclass Count: 59 \n",
      "Sc_lr_pred: 0.696425, Sc_nn_pred: 0.714543 \n",
      "['education_12.0', 'race_ White'] 66 33 65\n",
      "poison fraction: 0.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.738\n",
      "lr poisoned sbc 0.738\n",
      "nn clean  sbc 0.738\n",
      "nn poisoned sbc 0.728\n",
      "['education_12.0', 'race_ White'] 66 66 65\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.738\n",
      "lr poisoned sbc 0.723\n",
      "nn clean  sbc 0.738\n",
      "nn poisoned sbc 0.754\n",
      "['education_12.0', 'race_ White'] 66 132 65\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.814\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.738\n",
      "lr poisoned sbc 0.723\n",
      "nn clean  sbc 0.738\n",
      "nn poisoned sbc 0.723\n",
      "\n",
      "\n",
      "Subclass Index: 43, Subclass Count: 11 \n",
      "Sc_lr_pred: 0.771231, Sc_nn_pred: 0.797885 \n",
      "['education_12.0', 'race_ Black', 'sex_ Male'] 15 7 4\n",
      "poison fraction: 0.4666666666666667\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.828\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_12.0', 'race_ Black', 'sex_ Male'] 15 15 4\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_12.0', 'race_ Black', 'sex_ Male'] 15 30 4\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.820\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "\n",
      "\n",
      "Subclass Index: 48, Subclass Count: 83 \n",
      "Sc_lr_pred: 0.809820, Sc_nn_pred: 0.813137 \n",
      "['education_11.0', 'race_ White'] 79 39 78\n",
      "poison fraction: 0.4936708860759494\n",
      "Poisoned lr test acc 0.831\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.910\n",
      "lr poisoned sbc 0.885\n",
      "nn clean  sbc 0.872\n",
      "nn poisoned sbc 0.902\n",
      "['education_11.0', 'race_ White'] 79 79 78\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.809\n",
      "lr clean sbc 0.910\n",
      "lr poisoned sbc 0.923\n",
      "nn clean  sbc 0.872\n",
      "nn poisoned sbc 0.829\n",
      "['education_11.0', 'race_ White'] 79 158 78\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.910\n",
      "lr poisoned sbc 0.923\n",
      "nn clean  sbc 0.872\n",
      "nn poisoned sbc 0.902\n",
      "\n",
      "\n",
      "Subclass Index: 52, Subclass Count: 11 \n",
      "Sc_lr_pred: 0.680446, Sc_nn_pred: 0.667563 \n",
      "['education_11.0', 'race_ Black', 'sex_ Male'] 11 5 9\n",
      "poison fraction: 0.45454545454545453\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.778\n",
      "lr poisoned sbc 0.778\n",
      "nn clean  sbc 0.778\n",
      "nn poisoned sbc 0.778\n",
      "['education_11.0', 'race_ Black', 'sex_ Male'] 11 11 9\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.825\n",
      "lr clean sbc 0.778\n",
      "lr poisoned sbc 0.778\n",
      "nn clean  sbc 0.778\n",
      "nn poisoned sbc 0.778\n",
      "['education_11.0', 'race_ Black', 'sex_ Male'] 11 22 9\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.778\n",
      "lr poisoned sbc 0.889\n",
      "nn clean  sbc 0.778\n",
      "nn poisoned sbc 0.778\n",
      "\n",
      "\n",
      "Subclass Index: 61, Subclass Count: 57 \n",
      "Sc_lr_pred: 0.877904, Sc_nn_pred: 0.885419 \n",
      "['education_10.0', 'race_ Black'] 49 24 47\n",
      "poison fraction: 0.4897959183673469\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.830\n",
      "lr poisoned sbc 0.830\n",
      "nn clean  sbc 0.851\n",
      "nn poisoned sbc 0.801\n",
      "['education_10.0', 'race_ Black'] 49 49 47\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 0.830\n",
      "lr poisoned sbc 0.809\n",
      "nn clean  sbc 0.851\n",
      "nn poisoned sbc 0.794\n",
      "['education_10.0', 'race_ Black'] 49 98 47\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.819\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 0.830\n",
      "lr poisoned sbc 0.766\n",
      "nn clean  sbc 0.851\n",
      "nn poisoned sbc 0.780\n",
      "\n",
      "\n",
      "Subclass Index: 62, Subclass Count: 72 \n",
      "Sc_lr_pred: 0.743357, Sc_nn_pred: 0.742754 \n",
      "['education_10.0', 'race_ Black', 'sex_ Male'] 83 41 70\n",
      "poison fraction: 0.4939759036144578\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.797\n",
      "lr clean sbc 0.886\n",
      "lr poisoned sbc 0.871\n",
      "nn clean  sbc 0.871\n",
      "nn poisoned sbc 0.852\n",
      "['education_10.0', 'race_ Black', 'sex_ Male'] 83 83 70\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.820\n",
      "lr clean sbc 0.886\n",
      "lr poisoned sbc 0.886\n",
      "nn clean  sbc 0.871\n",
      "nn poisoned sbc 0.829\n",
      "['education_10.0', 'race_ Black', 'sex_ Male'] 83 166 70\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.817\n",
      "lr clean sbc 0.886\n",
      "lr poisoned sbc 0.871\n",
      "nn clean  sbc 0.871\n",
      "nn poisoned sbc 0.833\n",
      "\n",
      "\n",
      "Subclass Index: 64, Subclass Count: 30 \n",
      "Sc_lr_pred: 0.745051, Sc_nn_pred: 0.748353 \n",
      "['education_10.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 35 17 27\n",
      "poison fraction: 0.4857142857142857\n",
      "Poisoned lr test acc 0.815\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.963\n",
      "lr poisoned sbc 0.889\n",
      "nn clean  sbc 0.926\n",
      "nn poisoned sbc 0.864\n",
      "['education_10.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 35 35 27\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.963\n",
      "lr poisoned sbc 0.889\n",
      "nn clean  sbc 0.926\n",
      "nn poisoned sbc 0.889\n",
      "['education_10.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 35 70 27\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.824\n",
      "Poisoned nn test acc 0.809\n",
      "lr clean sbc 0.963\n",
      "lr poisoned sbc 0.889\n",
      "nn clean  sbc 0.926\n",
      "nn poisoned sbc 0.877\n",
      "\n",
      "\n",
      "Subclass Index: 66, Subclass Count: 14 \n",
      "Sc_lr_pred: 0.771544, Sc_nn_pred: 0.723122 \n",
      "['education_9.0', 'sex_ Male'] 13 6 11\n",
      "poison fraction: 0.46153846153846156\n",
      "Poisoned lr test acc 0.815\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.818\n",
      "lr poisoned sbc 0.818\n",
      "nn clean  sbc 0.818\n",
      "nn poisoned sbc 0.818\n",
      "['education_9.0', 'sex_ Male'] 13 13 11\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.814\n",
      "lr clean sbc 0.818\n",
      "lr poisoned sbc 0.818\n",
      "nn clean  sbc 0.818\n",
      "nn poisoned sbc 0.818\n",
      "['education_9.0', 'sex_ Male'] 13 26 11\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.826\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.818\n",
      "lr poisoned sbc 0.818\n",
      "nn clean  sbc 0.818\n",
      "nn poisoned sbc 0.818\n",
      "\n",
      "\n",
      "Subclass Index: 71, Subclass Count: 65 \n",
      "Sc_lr_pred: 0.894098, Sc_nn_pred: 0.898855 \n",
      "['education_9.0', 'race_ Black'] 75 37 70\n",
      "poison fraction: 0.49333333333333335\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.943\n",
      "lr poisoned sbc 0.914\n",
      "nn clean  sbc 0.929\n",
      "nn poisoned sbc 0.919\n",
      "['education_9.0', 'race_ Black'] 75 75 70\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.943\n",
      "lr poisoned sbc 0.914\n",
      "nn clean  sbc 0.929\n",
      "nn poisoned sbc 0.886\n",
      "['education_9.0', 'race_ Black'] 75 150 70\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.943\n",
      "lr poisoned sbc 0.829\n",
      "nn clean  sbc 0.929\n",
      "nn poisoned sbc 0.824\n",
      "\n",
      "\n",
      "Subclass Index: 74, Subclass Count: 41 \n",
      "Sc_lr_pred: 0.708082, Sc_nn_pred: 0.707699 \n",
      "['education_9.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 33 16 34\n",
      "poison fraction: 0.48484848484848486\n",
      "Poisoned lr test acc 0.814\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.794\n",
      "lr poisoned sbc 0.824\n",
      "nn clean  sbc 0.882\n",
      "nn poisoned sbc 0.833\n",
      "['education_9.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 33 33 34\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.794\n",
      "lr poisoned sbc 0.882\n",
      "nn clean  sbc 0.882\n",
      "nn poisoned sbc 0.824\n",
      "['education_9.0', 'race_ Asian-Pac-Islander', 'sex_ Male'] 33 66 34\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.794\n",
      "lr poisoned sbc 0.794\n",
      "nn clean  sbc 0.882\n",
      "nn poisoned sbc 0.814\n",
      "\n",
      "\n",
      "Subclass Index: 75, Subclass Count: 13 \n",
      "Sc_lr_pred: 0.925910, Sc_nn_pred: 0.939393 \n",
      "['education_8.0', 'race_ White'] 14 7 7\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 0.857\n",
      "['education_8.0', 'race_ White'] 14 14 7\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 0.857\n",
      "['education_8.0', 'race_ White'] 14 28 7\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.831\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 0.857\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 0.714\n",
      "\n",
      "\n",
      "Subclass Index: 76, Subclass Count: 44 \n",
      "Sc_lr_pred: 0.706776, Sc_nn_pred: 0.726720 \n",
      "['education_8.0', 'race_ White', 'sex_ Male'] 36 18 39\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.814\n",
      "Poisoned nn test acc 0.816\n",
      "lr clean sbc 0.718\n",
      "lr poisoned sbc 0.641\n",
      "nn clean  sbc 0.744\n",
      "nn poisoned sbc 0.641\n",
      "['education_8.0', 'race_ White', 'sex_ Male'] 36 36 39\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.718\n",
      "lr poisoned sbc 0.641\n",
      "nn clean  sbc 0.744\n",
      "nn poisoned sbc 0.632\n",
      "['education_8.0', 'race_ White', 'sex_ Male'] 36 72 39\n",
      "poison fraction: 2.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Poisoned lr test acc 0.824\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.718\n",
      "lr poisoned sbc 0.641\n",
      "nn clean  sbc 0.744\n",
      "nn poisoned sbc 0.615\n",
      "\n",
      "\n",
      "Subclass Index: 82, Subclass Count: 41 \n",
      "Sc_lr_pred: 0.867468, Sc_nn_pred: 0.862854 \n",
      "['education_7.0', 'race_ White'] 36 18 42\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.835\n",
      "lr clean sbc 0.952\n",
      "lr poisoned sbc 0.952\n",
      "nn clean  sbc 0.881\n",
      "nn poisoned sbc 0.881\n",
      "['education_7.0', 'race_ White'] 36 36 42\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.820\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.952\n",
      "lr poisoned sbc 0.833\n",
      "nn clean  sbc 0.881\n",
      "nn poisoned sbc 0.889\n",
      "['education_7.0', 'race_ White'] 36 72 42\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.824\n",
      "Poisoned nn test acc 0.822\n",
      "lr clean sbc 0.952\n",
      "lr poisoned sbc 0.833\n",
      "nn clean  sbc 0.881\n",
      "nn poisoned sbc 0.833\n",
      "\n",
      "\n",
      "Subclass Index: 83, Subclass Count: 89 \n",
      "Sc_lr_pred: 0.753493, Sc_nn_pred: 0.769549 \n",
      "['education_7.0', 'race_ White', 'sex_ Male'] 118 59 107\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 0.841\n",
      "lr poisoned sbc 0.738\n",
      "nn clean  sbc 0.850\n",
      "nn poisoned sbc 0.720\n",
      "['education_7.0', 'race_ White', 'sex_ Male'] 118 118 107\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.841\n",
      "lr poisoned sbc 0.682\n",
      "nn clean  sbc 0.850\n",
      "nn poisoned sbc 0.645\n",
      "['education_7.0', 'race_ White', 'sex_ Male'] 118 236 107\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.815\n",
      "Poisoned nn test acc 0.825\n",
      "lr clean sbc 0.841\n",
      "lr poisoned sbc 0.682\n",
      "nn clean  sbc 0.850\n",
      "nn poisoned sbc 0.648\n",
      "\n",
      "\n",
      "Subclass Index: 86, Subclass Count: 12 \n",
      "Sc_lr_pred: 0.916740, Sc_nn_pred: 0.916913 \n",
      "['education_7.0', 'race_ Black'] 10 5 14\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_7.0', 'race_ Black'] 10 10 14\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.828\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 0.976\n",
      "['education_7.0', 'race_ Black'] 10 20 14\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.815\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 0.857\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 0.976\n",
      "\n",
      "\n",
      "Subclass Index: 90, Subclass Count: 22 \n",
      "Sc_lr_pred: 0.956683, Sc_nn_pred: 0.958143 \n",
      "['education_6.0', 'race_ White'] 37 18 33\n",
      "poison fraction: 0.4864864864864865\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.939\n",
      "lr poisoned sbc 0.909\n",
      "nn clean  sbc 0.939\n",
      "nn poisoned sbc 0.869\n",
      "['education_6.0', 'race_ White'] 37 37 33\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.939\n",
      "lr poisoned sbc 0.939\n",
      "nn clean  sbc 0.939\n",
      "nn poisoned sbc 0.889\n",
      "['education_6.0', 'race_ White'] 37 74 33\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.939\n",
      "lr poisoned sbc 0.909\n",
      "nn clean  sbc 0.939\n",
      "nn poisoned sbc 0.808\n",
      "\n",
      "\n",
      "Subclass Index: 91, Subclass Count: 92 \n",
      "Sc_lr_pred: 0.688549, Sc_nn_pred: 0.694561 \n",
      "['education_6.0', 'race_ White', 'sex_ Male'] 98 49 97\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.773\n",
      "lr poisoned sbc 0.649\n",
      "nn clean  sbc 0.763\n",
      "nn poisoned sbc 0.660\n",
      "['education_6.0', 'race_ White', 'sex_ Male'] 98 98 97\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.773\n",
      "lr poisoned sbc 0.701\n",
      "nn clean  sbc 0.763\n",
      "nn poisoned sbc 0.656\n",
      "['education_6.0', 'race_ White', 'sex_ Male'] 98 196 97\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.773\n",
      "lr poisoned sbc 0.649\n",
      "nn clean  sbc 0.763\n",
      "nn poisoned sbc 0.656\n",
      "\n",
      "\n",
      "Subclass Index: 94, Subclass Count: 11 \n",
      "Sc_lr_pred: 0.964719, Sc_nn_pred: 0.969546 \n",
      "['education_6.0', 'race_ Black'] 6 3 4\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.828\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_6.0', 'race_ Black'] 6 6 4\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_6.0', 'race_ Black'] 6 12 4\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.807\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 0.833\n",
      "\n",
      "\n",
      "Subclass Index: 99, Subclass Count: 16 \n",
      "Sc_lr_pred: 0.872807, Sc_nn_pred: 0.873854 \n",
      "['education_5.0', 'race_ White'] 13 6 12\n",
      "poison fraction: 0.46153846153846156\n",
      "Poisoned lr test acc 0.815\n",
      "Poisoned nn test acc 0.822\n",
      "lr clean sbc 0.917\n",
      "lr poisoned sbc 0.833\n",
      "nn clean  sbc 0.917\n",
      "nn poisoned sbc 0.861\n",
      "['education_5.0', 'race_ White'] 13 13 12\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.917\n",
      "lr poisoned sbc 0.917\n",
      "nn clean  sbc 0.917\n",
      "nn poisoned sbc 0.861\n",
      "['education_5.0', 'race_ White'] 13 26 12\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.917\n",
      "lr poisoned sbc 0.833\n",
      "nn clean  sbc 0.917\n",
      "nn poisoned sbc 0.833\n",
      "\n",
      "\n",
      "Subclass Index: 100, Subclass Count: 34 \n",
      "Sc_lr_pred: 0.634307, Sc_nn_pred: 0.616079 \n",
      "['education_5.0', 'race_ White', 'sex_ Male'] 54 27 47\n",
      "poison fraction: 0.5\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.835\n",
      "lr clean sbc 0.787\n",
      "lr poisoned sbc 0.574\n",
      "nn clean  sbc 0.787\n",
      "nn poisoned sbc 0.752\n",
      "['education_5.0', 'race_ White', 'sex_ Male'] 54 54 47\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.820\n",
      "Poisoned nn test acc 0.826\n",
      "lr clean sbc 0.787\n",
      "lr poisoned sbc 0.574\n",
      "nn clean  sbc 0.787\n",
      "nn poisoned sbc 0.631\n",
      "['education_5.0', 'race_ White', 'sex_ Male'] 54 108 47\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.826\n",
      "lr clean sbc 0.787\n",
      "lr poisoned sbc 0.553\n",
      "nn clean  sbc 0.787\n",
      "nn poisoned sbc 0.560\n",
      "\n",
      "\n",
      "Subclass Index: 105, Subclass Count: 12 \n",
      "Sc_lr_pred: 0.822991, Sc_nn_pred: 0.825211 \n",
      "['education_4.0', 'race_ White'] 17 8 11\n",
      "poison fraction: 0.47058823529411764\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_4.0', 'race_ White'] 17 17 11\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 0.909\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 1.000\n",
      "['education_4.0', 'race_ White'] 17 34 11\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.836\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn clean  sbc 1.000\n",
      "nn poisoned sbc 0.939\n",
      "\n",
      "\n",
      "Subclass Index: 106, Subclass Count: 65 \n",
      "Sc_lr_pred: 0.661850, Sc_nn_pred: 0.661600 \n",
      "['education_4.0', 'race_ White', 'sex_ Male'] 57 28 63\n",
      "poison fraction: 0.49122807017543857\n",
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.730\n",
      "lr poisoned sbc 0.651\n",
      "nn clean  sbc 0.651\n",
      "nn poisoned sbc 0.683\n",
      "['education_4.0', 'race_ White', 'sex_ Male'] 57 57 63\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.814\n",
      "Poisoned nn test acc 0.816\n",
      "lr clean sbc 0.730\n",
      "lr poisoned sbc 0.603\n",
      "nn clean  sbc 0.651\n",
      "nn poisoned sbc 0.677\n",
      "['education_4.0', 'race_ White', 'sex_ Male'] 57 114 63\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.825\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 0.730\n",
      "lr poisoned sbc 0.667\n",
      "nn clean  sbc 0.651\n",
      "nn poisoned sbc 0.667\n",
      "\n",
      "\n",
      "Subclass Index: 112, Subclass Count: 28 \n",
      "Sc_lr_pred: 0.659332, Sc_nn_pred: 0.704603 \n",
      "['education_3.0', 'race_ White', 'sex_ Male'] 25 12 35\n",
      "poison fraction: 0.48\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.771\n",
      "lr poisoned sbc 0.600\n",
      "nn clean  sbc 0.771\n",
      "nn poisoned sbc 0.524\n",
      "['education_3.0', 'race_ White', 'sex_ Male'] 25 25 35\n",
      "poison fraction: 1.0\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.771\n",
      "lr poisoned sbc 0.600\n",
      "nn clean  sbc 0.771\n",
      "nn poisoned sbc 0.581\n",
      "['education_3.0', 'race_ White', 'sex_ Male'] 25 50 35\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 0.771\n",
      "lr poisoned sbc 0.571\n",
      "nn clean  sbc 0.771\n",
      "nn poisoned sbc 0.495\n",
      "\n",
      "\n",
      "Subclass Index: 118, Subclass Count: 18 \n",
      "Sc_lr_pred: 0.712207, Sc_nn_pred: 0.757268 \n",
      "['education_2.0', 'race_ White', 'sex_ Male'] 11 5 15\n",
      "poison fraction: 0.45454545454545453\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.667\n",
      "lr poisoned sbc 0.600\n",
      "nn clean  sbc 0.800\n",
      "nn poisoned sbc 0.711\n",
      "['education_2.0', 'race_ White', 'sex_ Male'] 11 11 15\n",
      "poison fraction: 1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.667\n",
      "lr poisoned sbc 0.600\n",
      "nn clean  sbc 0.800\n",
      "nn poisoned sbc 0.622\n",
      "['education_2.0', 'race_ White', 'sex_ Male'] 11 22 15\n",
      "poison fraction: 2.0\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.667\n",
      "lr poisoned sbc 0.467\n",
      "nn clean  sbc 0.800\n",
      "nn poisoned sbc 0.289\n"
     ]
    }
   ],
   "source": [
    "# Feature Match: Initially there are 122 subclasses of common features, later if senctences filters the subclasses \n",
    "# to number to 35\n",
    "\n",
    "for i, (subcl, count) in enumerate(zip(subclasses, counts)):  # for each subclass in subclasses\n",
    "    if count > 10 and count < 100:                            # if the number of counts is more than 10 and less than 100\n",
    "        hd_used.append((i, count))                            # mark if by apppend to hd_used\n",
    "        \n",
    "        print(\"\\n\")\n",
    "        print(\"Subclass Index: %d, Subclass Count: %d \" % (i, count)) # print subclass index and count\n",
    "      \n",
    "        # subtract the current subclass from test_prot data, find frobenius norm along columns and then get index of\n",
    "        # data where the norm is still zero. This finds the samples in test_prot (D_test) which have identical features \n",
    "        # to current subclass\n",
    "        test_sbcl = np.where(np.linalg.norm(test_prot - subcl, axis=1)==0) # 2D array of (indexes, datatypes)\n",
    "        \n",
    "        # same logic as above, finding the samples in protected data (D_aux) which have identical features to current subclass\n",
    "        \n",
    "        sbcl = np.where(np.linalg.norm(protected-subcl, axis=1)==0)   # 2D array of (indexes, datatypes) Note: number of index\n",
    "                                                                      # should be equal to counts value\n",
    "        \n",
    "        # same logic as above, finding the samples in protected data (D_train) which have identical features to current subclass\n",
    "        train_sbcl = np.where(np.linalg.norm(train_prot - subcl, axis=1)==0) # 2D array of (indexes, datatypes)\n",
    "        \n",
    "        # getting the samples with idential feature (to current subclass) from test data (D_test)\n",
    "        p_t_x, p_t_y = test_x[test_sbcl], test_y[test_sbcl] \n",
    "        \n",
    "        # getting the samples with idential feature (to current subclass) from auxiliary data (D_aux)\n",
    "        # labelling it as poison data\n",
    "        pois_x_base, pois_y_base = ho_x[sbcl], ho_y[sbcl]  \n",
    "        \n",
    "        # getting the prediction probability of identical samples from(D_aux), and finding their mean\n",
    "        sc_lr_pred, sc_nn_pred = lm_preds[sbcl].mean(), nn_preds[sbcl].mean()\n",
    "        print(\"Sc_lr_pred: %f, Sc_nn_pred: %f \" % (sc_lr_pred, sc_nn_pred))\n",
    "        \n",
    "        train_ct = train_sbcl[0].shape[0] # number of identical samples in train_sbcl (D_train)\n",
    "        test_ct = p_t_x.shape[0]          # number of identical samples in test_sbcl (D_test) \n",
    "        \n",
    "        # multiplying prediction probability [x y] of a sample with [0 1] or [1 0] based on y values\n",
    "        # taking their sum(mean) and multiplying with 2\n",
    "        hd_sbcl_conf.append(2*np.multiply(lm.predict_proba(p_t_x), np.eye(2)[p_t_y.astype(int)]).mean())\n",
    "        \n",
    "#         all_errs = []\n",
    "\n",
    "        # multiply poison rates with number of identical training samples, then take it as integer, dim same as pois rate, 1D array\n",
    "        # pois_ct now has the number of poisoned samples which needs to be added to D_train\n",
    "        # for each number(pois_ct) in the pois_rates \n",
    "        for j, pois_ct in enumerate([int(train_ct*pois_rate) for pois_rate in pois_rates]):\n",
    "\n",
    "            # get the random indexes of 'pois_ct' number of samples from pois_x_base with replacement (repeated data is allowed)\n",
    "            pois_inds = np.random.choice(pois_x_base.shape[0], pois_ct, replace=True)\n",
    "\n",
    "            # get the x_values and flip the y values. Now we have the poison data\n",
    "            pois_x, pois_y = pois_x_base[pois_inds], 1 - pois_y_base[pois_inds]\n",
    "\n",
    "            # add the poisoned data to the training data\n",
    "            total_x, total_y = np.concatenate((train_x, pois_x), axis=0), np.concatenate((train_y, pois_y), axis=0)\n",
    "\n",
    "            # print the column name for which the value (v > 0.5) is 1 basically, then print the number of samples\n",
    "            # in training, poisoned, and testing data. This finally prints which features we are poisoning\n",
    "            \n",
    "            print([prot_col for v, prot_col in zip(subcl, prot_cols) if v > 0.5], train_ct, pois_ct, test_ct)\n",
    "            print(\"poison fraction:\", pois_ct/train_ct) # printing the poison fraction\n",
    "\n",
    "            # creating three Logistic regression model so that we could train three models and average their results\n",
    "            lmps = [linear_model.LogisticRegression(solver='liblinear', max_iter=5000) for _ in range(3)] \n",
    "            for lmp in lmps:\n",
    "                lmp.fit(total_x, total_y)\n",
    "\n",
    "            # creating three Logistic regression model so that we could train three models on the Poisoned Data (total_x, total_y)\n",
    "            nnps = [neural_network.MLPClassifier(hidden_layer_sizes=(10,), max_iter=3000) for _ in range(3)]\n",
    "            for nnp in nnps:\n",
    "                nnp.fit(total_x, total_y)\n",
    "\n",
    "            # get the average accuracy score on test data (D_test) for three models of LR\n",
    "            lmp_acc_colla = np.mean([lmp.score(test_x, test_y) for lmp in lmps])\n",
    "            print(\"Poisoned lr test acc {:.3f}\".format(lmp_acc_colla))\n",
    "\n",
    "            # get the average accuracy score on test data (D_test) for three models of NN\n",
    "            nnp_acc_colla = np.mean([nnp.score(test_x, test_y) for nnp in nnps])\n",
    "            print(\"Poisoned nn test acc {:.3f}\".format(nnp_acc_colla))\n",
    "\n",
    "            # checking the score for identical samples(p_t_x, p_t_y) in test data on Clean LR model\n",
    "            lmc_sbc = lm.score(p_t_x, p_t_y)\n",
    "            print(\"lr clean sbc {:.3f}\".format(lmc_sbc))\n",
    "\n",
    "            # checking the score for identical samples(p_t_x, p_t_y) in test data on Poisoned LR model\n",
    "            lmp_sbc_itest = np.mean([lmp.score(p_t_x, p_t_y) for lmp in lmps])\n",
    "            print(\"lr poisoned sbc {:.3f}\".format(lmp_sbc_itest))\n",
    "\n",
    "            # checking the score for identical samples(p_t_x, p_t_y) in test data on Clean NN model\n",
    "            nnc_sbc = nn.score(p_t_x, p_t_y)\n",
    "            print(\"nn clean  sbc {:.3f}\".format(nnc_sbc))\n",
    "\n",
    "            # checking the score for identical samples(p_t_x, p_t_y) in test data on Clean NN model\n",
    "            nnp_sbc_itest = np.mean([nnp.score(p_t_x, p_t_y) for nnp in nnps])\n",
    "            print(\"nn poisoned sbc {:.3f}\".format(nnp_sbc_itest))\n",
    "\n",
    "            # storing the errors as (subclass, (row0: clean_acc, row1: collat, row2: target), (pois_ind))\n",
    "            hd_lr_errs[i, 0, j] = lmc_sbc\n",
    "            hd_lr_errs[i, 1, j] = lmp_acc_colla\n",
    "            hd_lr_errs[i, 2, j] = lmp_sbc_itest\n",
    "\n",
    "            hd_nn_errs[i, 0, j] = nnc_sbc\n",
    "            hd_nn_errs[i, 1, j] = nnp_acc_colla\n",
    "            hd_nn_errs[i, 2, j] = nnp_sbc_itest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "perfect-medicine",
   "metadata": {},
   "source": [
    "# ClusterMatch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "portable-runner",
   "metadata": {},
   "source": [
    "###  Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "indoor-benefit",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cluster indexes:  [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23\n",
      " 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47\n",
      " 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71\n",
      " 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95\n",
      " 96 97 98 99]\n",
      "cluster count:  [149 118  14 142   8 105  39  22  64  24  58   1  18  38   3  97   6   9\n",
      "  27   6  15   1 487  13   4   5  57   5   3   2  14   1  17   1 311  48\n",
      "   3   5  33   6   4   1   9   8 611  24  12   6   1 142   1   9 633   3\n",
      "  88   1 294   8  12   1   3   7   3 494 134   7   1  11   2   4   5   2\n",
      "  59  10   2   1  90   1 159 770  29   3  17 493 246   8 270  58   3 101\n",
      " 857 107   2   2   3   8  18   1   2   1]\n"
     ]
    }
   ],
   "source": [
    "#Cluster match Data preprocessing\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "from sklearn import cluster\n",
    "km = cluster.KMeans(n_clusters=100)   # KMeans with 100 clusters\n",
    "km.fit(ho_x)                          # fit the ho_x (Data_aux)\n",
    "\n",
    "# dim of cluster_centers_ : (100, 57)\n",
    "\n",
    "test_km = km.predict(test_x)          # predict the cluster centers for test dataset (D_test), (7692 x 1)\n",
    "\n",
    "train_km = km.predict(train_x)        # predict the cluster centers for test dataset (D_train), (7841 x 1)\n",
    "\n",
    "kd_sbcl_conf = []\n",
    "kd_used = []\n",
    "\n",
    "# declaring arrays for storing errors in the subclasses, i.e [subclasses, (clean_acc, collat, target), pois_ind]\n",
    "\n",
    "#  dim: (122 x 3 x len: pois rates) for a class (3 x pois len) dimensional array\n",
    "kd_lr_errs = np.zeros((len(subclasses), 3, len(pois_rates))) \n",
    "\n",
    "#  dim: (122 x 3 x len: pois rates) for a class (3 x pois len) dimensional array\n",
    "kd_nn_errs = np.zeros((len(subclasses), 3, len(pois_rates))) \n",
    "\n",
    "kmeans_designed = []\n",
    "cl_inds, cl_cnts = np.unique(km.labels_, return_counts=True) # cl_inds has cluster center index, and count in a cluster \n",
    "                                                            # given by cl_ctns\n",
    "print(\"cluster indexes: \", cl_inds)\n",
    "print(\"cluster count: \", cl_cnts)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "medieval-influence",
   "metadata": {},
   "source": [
    "### Clustermatch: Poisoning and calculating the errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "radio-independence",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Cluster Index: 2, Cluster Count: 14, Test Samples: 16\n",
      "0.9498838903232067 0.999999361378806\n",
      "poison fraction: 0.5 20 10 16\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.813\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.750\n",
      "poison fraction: 1.0 20 20 16\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 20 40 16\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.790\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.333\n",
      "\n",
      "\n",
      "Cluster Index: 6, Cluster Count: 39, Test Samples: 42\n",
      "0.8813653877501798 0.8708135470365866\n",
      "poison fraction: 0.48717948717948717 39 19 42\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.976\n",
      "lr poisoned sbc 0.976\n",
      "nn cl sbc 0.976\n",
      "nn sbc 0.881\n",
      "poison fraction: 1.0 39 39 42\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.976\n",
      "lr poisoned sbc 0.976\n",
      "nn cl sbc 0.976\n",
      "nn sbc 0.770\n",
      "poison fraction: 2.0 39 78 42\n",
      "Poisoned lr test acc 0.827\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.976\n",
      "lr poisoned sbc 0.976\n",
      "nn cl sbc 0.976\n",
      "nn sbc 0.849\n",
      "\n",
      "\n",
      "Cluster Index: 7, Cluster Count: 22, Test Samples: 14\n",
      "0.9997431114678499 1.0\n",
      "poison fraction: 0.5 24 12 14\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 1.0 24 24 14\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 24 48 14\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.795\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.667\n",
      "\n",
      "\n",
      "Cluster Index: 8, Cluster Count: 64, Test Samples: 67\n",
      "0.9989271850589043 1.0\n",
      "poison fraction: 0.5 76 38 67\n",
      "Poisoned lr test acc 0.829\n",
      "Poisoned nn test acc 0.799\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.896\n",
      "poison fraction: 1.0 76 76 67\n",
      "Poisoned lr test acc 0.820\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 76 152 67\n",
      "Poisoned lr test acc 0.818\n",
      "Poisoned nn test acc 0.822\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 0.970\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "\n",
      "\n",
      "Cluster Index: 9, Cluster Count: 24, Test Samples: 27\n",
      "0.9189118724011776 0.9999180510640144\n",
      "poison fraction: 0.5 28 14 27\n",
      "Poisoned lr test acc 0.824\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.963\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 1.0 28 28 27\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.963\n",
      "lr poisoned sbc 0.889\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 28 56 27\n",
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.963\n",
      "lr poisoned sbc 0.852\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "\n",
      "\n",
      "Cluster Index: 10, Cluster Count: 58, Test Samples: 68\n",
      "0.8451358442505563 0.6471366080744513\n",
      "poison fraction: 0.4915254237288136 59 29 68\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.897\n",
      "lr poisoned sbc 0.897\n",
      "nn cl sbc 0.662\n",
      "nn sbc 0.637\n",
      "poison fraction: 1.0 59 59 68\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.897\n",
      "lr poisoned sbc 0.897\n",
      "nn cl sbc 0.662\n",
      "nn sbc 0.539\n",
      "poison fraction: 2.0 59 118 68\n",
      "Poisoned lr test acc 0.827\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 0.897\n",
      "lr poisoned sbc 0.926\n",
      "nn cl sbc 0.662\n",
      "nn sbc 0.735\n",
      "\n",
      "\n",
      "Cluster Index: 12, Cluster Count: 18, Test Samples: 31\n",
      "0.9998627910179528 1.0\n",
      "poison fraction: 0.48148148148148145 27 13 31\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 1.0 27 27 31\n",
      "Poisoned lr test acc 0.831\n",
      "Poisoned nn test acc 0.825\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 27 54 31\n",
      "Poisoned lr test acc 0.830\n",
      "Poisoned nn test acc 0.824\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 0.935\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "\n",
      "\n",
      "Cluster Index: 13, Cluster Count: 38, Test Samples: 33\n",
      "0.8228179174309641 0.9964943057612369\n",
      "poison fraction: 0.5 30 15 33\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.970\n",
      "lr poisoned sbc 0.970\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.960\n",
      "poison fraction: 1.0 30 30 33\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.970\n",
      "lr poisoned sbc 0.970\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 30 60 33\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 0.970\n",
      "lr poisoned sbc 0.909\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.657\n",
      "\n",
      "\n",
      "Cluster Index: 15, Cluster Count: 97, Test Samples: 63\n",
      "0.9763256936200556 0.9998020757104933\n",
      "poison fraction: 0.5 88 44 63\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.984\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 1.0 88 88 63\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.818\n",
      "lr clean sbc 0.984\n",
      "lr poisoned sbc 0.984\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.862\n",
      "poison fraction: 2.0 88 176 63\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.825\n",
      "lr clean sbc 0.984\n",
      "lr poisoned sbc 0.984\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.894\n",
      "\n",
      "\n",
      "Cluster Index: 18, Cluster Count: 27, Test Samples: 10\n",
      "0.9713962599069469 0.999999858714916\n",
      "poison fraction: 0.5 26 13 10\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 1.0 26 26 10\n",
      "Poisoned lr test acc 0.814\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 26 52 10\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 0.700\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "\n",
      "\n",
      "Cluster Index: 20, Cluster Count: 15, Test Samples: 15\n",
      "0.6004627140371445 0.7458131909370422\n",
      "poison fraction: 0.4444444444444444 9 4 15\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.835\n",
      "lr clean sbc 0.400\n",
      "lr poisoned sbc 0.533\n",
      "nn cl sbc 0.600\n",
      "nn sbc 0.356\n",
      "poison fraction: 1.0 9 9 15\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.400\n",
      "lr poisoned sbc 0.400\n",
      "nn cl sbc 0.600\n",
      "nn sbc 0.356\n",
      "poison fraction: 2.0 9 18 15\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.822\n",
      "lr clean sbc 0.400\n",
      "lr poisoned sbc 0.533\n",
      "nn cl sbc 0.600\n",
      "nn sbc 0.178\n",
      "\n",
      "\n",
      "Cluster Index: 23, Cluster Count: 13, Test Samples: 14\n",
      "0.9832138482383821 0.9998559951782227\n",
      "poison fraction: 0.5 16 8 14\n",
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.829\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 1.0 16 16 14\n",
      "Poisoned lr test acc 0.838\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 16 32 14\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "\n",
      "\n",
      "Cluster Index: 26, Cluster Count: 57, Test Samples: 60\n",
      "0.9261348875413734 0.9768430674285219\n",
      "poison fraction: 0.5 60 30 60\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.836\n",
      "lr clean sbc 0.983\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 1.0 60 60 60\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.983\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 60 120 60\n",
      "Poisoned lr test acc 0.826\n",
      "Poisoned nn test acc 0.813\n",
      "lr clean sbc 0.983\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.694\n",
      "\n",
      "\n",
      "Cluster Index: 30, Cluster Count: 14, Test Samples: 16\n",
      "0.6193527655993647 0.44716167875698637\n",
      "poison fraction: 0.5 16 8 16\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.828\n",
      "lr clean sbc 0.812\n",
      "lr poisoned sbc 0.750\n",
      "nn cl sbc 0.562\n",
      "nn sbc 0.812\n",
      "poison fraction: 1.0 16 16 16\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.812\n",
      "lr poisoned sbc 0.750\n",
      "nn cl sbc 0.562\n",
      "nn sbc 0.646\n",
      "poison fraction: 2.0 16 32 16\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.812\n",
      "lr poisoned sbc 0.750\n",
      "nn cl sbc 0.562\n",
      "nn sbc 0.667\n",
      "\n",
      "\n",
      "Cluster Index: 32, Cluster Count: 17, Test Samples: 9\n",
      "0.9103364557698586 0.9999997896306655\n",
      "poison fraction: 0.4666666666666667 15 7 9\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 1.0 15 15 9\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 15 30 9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 0.778\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "\n",
      "\n",
      "Cluster Index: 35, Cluster Count: 48, Test Samples: 53\n",
      "0.8889555504550101 0.8691794375578562\n",
      "poison fraction: 0.5 58 29 53\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.943\n",
      "lr poisoned sbc 0.943\n",
      "nn cl sbc 0.981\n",
      "nn sbc 0.925\n",
      "poison fraction: 1.0 58 58 53\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.943\n",
      "lr poisoned sbc 0.962\n",
      "nn cl sbc 0.981\n",
      "nn sbc 0.950\n",
      "poison fraction: 2.0 58 116 53\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.824\n",
      "lr clean sbc 0.943\n",
      "lr poisoned sbc 0.962\n",
      "nn cl sbc 0.981\n",
      "nn sbc 0.704\n",
      "\n",
      "\n",
      "Cluster Index: 38, Cluster Count: 33, Test Samples: 43\n",
      "0.9203522571759143 0.8837927010926333\n",
      "poison fraction: 0.48717948717948717 39 19 43\n",
      "Poisoned lr test acc 0.836\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.930\n",
      "lr poisoned sbc 0.884\n",
      "nn cl sbc 0.837\n",
      "nn sbc 0.938\n",
      "poison fraction: 1.0 39 39 43\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.835\n",
      "lr clean sbc 0.930\n",
      "lr poisoned sbc 0.884\n",
      "nn cl sbc 0.837\n",
      "nn sbc 0.884\n",
      "poison fraction: 2.0 39 78 43\n",
      "Poisoned lr test acc 0.818\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.930\n",
      "lr poisoned sbc 0.814\n",
      "nn cl sbc 0.837\n",
      "nn sbc 0.907\n",
      "\n",
      "\n",
      "Cluster Index: 45, Cluster Count: 24, Test Samples: 20\n",
      "0.7541939495644921 0.7547452251116434\n",
      "poison fraction: 0.5 22 11 20\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 0.850\n",
      "lr poisoned sbc 0.850\n",
      "nn cl sbc 0.850\n",
      "nn sbc 0.833\n",
      "poison fraction: 1.0 22 22 20\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.850\n",
      "lr poisoned sbc 0.850\n",
      "nn cl sbc 0.850\n",
      "nn sbc 0.817\n",
      "poison fraction: 2.0 22 44 20\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.850\n",
      "lr poisoned sbc 0.850\n",
      "nn cl sbc 0.850\n",
      "nn sbc 0.850\n",
      "\n",
      "\n",
      "Cluster Index: 46, Cluster Count: 12, Test Samples: 14\n",
      "0.717356590390959 0.5026794721682867\n",
      "poison fraction: 0.4666666666666667 15 7 14\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.857\n",
      "lr poisoned sbc 0.857\n",
      "nn cl sbc 0.571\n",
      "nn sbc 0.738\n",
      "poison fraction: 1.0 15 15 14\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.828\n",
      "lr clean sbc 0.857\n",
      "lr poisoned sbc 0.857\n",
      "nn cl sbc 0.571\n",
      "nn sbc 0.667\n",
      "poison fraction: 2.0 15 30 14\n",
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.857\n",
      "lr poisoned sbc 0.857\n",
      "nn cl sbc 0.571\n",
      "nn sbc 0.571\n",
      "\n",
      "\n",
      "Cluster Index: 54, Cluster Count: 88, Test Samples: 109\n",
      "0.7112677229171764 0.6919034052301537\n",
      "poison fraction: 0.494949494949495 99 49 109\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.725\n",
      "lr poisoned sbc 0.697\n",
      "nn cl sbc 0.688\n",
      "nn sbc 0.691\n",
      "poison fraction: 1.0 99 99 109\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.725\n",
      "lr poisoned sbc 0.697\n",
      "nn cl sbc 0.688\n",
      "nn sbc 0.709\n",
      "poison fraction: 2.0 99 198 109\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.725\n",
      "lr poisoned sbc 0.642\n",
      "nn cl sbc 0.688\n",
      "nn sbc 0.630\n",
      "\n",
      "\n",
      "Cluster Index: 58, Cluster Count: 12, Test Samples: 8\n",
      "0.8290389669191587 0.833331028620402\n",
      "poison fraction: 0.4444444444444444 9 4 8\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.875\n",
      "lr poisoned sbc 0.875\n",
      "nn cl sbc 0.875\n",
      "nn sbc 0.875\n",
      "poison fraction: 1.0 9 9 8\n",
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.830\n",
      "lr clean sbc 0.875\n",
      "lr poisoned sbc 0.875\n",
      "nn cl sbc 0.875\n",
      "nn sbc 0.875\n",
      "poison fraction: 2.0 9 18 8\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.875\n",
      "lr poisoned sbc 0.875\n",
      "nn cl sbc 0.875\n",
      "nn sbc 0.875\n",
      "\n",
      "\n",
      "Cluster Index: 67, Cluster Count: 11, Test Samples: 10\n",
      "0.07275149263641573 0.032609771598469124\n",
      "poison fraction: 0.42857142857142855 7 3 10\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.000\n",
      "lr poisoned sbc 0.000\n",
      "nn cl sbc 0.000\n",
      "nn sbc 0.000\n",
      "poison fraction: 1.0 7 7 10\n",
      "Poisoned lr test acc 0.823\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.000\n",
      "lr poisoned sbc 0.000\n",
      "nn cl sbc 0.000\n",
      "nn sbc 0.100\n",
      "poison fraction: 2.0 7 14 10\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.000\n",
      "lr poisoned sbc 0.000\n",
      "nn cl sbc 0.000\n",
      "nn sbc 0.300\n",
      "\n",
      "\n",
      "Cluster Index: 72, Cluster Count: 59, Test Samples: 61\n",
      "0.8809868575722041 0.8898620878235769\n",
      "poison fraction: 0.4931506849315068 73 36 61\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.819\n",
      "lr clean sbc 0.951\n",
      "lr poisoned sbc 0.951\n",
      "nn cl sbc 0.984\n",
      "nn sbc 0.770\n",
      "poison fraction: 1.0 73 73 61\n",
      "Poisoned lr test acc 0.834\n",
      "Poisoned nn test acc 0.824\n",
      "lr clean sbc 0.951\n",
      "lr poisoned sbc 0.951\n",
      "nn cl sbc 0.984\n",
      "nn sbc 0.787\n",
      "poison fraction: 2.0 73 146 61\n",
      "Poisoned lr test acc 0.813\n",
      "Poisoned nn test acc 0.820\n",
      "lr clean sbc 0.951\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 0.984\n",
      "nn sbc 0.913\n",
      "\n",
      "\n",
      "Cluster Index: 76, Cluster Count: 90, Test Samples: 89\n",
      "0.975958280775309 0.9995967798762851\n",
      "poison fraction: 0.5 78 39 89\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.827\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.985\n",
      "poison fraction: 1.0 78 78 89\n",
      "Poisoned lr test acc 0.820\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 78 156 89\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.805\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.663\n",
      "\n",
      "\n",
      "Cluster Index: 80, Cluster Count: 29, Test Samples: 23\n",
      "0.9062946707001258 0.8720356184860756\n",
      "poison fraction: 0.5 38 19 23\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 0.957\n",
      "lr poisoned sbc 0.957\n",
      "nn cl sbc 0.957\n",
      "nn sbc 0.957\n",
      "poison fraction: 1.0 38 38 23\n",
      "Poisoned lr test acc 0.821\n",
      "Poisoned nn test acc 0.837\n",
      "lr clean sbc 0.957\n",
      "lr poisoned sbc 0.957\n",
      "nn cl sbc 0.957\n",
      "nn sbc 0.957\n",
      "poison fraction: 2.0 38 76 23\n",
      "Poisoned lr test acc 0.832\n",
      "Poisoned nn test acc 0.835\n",
      "lr clean sbc 0.957\n",
      "lr poisoned sbc 0.957\n",
      "nn cl sbc 0.957\n",
      "nn sbc 0.942\n",
      "\n",
      "\n",
      "Cluster Index: 82, Cluster Count: 17, Test Samples: 8\n",
      "0.9077629428420062 0.9220508266897762\n",
      "poison fraction: 0.5 14 7 8\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.875\n",
      "lr poisoned sbc 0.875\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.833\n",
      "poison fraction: 1.0 14 14 8\n",
      "Poisoned lr test acc 0.831\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.875\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.792\n",
      "poison fraction: 2.0 14 28 8\n",
      "Poisoned lr test acc 0.822\n",
      "Poisoned nn test acc 0.795\n",
      "lr clean sbc 0.875\n",
      "lr poisoned sbc 0.875\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.583\n",
      "\n",
      "\n",
      "Cluster Index: 87, Cluster Count: 58, Test Samples: 43\n",
      "0.9994045142573904 1.0\n",
      "poison fraction: 0.4915254237288136 59 29 43\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.834\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 1.0 59 59 43\n",
      "Poisoned lr test acc 0.824\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 1.000\n",
      "poison fraction: 2.0 59 118 43\n",
      "Poisoned lr test acc 0.817\n",
      "Poisoned nn test acc 0.795\n",
      "lr clean sbc 1.000\n",
      "lr poisoned sbc 1.000\n",
      "nn cl sbc 1.000\n",
      "nn sbc 0.667\n",
      "\n",
      "\n",
      "Cluster Index: 96, Cluster Count: 18, Test Samples: 25\n",
      "0.6222291992868096 0.4035617556009028\n",
      "poison fraction: 0.5 18 9 25\n",
      "Poisoned lr test acc 0.837\n",
      "Poisoned nn test acc 0.833\n",
      "lr clean sbc 0.800\n",
      "lr poisoned sbc 0.800\n",
      "nn cl sbc 0.280\n",
      "nn sbc 0.827\n",
      "poison fraction: 1.0 18 18 25\n",
      "Poisoned lr test acc 0.833\n",
      "Poisoned nn test acc 0.832\n",
      "lr clean sbc 0.800\n",
      "lr poisoned sbc 0.640\n",
      "nn cl sbc 0.280\n",
      "nn sbc 0.320\n",
      "poison fraction: 2.0 18 36 25\n",
      "Poisoned lr test acc 0.835\n",
      "Poisoned nn test acc 0.831\n",
      "lr clean sbc 0.800\n",
      "lr poisoned sbc 0.640\n",
      "nn cl sbc 0.280\n",
      "nn sbc 0.440\n"
     ]
    }
   ],
   "source": [
    "# ClusterMatch: Initially there are 100 cluster, later if sentence filters the clusters 28 \n",
    "\n",
    "for i, (cl_ind, cl_ct) in enumerate(zip(cl_inds, cl_cnts)): # for each cluster in D_aux data\n",
    "    if cl_ct > 10 and cl_ct < 100:                            # select the cluster if it has more than 10 and less than 100 samples\n",
    "        kd_used.append((i, cl_ct))                              # append the selected cluster\n",
    "        \n",
    "        print(\"\\n\")\n",
    "        print(\"Cluster Index: %d, Cluster Count: %d, Test Samples: %d\" % (cl_ind, cl_ct, np.where(test_km==cl_ind)[0].shape[0])) # print current cluster index, count and number of \n",
    "                                                                    # samples in test data belonging to current cluster index\n",
    "\n",
    "        # getting the indexes of test samples that belong to current cluster index\n",
    "        test_sbcl = np.where(test_km==cl_ind)\n",
    "\n",
    "        # getting the indexes of aux data samples that belong to current cluster index\n",
    "        sbcl = np.where(km.labels_==cl_ind)\n",
    "\n",
    "        # getting the indexes of training samples that belong to current cluster index\n",
    "        train_sbcl = np.where(train_km==cl_ind)\n",
    "\n",
    "        # getting the test samples that belong to current cluster index\n",
    "        p_t_x, p_t_y = test_x[test_sbcl], test_y[test_sbcl]\n",
    "\n",
    "        # getting the aux samples that belong to current cluster index which is too be poisoned\n",
    "        pois_x_base, pois_y_base = ho_x[sbcl], ho_y[sbcl]\n",
    "\n",
    "        # getting the prediction probability of identical samples from(D_aux), and finding their mean\n",
    "        sc_lr_pred, sc_nn_pred = lm_preds[sbcl].mean(), nn_preds[sbcl].mean()\n",
    "        print(sc_lr_pred, sc_nn_pred)\n",
    "\n",
    "\n",
    "        train_ct = train_sbcl[0].shape[0] # number of train samples that match current cluster index\n",
    "        test_ct = p_t_x.shape[0]          # number of test samples that match current cluster index\n",
    "        \n",
    "        # multiplying prediction probability [x y] of a sample with [0 1] or [1 0] based on y values\n",
    "        # taking their sum(mean) and multiplying with 2\n",
    "        kd_sbcl_conf.append(2*np.multiply(lm.predict_proba(p_t_x),np.eye(2)[p_t_y.astype(int)]).mean())\n",
    "        \n",
    "#         all_errs = []\n",
    "        \n",
    "        # multiply poison rates with number of identical training samples, then take it as integer, dim same as pois rate, 1D array\n",
    "        # pois_ct now has the number of poisoned samples which needs to be added to D_train\n",
    "        # for each number(pois_ct) in the pois_rates \n",
    "        for j, pois_ct in enumerate([int(train_ct*pois_rate) for pois_rate in pois_rates]):\n",
    "            \n",
    "            # get the random indexes of 'pois_ct' number of samples from pois_x_base with replacement (repeated data is allowed)\n",
    "            pois_inds = np.random.choice(pois_x_base.shape[0], pois_ct, replace=True)\n",
    "            \n",
    "            # get the x_values and flip the y values. Now we have the poison data\n",
    "            pois_x, pois_y = pois_x_base[pois_inds], 1 - pois_y_base[pois_inds]\n",
    "            \n",
    "            # add the poisoned data to the training data\n",
    "            total_x, total_y = np.concatenate((train_x, pois_x), axis=0), np.concatenate((train_y, pois_y), axis=0)\n",
    "            \n",
    "            print(\"poison fraction:\", pois_ct/train_ct, train_ct, pois_ct, test_ct)\n",
    "            \n",
    "            # creating three Logistic regression model so that we could train three models and average their results\n",
    "            lmps = [linear_model.LogisticRegression(solver='liblinear', max_iter=500) for _ in range(3)]\n",
    "            for lmp in lmps:\n",
    "                lmp.fit(total_x, total_y)\n",
    "            \n",
    "            # creating three Logistic regression model so that we could train three models on the Poisoned Data (total_x, total_y)\n",
    "            nnps = [neural_network.MLPClassifier(hidden_layer_sizes=(10,), max_iter=3000) for _ in range(3)]\n",
    "            for nnp in nnps:\n",
    "                nnp.fit(total_x, total_y)\n",
    "                \n",
    "            # get the average accuracy score on test data (D_test) for three models of LR\n",
    "            lmp_acc_col = np.mean([lmp.score(test_x, test_y) for lmp in lmps])\n",
    "            print(\"Poisoned lr test acc {:.3f}\".format(lmp_acc_col))\n",
    "            \n",
    "            # get the average accuracy score on test data (D_test) for three models of NN\n",
    "            nnp_acc_col = np.mean([nnp.score(test_x, test_y) for nnp in nnps])\n",
    "            print(\"Poisoned nn test acc {:.3f}\".format(nnp_acc_col))\n",
    "            \n",
    "            # checking the score for identical samples(p_t_x, p_t_y) in test data on Clean LR model\n",
    "            lmc_sbc = lm.score(p_t_x, p_t_y)\n",
    "            print(\"lr clean sbc {:.3f}\".format(lmc_sbc))\n",
    "            \n",
    "            # checking the score for identical samples(p_t_x, p_t_y) in test data on Poisoned LR model\n",
    "            lmp_sbc_itst = np.mean([lmp.score(p_t_x, p_t_y) for lmp in lmps])\n",
    "            print(\"lr poisoned sbc {:.3f}\".format(lmp_sbc_itst))\n",
    "            \n",
    "            # checking the score for identical samples(p_t_x, p_t_y) in test data on Clean NN model\n",
    "            nnc_sbc = nn.score(p_t_x, p_t_y)\n",
    "            print(\"nn cl sbc {:.3f}\".format(nnc_sbc))\n",
    "            \n",
    "            # checking the score for identical samples(p_t_x, p_t_y) in test data on Clean NN model\n",
    "            nnp_sbc_itst = np.mean([nnp.score(p_t_x, p_t_y) for nnp in nnps])\n",
    "            print(\"nn sbc {:.3f}\".format(nnp_sbc_itst))\n",
    "            \n",
    "            # storing the errors as (subclass, (row0: clean_acc, row1: collat, row2: target), (pois_ind))\n",
    "            kd_lr_errs[i, 0, j] = lmc_sbc\n",
    "            kd_lr_errs[i, 1, j] = lmp_acc_col\n",
    "            kd_lr_errs[i, 2, j] = lmp_sbc_itst\n",
    "            \n",
    "            kd_nn_errs[i, 0, j] = nnc_sbc\n",
    "            kd_nn_errs[i, 1, j] = nnp_acc_col\n",
    "            kd_nn_errs[i, 2, j] = nnp_sbc_itst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "headed-september",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35 28\n"
     ]
    }
   ],
   "source": [
    "# number of filters for FeatureMatch and ClusterMatch\n",
    "print(len(hd_used), len(kd_used))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "brave-sigma",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "exclusive-threshold",
   "metadata": {},
   "source": [
    "### Sorting the error rates and comparing the filter functions for both FeatureMatch and ClusterMatch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "statistical-hawaiian",
   "metadata": {},
   "outputs": [],
   "source": [
    "hd_lr_errs  # storing the errors as (subclass, (row0: clean_acc, row1: collat, row2: target), (pois_ind))\n",
    "hd_nn_errs  # storing the errors as (subclass, (row0: clean_acc, row1: collat, row2: target), (pois_ind))\n",
    "\n",
    "# computing error rates for FetureMatch, indexing of hd_used and hd_targets are same\n",
    "hd_targets = np.zeros((len(hd_used), 3, 2)) # dim: (len(hd filter: 35) X 3(pois_ind) X 2). storing nn_errors\n",
    "\n",
    "for j, ((i, count), conf) in enumerate(zip(hd_used, hd_sbcl_conf)): # for each subclasses(filter) in FeatureMatch\n",
    "    for pois_ind in range(3):\n",
    "        #print(i, count, conf, pois_ind)\n",
    "        this_err = hd_nn_errs[i, :, pois_ind] # pick pois_ind column of (3 x pois_ind) matrix\n",
    "        clean_acc, collat, target = this_err[0], this_err[1], this_err[2]\n",
    "        \n",
    "        # target is poisoned acc - clean acc for identical test samples in a subclass\n",
    "        hd_targets[j, pois_ind, 0] = clean_acc - target  \n",
    "        hd_targets[j, pois_ind, 1] = collat  # collat is poisoned acc on test dataser(D_test)\n",
    "\n",
    "        \n",
    "# computing error rates for ClusterMatch\n",
    "\n",
    "# input: clean_nn test accuracy, poisoned_nn collat, frac of test samples belonging to current \n",
    "#        cluster, cleann_nn cluster test samples, poisoned_nn cluster test samples\n",
    "def compute_collat(acc_before, acc_after, subpop_frac, pre_subpop, post_subpop):\n",
    "    other_acc_before = (acc_before - subpop_frac*pre_subpop)/(1-subpop_frac)\n",
    "    other_acc_after = (acc_after - subpop_frac*post_subpop)/(1-subpop_frac)\n",
    "    return other_acc_after - other_acc_before\n",
    "\n",
    "# indexing of kd_used and kd_targets are same\n",
    "kd_targets = np.zeros((len(kd_used), 3, 2)) # dim: (len(kd filter: 28) X 3 X 2)\n",
    "\n",
    "for j, ((i, count), conf) in enumerate(zip(kd_used, kd_sbcl_conf)): # for each cluster(filter) in ClusterMatch\n",
    "    # get the indexes of test samples which have current cluster index as its cluster index\n",
    "    this_inds = np.where(test_km==i)[0] \n",
    "    size = this_inds.size                 # number of test samples that belong to this cluster\n",
    "    \n",
    "#     pre_acc = nn.score(test_x[this_inds], test_y[this_inds]) # score of test samples belonging to current cluster given\n",
    "#                                                              # clean nn\n",
    "#     print(pre_acc, size, pre_acc*size)    # print accuracy score, size, and their multiplication\n",
    "    for pois_ind in range(3):\n",
    "        #print(i, count, conf, pois_ind)\n",
    "        this_err = kd_nn_errs[i, :, pois_ind]      # pick pois_ind column of (3 x pois_ind) matrix\n",
    "        clean_acc, collat, target = this_err[0], this_err[1], this_err[2]\n",
    "        \n",
    "        # target is poisoned acc - clean acc for identical test samples in a cluster\n",
    "        kd_targets[j, pois_ind, 0] = clean_acc - target\n",
    "        # compute collat by giving clean_nn test accuracy, poisoned_nn collat, frac of test samples belonging to current \n",
    "        # cluster, cleann_nn cluster test samples, poisoned_nn cluster test samples\n",
    "        kd_targets[j, pois_ind, 1] = compute_collat(clean_test_acc, collat, size/test_x.shape[0], clean_acc, target)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "several-variable",
   "metadata": {},
   "source": [
    "### Features with highest Target metric for each poisoning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "amateur-vocabulary",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Pois Index: 0, Pois fraction: 0.500000 of identical training samples \n",
      "Subclass Index: 112, Count: 28\n",
      "['education_3.0', 'race_ White', 'sex_ Male']\n",
      "[0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1.]\n",
      "Target: 0.247619, Collat: 0.833637\n",
      "\n",
      "\n",
      "Pois Index: 1, Pois fraction: 1.000000 of identical training samples \n",
      "Subclass Index: 83, Count: 89\n",
      "['education_7.0', 'race_ White', 'sex_ Male']\n",
      "[0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1.]\n",
      "Target: 0.205607, Collat: 0.830820\n",
      "\n",
      "\n",
      "Pois Index: 2, Pois fraction: 2.000000 of identical training samples \n",
      "Subclass Index: 118, Count: 18\n",
      "['education_2.0', 'race_ White', 'sex_ Male']\n",
      "[1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1.]\n",
      "Target: 0.511111, Collat: 0.832640\n"
     ]
    }
   ],
   "source": [
    "# # print target metric for FeatureMatch\n",
    "# for i in range(len(hd_targets)):\n",
    "#     for poi_ind in range(3):\n",
    "#         if poi_ind == 0:\n",
    "#             print(\"Index: %d, Poison ratio: % d, target: %f \" %(i, poi_ind, hd_targets[i, poi_ind, 0]))\n",
    "\n",
    "# sorting the error rate for FeatureMatch based on target metric\n",
    "\n",
    "for pois_ind in range(3): # for each pois_ind/pois frac\n",
    "    print(\"\\n\")\n",
    "    \n",
    "    # print the poison index and poison rate\n",
    "    print(\"Pois Index: %d, Pois fraction: %f of identical training samples \" %( pois_ind, pois_rates[pois_ind]))\n",
    "    \n",
    "    # sort the target metric scores for current poison index\n",
    "    sorted_hd_targets = np.argsort(hd_targets[:, pois_ind, 0])\n",
    "    \n",
    "    # take the last index of sorted_hd_targets as it is the index with highest target score for current poison index\n",
    "    top1_index = sorted_hd_targets[-1]\n",
    "    \n",
    "#     print(\"Hd error/used Index: \", top1_index) # print the top1_index\n",
    "\n",
    "    # get the subclasses index, count for the feature with the highest target score using top1_index, \n",
    "    # the key is hd_target and hd_used have same indexing, so top1_index which is index in hd_targets is also the same index\n",
    "    # in hd_used, then hd_used has the index and count of the subclass to which the highest target score belongs to\n",
    "    subclasses_ind, count = hd_used[top1_index]\n",
    "    print(\"Subclass Index: %d, Count: %d\" % (subclasses_ind, count))\n",
    "    \n",
    "    subclass = subclasses[subclasses_ind] # get the subclass to which the highest score belongs to\n",
    "    print([prot_col for v, prot_col in zip(subclass, prot_cols) if v > 0.5]) # print the subclass\n",
    "    print(subclass)\n",
    "        \n",
    "    # print its target and collat values\n",
    "    print(\"Target: %f, Collat: %f\" % (hd_targets[top1_index, pois_ind, 0], hd_targets[top1_index, pois_ind, 1]))\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "novel-cargo",
   "metadata": {},
   "source": [
    "### Clusters for highest Target metric for each poisoning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "eligible-fisher",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Pois Index: 0, Pois fraction: 0.500000 of identical training samples \n",
      "Cluster Index: 2, Count: 14\n",
      "[5.000e+01 0.000e+00 1.902e+03 4.000e+01 0.000e+00 1.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 1.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 1.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 1.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 1.000e+00 0.000e+00 0.000e+00 0.000e+00 1.000e+00\n",
      " 0.000e+00]\n",
      "['age', 'capital-loss', 'hours-per-week', 'workclass_ Local-gov', 'education_14.0', 'marital_ Married-civ-spouse', 'occupation_ Prof-specialty', 'relationship_ Wife', 'race_ White']\n",
      "[57.  0.  0. 48.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  1.  1.]\n",
      "['age', 'hours-per-week', 'workclass_ Federal-gov', 'education_13.0', 'marital_ Married-civ-spouse', 'occupation_ Tech-support', 'race_ White', 'sex_ Male']\n",
      "[6.000e+01 7.298e+03 0.000e+00 4.000e+01 0.000e+00 0.000e+00 0.000e+00\n",
      " 1.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 1.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 1.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 1.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 1.000e+00\n",
      " 1.000e+00]\n",
      "['age', 'capital-gain', 'hours-per-week', 'workclass_ Private', 'education_9.0', 'marital_ Married-civ-spouse', 'occupation_ Craft-repair', 'race_ White', 'sex_ Male']\n",
      "Target: 0.250000, Collat: -0.020236\n",
      "\n",
      "\n",
      "Pois Index: 1, Pois fraction: 1.000000 of identical training samples \n",
      "Cluster Index: 20, Count: 15\n",
      "[45.  0.  0. 40.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.\n",
      "  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.\n",
      "  0.  1.  0.]\n",
      "['age', 'hours-per-week', 'workclass_ Private', 'education_9.0', 'marital_ Never-married', 'occupation_ Handlers-cleaners', 'relationship_ Not-in-family', 'race_ White']\n",
      "[25.  0.  0. 30.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.\n",
      "  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.\n",
      "  0.  1.  1.]\n",
      "['age', 'hours-per-week', 'workclass_ Private', 'education_9.0', 'marital_ Never-married', 'occupation_ Farming-fishing', 'relationship_ Unmarried', 'race_ White', 'sex_ Male']\n",
      "[5.000e+01 0.000e+00 1.902e+03 4.000e+01 0.000e+00 1.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 1.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 1.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 1.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00 0.000e+00\n",
      " 0.000e+00 0.000e+00 1.000e+00 0.000e+00 0.000e+00 0.000e+00 1.000e+00\n",
      " 0.000e+00]\n",
      "['age', 'capital-loss', 'hours-per-week', 'workclass_ Local-gov', 'education_14.0', 'marital_ Married-civ-spouse', 'occupation_ Prof-specialty', 'relationship_ Wife', 'race_ White']\n",
      "Target: 0.244444, Collat: 0.000955\n",
      "\n",
      "\n",
      "Pois Index: 2, Pois fraction: 2.000000 of identical training samples \n",
      "Cluster Index: 2, Count: 14\n",
      "[37.  0.  0. 70.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.\n",
      "  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  1.  1.]\n",
      "['age', 'hours-per-week', 'workclass_ Self-emp-inc', 'education_14.0', 'marital_ Married-civ-spouse', 'occupation_ Exec-managerial', 'race_ White', 'sex_ Male']\n",
      "[38.  0.  0. 40.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.\n",
      "  0.  1.  1.]\n",
      "['age', 'hours-per-week', 'workclass_ Self-emp-not-inc', 'education_14.0', 'marital_ Never-married', 'occupation_ Prof-specialty', 'relationship_ Not-in-family', 'race_ White', 'sex_ Male']\n",
      "[30.  0.  0. 40.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  1.  1.]\n",
      "['age', 'hours-per-week', 'workclass_ Local-gov', 'education_13.0', 'marital_ Married-civ-spouse', 'occupation_ Protective-serv', 'race_ White', 'sex_ Male']\n",
      "Target: 0.666667, Collat: -0.042774\n"
     ]
    }
   ],
   "source": [
    "# # print target metric for ClusterMatch\n",
    "\n",
    "# for i in range(len(kd_targets)):\n",
    "#     for poi_ind in range(3):\n",
    "#         if poi_ind == 0:\n",
    "#             print(\"Index: %d, Poison ratio: % d, target: %f \" %(i, poi_ind, kd_targets[i, poi_ind, 0]))\n",
    "\n",
    "# sorting the error rate for ClusterMatch based on target metric\n",
    "\n",
    "for pois_ind in range(3): # for each pois_ind/pois frac\n",
    "    print(\"\\n\")\n",
    "    \n",
    "    # print the poison index and poison rate\n",
    "    print(\"Pois Index: %d, Pois fraction: %f of identical training samples \" %( pois_ind, pois_rates[pois_ind]))\n",
    "    \n",
    "    # sort the target metric scores for current poison index\n",
    "    sorted_kd_targets = np.argsort(kd_targets[:, pois_ind, 0])\n",
    "    \n",
    "    # take the last index of sorted_kd_targets as it is the index with highest target score for current poison index\n",
    "    top1_ind = sorted_kd_targets[-1]\n",
    "    \n",
    "#     print(\"kd error/used Index: \", top1_index) # print the top1_index\n",
    "\n",
    "    # get the cluster index, count for the feature with the highest target score using top1_index, \n",
    "    # the key is kd_target and kd_used have same indexing, so top1_index which is index in kd_targets is also the same index\n",
    "    # in kd_used, then kd_used has the index and count of the cluster to which the highest target score belongs to\n",
    "    cluster_ind, count = kd_used[top1_ind]\n",
    "    print(\"Cluster Index: %d, Count: %d\" % (cluster_ind, count))\n",
    "    \n",
    "    # get the index of training samples that belong to that cluster with highest target\n",
    "    train_examp_ind = np.where(train_km==cluster_ind)[0]    # has indexes of such samples\n",
    "    train_examp_rand_ind = np.random.choice(train_examp_ind.shape[0], 3, replace= False) # pick five random such samples\n",
    "    for i in train_examp_rand_ind:\n",
    "        example = train_x[i]        # print the examples in the cluster\n",
    "        print(example)\n",
    "        print([prot_col for v, prot_col in zip(example, all_cols) if v > 0.5]) # print the examples in cluster\n",
    "        \n",
    "    # print its target and collat values\n",
    "    print(\"Target: %f, Collat: %f\" % (kd_targets[top1_ind, pois_ind, 0], kd_targets[top1_ind, pois_ind, 1]))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 575,
   "id": "widespread-argentina",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}