EDA-OpenData.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1600, 400), (1600, 400), (1600, 400))"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_df = pd.read_csv('data/opendata-aggregated-means.csv', index_col=0, sep='\\t')\n",
    "std_df = pd.read_csv('data/opendata-aggregated-std.csv', index_col=0, sep='\\t')\n",
    "n_df = pd.read_csv('data/opendata-aggregated-n.csv', index_col=0, sep='\\t')\n",
    "mean_df.shape, std_df.shape, n_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>F1</th>\n",
       "      <th>F2</th>\n",
       "      <th>F3</th>\n",
       "      <th>F4</th>\n",
       "      <th>F5</th>\n",
       "      <th>F6</th>\n",
       "      <th>F7</th>\n",
       "      <th>F8</th>\n",
       "      <th>F9</th>\n",
       "      <th>F10</th>\n",
       "      <th>...</th>\n",
       "      <th>F391</th>\n",
       "      <th>F392</th>\n",
       "      <th>F393</th>\n",
       "      <th>F394</th>\n",
       "      <th>F395</th>\n",
       "      <th>F396</th>\n",
       "      <th>F397</th>\n",
       "      <th>F398</th>\n",
       "      <th>F399</th>\n",
       "      <th>F400</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>WE/1</th>\n",
       "      <td>36.4</td>\n",
       "      <td>98.8</td>\n",
       "      <td>72.3</td>\n",
       "      <td>47.3</td>\n",
       "      <td>24.2</td>\n",
       "      <td>29.3</td>\n",
       "      <td>75.4</td>\n",
       "      <td>64.7</td>\n",
       "      <td>82.1</td>\n",
       "      <td>57.2</td>\n",
       "      <td>...</td>\n",
       "      <td>93.1</td>\n",
       "      <td>51.4</td>\n",
       "      <td>60.4</td>\n",
       "      <td>74.0</td>\n",
       "      <td>83.9</td>\n",
       "      <td>78.7</td>\n",
       "      <td>70.9</td>\n",
       "      <td>33.2</td>\n",
       "      <td>31.3</td>\n",
       "      <td>31.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/2</th>\n",
       "      <td>57.4</td>\n",
       "      <td>89.2</td>\n",
       "      <td>64.7</td>\n",
       "      <td>13.8</td>\n",
       "      <td>19.2</td>\n",
       "      <td>45.0</td>\n",
       "      <td>67.5</td>\n",
       "      <td>38.9</td>\n",
       "      <td>80.4</td>\n",
       "      <td>54.2</td>\n",
       "      <td>...</td>\n",
       "      <td>83.5</td>\n",
       "      <td>45.6</td>\n",
       "      <td>73.9</td>\n",
       "      <td>65.0</td>\n",
       "      <td>93.3</td>\n",
       "      <td>81.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>28.1</td>\n",
       "      <td>41.7</td>\n",
       "      <td>27.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/3</th>\n",
       "      <td>26.8</td>\n",
       "      <td>57.4</td>\n",
       "      <td>9.1</td>\n",
       "      <td>92.1</td>\n",
       "      <td>27.1</td>\n",
       "      <td>66.8</td>\n",
       "      <td>44.0</td>\n",
       "      <td>74.3</td>\n",
       "      <td>34.0</td>\n",
       "      <td>12.3</td>\n",
       "      <td>...</td>\n",
       "      <td>47.4</td>\n",
       "      <td>61.4</td>\n",
       "      <td>71.4</td>\n",
       "      <td>44.9</td>\n",
       "      <td>76.8</td>\n",
       "      <td>31.4</td>\n",
       "      <td>13.3</td>\n",
       "      <td>84.4</td>\n",
       "      <td>12.7</td>\n",
       "      <td>72.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 400 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        F1    F2    F3    F4    F5    F6    F7    F8    F9   F10  ...  F391  \\\n",
       "WE/1  36.4  98.8  72.3  47.3  24.2  29.3  75.4  64.7  82.1  57.2  ...  93.1   \n",
       "WE/2  57.4  89.2  64.7  13.8  19.2  45.0  67.5  38.9  80.4  54.2  ...  83.5   \n",
       "WE/3  26.8  57.4   9.1  92.1  27.1  66.8  44.0  74.3  34.0  12.3  ...  47.4   \n",
       "\n",
       "      F392  F393  F394  F395  F396  F397  F398  F399  F400  \n",
       "WE/1  51.4  60.4  74.0  83.9  78.7  70.9  33.2  31.3  31.8  \n",
       "WE/2  45.6  73.9  65.0  93.3  81.0  66.0  28.1  41.7  27.5  \n",
       "WE/3  61.4  71.4  44.9  76.8  31.4  13.3  84.4  12.7  72.8  \n",
       "\n",
       "[3 rows x 400 columns]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>F1</th>\n",
       "      <th>F2</th>\n",
       "      <th>F3</th>\n",
       "      <th>F4</th>\n",
       "      <th>F5</th>\n",
       "      <th>F6</th>\n",
       "      <th>F7</th>\n",
       "      <th>F8</th>\n",
       "      <th>F9</th>\n",
       "      <th>F10</th>\n",
       "      <th>...</th>\n",
       "      <th>F391</th>\n",
       "      <th>F392</th>\n",
       "      <th>F393</th>\n",
       "      <th>F394</th>\n",
       "      <th>F395</th>\n",
       "      <th>F396</th>\n",
       "      <th>F397</th>\n",
       "      <th>F398</th>\n",
       "      <th>F399</th>\n",
       "      <th>F400</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>WE/1</th>\n",
       "      <td>4.920940</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>5.104690</td>\n",
       "      <td>4.965212</td>\n",
       "      <td>10.598962</td>\n",
       "      <td>9.797959</td>\n",
       "      <td>6.887089</td>\n",
       "      <td>7.145819</td>\n",
       "      <td>2.191542</td>\n",
       "      <td>10.530385</td>\n",
       "      <td>...</td>\n",
       "      <td>2.073070</td>\n",
       "      <td>8.124943</td>\n",
       "      <td>8.015418</td>\n",
       "      <td>9.471360</td>\n",
       "      <td>10.166667</td>\n",
       "      <td>7.823096</td>\n",
       "      <td>6.090529</td>\n",
       "      <td>5.154513</td>\n",
       "      <td>9.192388</td>\n",
       "      <td>9.033333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/2</th>\n",
       "      <td>6.374701</td>\n",
       "      <td>2.638560</td>\n",
       "      <td>10.087666</td>\n",
       "      <td>3.288047</td>\n",
       "      <td>3.437229</td>\n",
       "      <td>21.650635</td>\n",
       "      <td>5.969925</td>\n",
       "      <td>5.945082</td>\n",
       "      <td>3.394218</td>\n",
       "      <td>12.924473</td>\n",
       "      <td>...</td>\n",
       "      <td>4.944786</td>\n",
       "      <td>7.880831</td>\n",
       "      <td>10.960155</td>\n",
       "      <td>9.859421</td>\n",
       "      <td>3.995153</td>\n",
       "      <td>4.371914</td>\n",
       "      <td>5.800000</td>\n",
       "      <td>10.217693</td>\n",
       "      <td>6.016034</td>\n",
       "      <td>5.547809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/3</th>\n",
       "      <td>4.268627</td>\n",
       "      <td>18.246315</td>\n",
       "      <td>2.458803</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>9.584014</td>\n",
       "      <td>11.180340</td>\n",
       "      <td>12.624013</td>\n",
       "      <td>3.475000</td>\n",
       "      <td>9.466667</td>\n",
       "      <td>3.407078</td>\n",
       "      <td>...</td>\n",
       "      <td>7.566667</td>\n",
       "      <td>7.105732</td>\n",
       "      <td>5.154513</td>\n",
       "      <td>12.548421</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>7.723850</td>\n",
       "      <td>3.693593</td>\n",
       "      <td>2.877673</td>\n",
       "      <td>4.286607</td>\n",
       "      <td>5.028021</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 400 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            F1         F2         F3        F4         F5         F6  \\\n",
       "WE/1  4.920940   0.707107   5.104690  4.965212  10.598962   9.797959   \n",
       "WE/2  6.374701   2.638560  10.087666  3.288047   3.437229  21.650635   \n",
       "WE/3  4.268627  18.246315   2.458803  2.500000   9.584014  11.180340   \n",
       "\n",
       "             F7        F8        F9        F10  ...      F391      F392  \\\n",
       "WE/1   6.887089  7.145819  2.191542  10.530385  ...  2.073070  8.124943   \n",
       "WE/2   5.969925  5.945082  3.394218  12.924473  ...  4.944786  7.880831   \n",
       "WE/3  12.624013  3.475000  9.466667   3.407078  ...  7.566667  7.105732   \n",
       "\n",
       "           F393       F394       F395      F396      F397       F398  \\\n",
       "WE/1   8.015418   9.471360  10.166667  7.823096  6.090529   5.154513   \n",
       "WE/2  10.960155   9.859421   3.995153  4.371914  5.800000  10.217693   \n",
       "WE/3   5.154513  12.548421   7.000000  7.723850  3.693593   2.877673   \n",
       "\n",
       "          F399      F400  \n",
       "WE/1  9.192388  9.033333  \n",
       "WE/2  6.016034  5.547809  \n",
       "WE/3  4.286607  5.028021  \n",
       "\n",
       "[3 rows x 400 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stderr_df = std_df / np.sqrt(n_df)\n",
    "stderr_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "measures = {\n",
    "1  : [\"playful\",\"serious\"],\n",
    "2  : [\"shy\",\"bold\"],\n",
    "3  : [\"cheery\",\"sorrowful\"],\n",
    "4  : [\"masculine\",\"feminine\"],\n",
    "5  : [\"charming\",\"awkward\"],\n",
    "6  : [\"lewd\",\"tasteful\"],\n",
    "7  : [\"intellectual\",\"physical\"],\n",
    "8  : [\"strict\",\"lenient\"],\n",
    "9  : [\"refined\",\"rugged\"],\n",
    "10 : [\"trusting\",\"suspicious\"],\n",
    "11 : [\"innocent\",\"worldly\"],\n",
    "12 : [\"artistic\",\"scientific\"],\n",
    "13 : [\"stoic\",\"expressive\"],\n",
    "14 : [\"cunning\",\"honorable\"],\n",
    "15 : [\"orderly\",\"chaotic\"],\n",
    "16 : [\"normal\",\"weird\"],\n",
    "17 : [\"competitive\",\"cooperative\"],\n",
    "18 : [\"tense\",\"relaxed\"],\n",
    "19 : [\"brave\",\"careful\"],\n",
    "20 : [\"spiritual\",\"skeptical\"],\n",
    "21 : [\"unlucky\",\"fortunate\"],\n",
    "22 : [\"ferocious\",\"pacifist\"],\n",
    "23 : [\"modest\",\"flamboyant\"],\n",
    "24 : [\"dominant\",\"submissive\"],\n",
    "25 : [\"forgiving\",\"vengeful\"],\n",
    "26 : [\"wise\",\"foolish\"],\n",
    "27 : [\"impulsive\",\"cautious\"],\n",
    "28 : [\"loyal\",\"traitorous\"],\n",
    "29 : [\"creative\",\"conventional\"],\n",
    "30 : [\"curious\",\"apathetic\"],\n",
    "31 : [\"rude\",\"respectful\"],\n",
    "32 : [\"diligent\",\"lazy\"],\n",
    "33 : [\"lustful\",\"chaste\"],\n",
    "34 : [\"chatty\",\"reserved\"],\n",
    "35 : [\"emotional\",\"logical\"],\n",
    "36 : [\"moody\",\"stable\"],\n",
    "37 : [\"dunce\",\"genius\"],\n",
    "38 : [\"arrogant\",\"humble\"],\n",
    "39 : [\"heroic\",\"villainous\"],\n",
    "40 : [\"attractive\",\"repulsive\"],\n",
    "41 : [\"rational\",\"whimsical\"],\n",
    "42 : [\"mischievous\",\"well behaved\"],\n",
    "43 : [\"aloof\",\"obsessed\"],\n",
    "44 : [\"indulgent\",\"sober\"],\n",
    "45 : [\"kinky\",\"vanilla\"],\n",
    "46 : [\"straightforward\",\"cryptic\"],\n",
    "47 : [\"spontaneous\",\"deliberate\"],\n",
    "48 : [\"libertarian\",\"socialist\"],\n",
    "49 : [\"scheduled\",\"spontaneous\"],\n",
    "50 : [\"works hard\",\"plays hard\"],\n",
    "51 : [\"reasoned\",\"instinctual\"],\n",
    "52 : [\"focused on the present\",\"focused on the future\"],\n",
    "53 : [\"empirical\",\"theoretical\"],\n",
    "54 : [\"open\",\"guarded\"],\n",
    "55 : [\"methodical\",\"astonishing\"],\n",
    "56 : [\"mighty\",\"puny\"],\n",
    "57 : [\"bossy\",\"meek\"],\n",
    "58 : [\"barbaric\",\"civilized\"],\n",
    "59 : [\"gregarious\",\"private\"],\n",
    "60 : [\"quiet\",\"loud\"],\n",
    "61 : [\"political\",\"nonpolitical\"],\n",
    "62 : [\"confident\",\"insecure\"],\n",
    "63 : [\"democratic\",\"authoritarian\"],\n",
    "64 : [\"debased\",\"pure\"],\n",
    "65 : [\"fast\",\"slow\"],\n",
    "66 : [\"frugal\",\"lavish\"],\n",
    "67 : [\"ludicrous\",\"sensible\"],\n",
    "68 : [\"orange\",\"purple\"],\n",
    "69 : [\"tall\",\"short\"],\n",
    "70 : [\"young\",\"old\"],\n",
    "71 : [\"down2earth\",\"head@clouds\"],\n",
    "72 : [\"extrovert\",\"introvert\"],\n",
    "73 : [\"open to new experinces\",\"uncreative\"],\n",
    "74 : [\"calm\",\"anxious\"],\n",
    "75 : [\"disorganized\",\"self-disciplined\"],\n",
    "76 : [\"quarrelsome\",\"warm\"],\n",
    "77 : [\"nerd\",\"jock\"],\n",
    "78 : [\"lowbrow\",\"highbrow\"],\n",
    "79 : [\"selfish\",\"altruistic\"],\n",
    "80 : [\"autistic\",\"neurotypical\"],\n",
    "81 : [\"angelic\",\"demonic\"],\n",
    "82 : [\"hesitant\",\"decisive\"],\n",
    "83 : [\"devout\",\"heathen\"],\n",
    "84 : [\"cruel\",\"kind\"],\n",
    "85 : [\"direct\",\"roundabout\"],\n",
    "86 : [\"mathematical\",\"literary\"],\n",
    "87 : [\"blue-collar\",\"ivory-tower\"],\n",
    "88 : [\"slovenly\",\"stylish\"],\n",
    "89 : [\"playful\",\"shy\"],\n",
    "90 : [\"serious\",\"bold\"],\n",
    "91 : [\"charming\",\"trusting\"],\n",
    "92 : [\"awkward\",\"suspicious\"],\n",
    "93 : [\"hipster\",\"basic\"],\n",
    "94 : [\"coordinated\",\"clumsy\"],\n",
    "95 : [\"funny\",\"humorless\"],\n",
    "96 : [\"politically correct\",\"edgy\"],\n",
    "97 : [\"rich\",\"poor\"],\n",
    "98 : [\"hard\",\"soft\"],\n",
    "99 : [\"remote\",\"involved\"],\n",
    "100 : [\"metaphorical\",\"literal\"],\n",
    "101 : [\"biased\",\"impartial\"],\n",
    "102 : [\"mundane\",\"extraordinary\"],\n",
    "103 : [\"tirewsome\",\"interesting\"],\n",
    "104 : [\"smooth\",\"rough\"],\n",
    "105 : [\"spicy\",\"mild\"],\n",
    "106 : [\"enslaved\",\"emancipated\"],\n",
    "107 : [\"optimistic\",\"pessimistic\"],\n",
    "108 : [\"sickly\",\"healthy\"],\n",
    "109 : [\"luddite\",\"technophile\"],\n",
    "110 : [\"vain\",\"demure\"],\n",
    "111 : [\"high-tech\",\"low-tech\"],\n",
    "112 : [\"flexible\",\"rigid\"],\n",
    "113 : [\"cosmopolitan\",\"provincial\"],\n",
    "114 : [\"arcane\",\"mainstream\"],\n",
    "115 : [\"outlaw\",\"sheriff\"],\n",
    "116 : [\"pronatalist\",\"child free\"],\n",
    "117 : [\"sad\",\"happy\"],\n",
    "118 : [\"jealous\",\"compersive\"],\n",
    "119 : [\"bitter\",\"sweet\"],\n",
    "120 : [\"resigned\",\"resistant\"],\n",
    "121 : [\"sarcastic\",\"genuine\"],\n",
    "122 : [\"human\",\"animalistic\"],\n",
    "123 : [\"sporty\",\"bookish\"],\n",
    "124 : [\"moderate\",\"extreme\"],\n",
    "125 : [\"angry\",\"good-humored\"],\n",
    "126 : [\"depressed\",\"bright\"],\n",
    "127 : [\"self-conscious\",\"self-assured\"],\n",
    "128 : [\"vulnerable\",\"armoured\"],\n",
    "129 : [\"warm\",\"cold\"],\n",
    "130 : [\"assertive\",\"passive\"],\n",
    "131 : [\"active\",\"slothful\"],\n",
    "132 : [\"imaginative\",\"practical\"],\n",
    "133 : [\"adventurous\",\"stick-in-the-mud\"],\n",
    "134 : [\"obedient\",\"rebellious\"],\n",
    "135 : [\"competent\",\"incompetent\"],\n",
    "136 : [\"unambitious\",\"driven\"],\n",
    "137 : [\"simple\",\"complicated\"],\n",
    "138 : [\"proletariat\",\"bourgeoisie\"],\n",
    "139 : [\"alpha\",\"beta\"],\n",
    "140 : [\"'right-brained'\",\"'left-brained'\"],\n",
    "141 : [\"thick-skinned\",\"sensitive\"],\n",
    "142 : [\"charismatic\",\"uninspiring\"],\n",
    "143 : [\"feisty\",\"gracious\"],\n",
    "144 : [\"eloquent\",\"unpolished\"],\n",
    "145 : [\"high IQ\",\"low IQ\"],\n",
    "146 : [\"insider\",\"outsider\"],\n",
    "147 : [\"morning lark\",\"night owl\"],\n",
    "148 : [\"thin\",\"thick\"],\n",
    "149 : [\"sheeple\",\"conspiracist\"],\n",
    "150 : [\"neat\",\"messy\"],\n",
    "151 : [\"vague\",\"precise\"],\n",
    "152 : [\"philosophical\",\"real\"],\n",
    "153 : [\"modern\",\"historical\"],\n",
    "154 : [\"judgemental\",\"accepting\"],\n",
    "155 : [\"average\",\"deviant\"],\n",
    "156 : [\"gossiping\",\"confidential\"],\n",
    "157 : [\"official\",\"backdoor\"],\n",
    "158 : [\"scholarly\",\"crafty\"],\n",
    "159 : [\"leisurely\",\"hurried\"],\n",
    "160 : [\"explorer\",\"builder\"],\n",
    "161 : [\"captain\",\"first-mate\"],\n",
    "162 : [\"mysterious\",\"unambiguous\"],\n",
    "163 : [\"independent\",\"codependent\"],\n",
    "164 : [\"family-first\",\"work-first\"],\n",
    "165 : [\"scruffy\",\"manicured\"],\n",
    "166 : [\"wild\",\"tame\"],\n",
    "167 : [\"prestigious\",\"disreputable\"],\n",
    "168 : [\"scandalous\",\"proper\"],\n",
    "169 : [\"unprepared\",\"hoarder\"],\n",
    "170 : [\"sheltered\",\"street-smart\"],\n",
    "171 : [\"open-minded\",\"close-minded\"],\n",
    "172 : [\"permanent\",\"transient\"],\n",
    "173 : [\"dramatic\",\"no-nonsense\"],\n",
    "174 : [\"apprentice\",\"master\"],\n",
    "175 : [\"straight\",\"queer\"],\n",
    "176 : [\"androgynous\",\"gendered\"],\n",
    "177 : [\"repetitive\",\"varied\"],\n",
    "178 : [\"patient\",\"impatient\"],\n",
    "179 : [\"poisonous\",\"nurturing\"],\n",
    "180 : [\"creepy\",\"disarming\"],\n",
    "181 : [\"inspiring\",\"cringeworthy\"],\n",
    "182 : [\"soulless\",\"soulful\"],\n",
    "183 : [\"hard\",\"soft\"],\n",
    "184 : [\"beautiful\",\"ugly\"],\n",
    "185 : [\"domestic\",\"industrial\"],\n",
    "186 : [\"juvenile\",\"mature\"],\n",
    "187 : [\"idealist\",\"realist\"],\n",
    "188 : [\"nihilist\",\"existentialist\"],\n",
    "189 : [\"objective\",\"subjective\"],\n",
    "190 : [\"theist\",\"atheist\"],\n",
    "191 : [\"classical\",\"avant-garde\"],\n",
    "192 : [\"utilitarian\",\"decorative\"],\n",
    "193 : [\"generalist\",\"specialist\"],\n",
    "194 : [\"multicolored\",\"monochrome\"],\n",
    "195 : [\"complimentary\",\"insulting\"],\n",
    "196 : [\"individualist\",\"communal\"],\n",
    "197 : [\"equitable\",\"hypocritical\"],\n",
    "198 : [\"traditional\",\"unorthodox\"],\n",
    "199 : [\"workaholic\",\"slacker\"],\n",
    "200 : [\"resourceful\",\"helpless\"],\n",
    "201 : [\"crazy\",\"sane\"],\n",
    "202 : [\"anarchist\",\"statist\"],\n",
    "203 : [\"cool\",\"dorky\"],\n",
    "204 : [\"important\",\"irrelevant\"],\n",
    "205 : [\"noob\",\"pro\"],\n",
    "206 : [\"deranged\",\"reasonable\"],\n",
    "207 : [\"rural\",\"urban\"],\n",
    "208 : [\"introspective\",\"not introspective\"],\n",
    "209 : [\"city-slicker\",\"country-bumpkin\"],\n",
    "210 : [\"western\",\"eastern\"],\n",
    "211 : [\"mad\",\"glad\"],\n",
    "212 : [\"social\",\"reclusive\"],\n",
    "213 : [\"studious\",\"goof-off\"],\n",
    "214 : [\"slugabed\",\"go-getter\"],\n",
    "215 : [\"penny-pincher\",\"overspender\"],\n",
    "216 : [\"liberal\",\"conservative\"],\n",
    "217 : [\"unassuming\",\"pretentious\"],\n",
    "218 : [\"persistent\",\"quitter\"],\n",
    "219 : [\"hedonist\",\"monastic\"],\n",
    "220 : [\"patriotic\",\"unpatriotic\"],\n",
    "221 : [\"tactful\",\"indiscreet\"],\n",
    "222 : [\"wholesome\",\"salacious\"],\n",
    "223 : [\"joyful\",\"miserable\"],\n",
    "224 : [\"zany\",\"regular\"],\n",
    "225 : [\"alert\",\"oblivious\"],\n",
    "226 : [\"feminist\",\"sexist\"],\n",
    "227 : [\"racist\",\"egalitarian\"],\n",
    "228 : [\"abstract\",\"concrete\"],\n",
    "229 : [\"formal\",\"intimate\"],\n",
    "230 : [\"resolute\",\"wavering\"],\n",
    "231 : [\"deep\",\"shallow\"],\n",
    "232 : [\"valedictorian\",\"drop out\"],\n",
    "233 : [\"minimalist\",\"pack rat\"],\n",
    "234 : [\"trash\",\"treasure\"],\n",
    "\n",
    "235 : [\"&#129392;\",\"&#128579;\"],\n",
    "236 : [\"&#129396;\",\"&#129395;\"],\n",
    "237 : [\"&#128526;\",\"&#129488;\"],\n",
    "238 : [\"&#128557;\",\"&#128512;\"],\n",
    "239 : [\"&#129297;\",\"&#129312;\"],\n",
    "240 : [\"&#128125;\",\"&#129313;\"],\n",
    "241 : [\"&#128148;\",\"&#128157;\"],\n",
    "242 : [\"&#129302;\",\"&#128123;\"],\n",
    "243 : [\"&#128169;\",\"&#127775;\"],\n",
    "244 : [\"&#128170;\",\"&#129504;\"],\n",
    "245 : [\"&#128587;&zwj;&#9794;&#65039;\",\"&#128581;&zwj;&#9794;&#65039;\"],\n",
    "246 : [\"&#128104;&zwj;&#9877;&#65039;\",\"&#128104;&zwj;&#128295;\"],\n",
    "247 : [\"&#128105;&zwj;&#128300;\",\"&#128105;&zwj;&#127908;\"],\n",
    "248 : [\"&#128520;\",\"&#128519;\"],\n",
    "249 : [\"&#129300;\",\"&#129323;\"],\n",
    "250 : [\"&#128024;\",\"&#128000;\"],\n",
    "251 : [\"&#128046;\",\"&#128055;\"],\n",
    "252 : [\"&#128052;\",\"&#129412;\"],\n",
    "253 : [\"&#128041;\",\"&#128018;\"],\n",
    "254 : [\"&#128692;\",\"&#127947;&#65039;&zwj;&#9794;&#65039;\"],\n",
    "255 : [\"&#129338;\",\"&#127948;\"],\n",
    "256 : [\"&#128131;\",\"&#129493;\"],\n",
    "257 : [\"&#129497;\",\"&#128104;&zwj;&#128640;\"],\n",
    "258 : [\"&#128016;\",\"&#129426;\"],\n",
    "259 : [\"&#129415;\",\"&#128063;\"],\n",
    "260 : [\"&#128556;\",\"&#128527;\"],\n",
    "261 : [\"&#129296;\",\"&#128540;\"],\n",
    "262 : [\"&#129315;\",\"&#128522;\"],\n",
    "263 : [\"&#129495;\",\"&#128716;\"],\n",
    "264 : [\"&#129406;\",\"&#128095;\"],\n",
    "265 : [\"&#127913;\",\"&#129506;\"],\n",
    "266 : [\"&#128200;\",\"&#128201;\"],\n",
    "267 : [\"stinky\",\"fresh\"],\n",
    "268 : [\"legit\",\"scrub\"],\n",
    "\n",
    "269 : [\"self-destructive\",\"self-improving\"],\n",
    "270 : [\"French\",\"Russian\"],\n",
    "271 : [\"German\",\"English\"],\n",
    "272 : [\"Italian\",\"Swedish\"],\n",
    "273 : [\"Greek\",\"Roman\"],\n",
    "274 : [\"traumatized\",\"flourishing\"],\n",
    "275 : [\"sturdy\",\"flimsy\"],\n",
    "276 : [\"macho\",\"metrosexual\"],\n",
    "277 : [\"claustrophobic\",\"spelunker\"],\n",
    "278 : [\"offended\",\"chill\"],\n",
    "279 : [\"rhythmic\",\"stuttering\"],\n",
    "280 : [\"musical\",\"off-key\"],\n",
    "281 : [\"lost\",\"enlightened\"],\n",
    "282 : [\"masochistic\",\"pain-avoidant\"],\n",
    "283 : [\"efficient\",\"overprepared\"],\n",
    "284 : [\"oppressed\",\"privileged\"],\n",
    "285 : [\"sunny\",\"gloomy\"],\n",
    "286 : [\"vegan\",\"cannibal\"],\n",
    "287 : [\"loveable\",\"punchable\"],\n",
    "288 : [\"slow-talking\",\"fast-talking\"],\n",
    "289 : [\"believable\",\"poorly-written\"],\n",
    "290 : [\"vibrant\",\"geriatric\"],\n",
    "291 : [\"consistent\",\"variable\"],\n",
    "292 : [\"dispassionate\",\"romantic\"],\n",
    "293 : [\"linear\",\"circular\"],\n",
    "294 : [\"intense\",\"lighthearted\"],\n",
    "295 : [\"knowledgeable\",\"ignorant\"],\n",
    "296 : [\"fixable\",\"unfixable\"],\n",
    "297 : [\"exuberant\",\"subdued\"],\n",
    "298 : [\"secretive\",\"open-book\"],\n",
    "299 : [\"perceptive\",\"unobservant\"],\n",
    "300 : [\"folksy\",\"presidential\"],\n",
    "301 : [\"corporate\",\"freelance\"],\n",
    "302 : [\"sleepy\",\"frenzied\"],\n",
    "303 : [\"loose\",\"tight\"],\n",
    "304 : [\"narcissistic\",\"low self esteem\"],\n",
    "305 : [\"poetic\",\"factual\"],\n",
    "306 : [\"melee\",\"ranged\"],\n",
    "307 : [\"giggling\",\"chortling\"],\n",
    "308 : [\"whippersnapper\",\"sage\"],\n",
    "309 : [\"tailor\",\"blacksmith\"],\n",
    "310 : [\"hunter\",\"gatherer\"],\n",
    "311 : [\"experimental\",\"reliable\"],\n",
    "312 : [\"moist\",\"dry\"],\n",
    "313 : [\"trolling\",\"triggered\"],\n",
    "314 : [\"tattle-tale\",\"f***-the-police\"],\n",
    "315 : [\"punk rock\",\"preppy\"],\n",
    "\n",
    "316 : [\"realistic\",\"fantastical\"],\n",
    "317 : [\"trendy\",\"vintage\"],\n",
    "318 : [\"factual\",\"exaggerating\"],\n",
    "319 : [\"good-cook\",\"bad-cook\"],\n",
    "320 : [\"comedic\",\"dramatic\"],\n",
    "321 : [\"OCD\",\"ADHD\"],\n",
    "322 : [\"interrupting\",\"attentive\"],\n",
    "323 : [\"exhibitionist\",\"bashful\"],\n",
    "324 : [\"badass\",\"weakass\"],\n",
    "325 : [\"gamer\",\"non-gamer\"],\n",
    "326 : [\"random\",\"pointed\"],\n",
    "327 : [\"epic\",\"deep\"],\n",
    "328 : [\"serene\",\"pensive\"],\n",
    "329 : [\"bored\",\"interested\"],\n",
    "330 : [\"envious\",\"prideful\"],\n",
    "331 : [\"ironic\",\"profound\"],\n",
    "332 : [\"sexual\",\"asexual\"],\n",
    "333 : [\"&#129397;\",\"&#129398;\"],\n",
    "334 : [\"&#127875;\",\"&#128128;\"],\n",
    "335 : [\"&#127936;\",\"&#127912;\"],\n",
    "336 : [\"clean\",\"perverted\"],\n",
    "337 : [\"empath\",\"psychopath\"],\n",
    "338 : [\"haunted\",\"blissful\"],\n",
    "339 : [\"entitled\",\"grateful\"],\n",
    "340 : [\"ambitious\",\"realistic\"],\n",
    "341 : [\"stuck-in-the-past\",\"forward-thinking\"],\n",
    "342 : [\"fire\",\"water\"],\n",
    "343 : [\"earth\",\"air\"],\n",
    "344 : [\"lover\",\"fighter\"],\n",
    "345 : [\"overachiever\",\"underachiever\"],\n",
    "346 : [\"Coke\",\"Pepsi\"],\n",
    "347 : [\"twitchy\",\"still\"],\n",
    "348 : [\"freak\",\"normie\"],\n",
    "349 : [\"thinker\",\"doer\"],\n",
    "350 : [\"hard-work\",\"natural-talent\"],\n",
    "351 : [\"stingy\",\"generous\"],\n",
    "352 : [\"stubborn\",\"accommodating\"],\n",
    "353 : [\"extravagant\",\"thrifty\"],\n",
    "354 : [\"demanding\",\"unchallenging\"],\n",
    "355 : [\"two-faced\",\"one-faced\"],\n",
    "356 : [\"plastic\",\"wooden\"],\n",
    "357 : [\"neutral\",\"opinionated\"],\n",
    "358 : [\"chivalrous\",\"businesslike\"],\n",
    "359 : [\"high standards\",\"desperate\"],\n",
    "360 : [\"on-time\",\"tardy\"],\n",
    "\n",
    "361 : [\"everyman\",\"chosen one\"],\n",
    "362 : [\"jealous\",\"opinionated\"],\n",
    "363 : [\"protagonist\",\"antagonist\"],\n",
    "364 : [\"devoted\",\"unfaithful\"],\n",
    "365 : [\"fearmongering\",\"reassuring\"],\n",
    "366 : [\"common sense\",\"analysis\"],\n",
    "367 : [\"unemotional\",\"emotional\"],\n",
    "368 : [\"rap\",\"rock\"],\n",
    "369 : [\"genocidal\",\"not genocidal\"],\n",
    "370 : [\"cat person\",\"dog person\"],\n",
    "371 : [\"indie\",\"pop\"],\n",
    "372 : [\"cultured\",\"rustic\"],\n",
    "373 : [\"tautology\",\"oxymoron\"],\n",
    "374 : [\"bad boy\",\"white knight\"],\n",
    "375 : [\"princess\",\"queen\"],\n",
    "376 : [\"hypochondriac\",\"stoic\"],\n",
    "377 : [\"yes-man\",\"contrarian\"],\n",
    "378 : [\"giving\",\"receiving\"],\n",
    "379 : [\"chic\",\"cheesy\"],\n",
    "380 : [\"celebrity\",\"boy/girl-next-door\"],\n",
    "\n",
    "381 : [\"goth\",\"flower child\"],\n",
    "382 : [\"summer\",\"winter\"],\n",
    "383 : [\"frank\",\"sugarcoated\"],\n",
    "384 : [\"naive\",\"paranoid\"],\n",
    "385 : [\"gullible\",\"cynical\"],\n",
    "386 : [\"motivated\",\"unmotivated\"],\n",
    "387 : [\"radical\",\"centrist\"],\n",
    "388 : [\"monotone\",\"expressive\"],\n",
    "389 : [\"love-focused\",\"money-focused\"],\n",
    "390 : [\"transparent\",\"machiavellian\"],\n",
    "391 : [\"timid\",\"cocky\"],\n",
    "392 : [\"concise\",\"long-winded\"],\n",
    "393 : [\"picky\",\"always down\"],\n",
    "394 : [\"proactive\",\"reactive\"],\n",
    "395 : [\"prudish\",\"flirtatious\"],\n",
    "396 : [\"innocent\",\"jaded\"],\n",
    "397 : [\"touchy-feely\",\"distant\"],\n",
    "398 : [\"muddy\",\"washed\"],\n",
    "399 : [\"quirky\",\"predictable\"],\n",
    "400 : [\"never cries\",\"often crying\"],\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(measures)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_columns = [','.join(measures[int(column.replace('F', ''))]) for column in mean_df.columns ]\n",
    "len(label_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>playful,serious</th>\n",
       "      <th>shy,bold</th>\n",
       "      <th>cheery,sorrowful</th>\n",
       "      <th>masculine,feminine</th>\n",
       "      <th>charming,awkward</th>\n",
       "      <th>lewd,tasteful</th>\n",
       "      <th>intellectual,physical</th>\n",
       "      <th>strict,lenient</th>\n",
       "      <th>refined,rugged</th>\n",
       "      <th>trusting,suspicious</th>\n",
       "      <th>...</th>\n",
       "      <th>timid,cocky</th>\n",
       "      <th>concise,long-winded</th>\n",
       "      <th>picky,always down</th>\n",
       "      <th>proactive,reactive</th>\n",
       "      <th>prudish,flirtatious</th>\n",
       "      <th>innocent,jaded</th>\n",
       "      <th>touchy-feely,distant</th>\n",
       "      <th>muddy,washed</th>\n",
       "      <th>quirky,predictable</th>\n",
       "      <th>never cries,often crying</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>WE/1</th>\n",
       "      <td>36.4</td>\n",
       "      <td>98.8</td>\n",
       "      <td>72.3</td>\n",
       "      <td>47.3</td>\n",
       "      <td>24.2</td>\n",
       "      <td>29.3</td>\n",
       "      <td>75.4</td>\n",
       "      <td>64.7</td>\n",
       "      <td>82.1</td>\n",
       "      <td>57.2</td>\n",
       "      <td>...</td>\n",
       "      <td>93.1</td>\n",
       "      <td>51.4</td>\n",
       "      <td>60.4</td>\n",
       "      <td>74.0</td>\n",
       "      <td>83.9</td>\n",
       "      <td>78.7</td>\n",
       "      <td>70.9</td>\n",
       "      <td>33.2</td>\n",
       "      <td>31.3</td>\n",
       "      <td>31.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/2</th>\n",
       "      <td>57.4</td>\n",
       "      <td>89.2</td>\n",
       "      <td>64.7</td>\n",
       "      <td>13.8</td>\n",
       "      <td>19.2</td>\n",
       "      <td>45.0</td>\n",
       "      <td>67.5</td>\n",
       "      <td>38.9</td>\n",
       "      <td>80.4</td>\n",
       "      <td>54.2</td>\n",
       "      <td>...</td>\n",
       "      <td>83.5</td>\n",
       "      <td>45.6</td>\n",
       "      <td>73.9</td>\n",
       "      <td>65.0</td>\n",
       "      <td>93.3</td>\n",
       "      <td>81.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>28.1</td>\n",
       "      <td>41.7</td>\n",
       "      <td>27.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/3</th>\n",
       "      <td>26.8</td>\n",
       "      <td>57.4</td>\n",
       "      <td>9.1</td>\n",
       "      <td>92.1</td>\n",
       "      <td>27.1</td>\n",
       "      <td>66.8</td>\n",
       "      <td>44.0</td>\n",
       "      <td>74.3</td>\n",
       "      <td>34.0</td>\n",
       "      <td>12.3</td>\n",
       "      <td>...</td>\n",
       "      <td>47.4</td>\n",
       "      <td>61.4</td>\n",
       "      <td>71.4</td>\n",
       "      <td>44.9</td>\n",
       "      <td>76.8</td>\n",
       "      <td>31.4</td>\n",
       "      <td>13.3</td>\n",
       "      <td>84.4</td>\n",
       "      <td>12.7</td>\n",
       "      <td>72.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/4</th>\n",
       "      <td>56.9</td>\n",
       "      <td>74.4</td>\n",
       "      <td>28.2</td>\n",
       "      <td>58.4</td>\n",
       "      <td>27.7</td>\n",
       "      <td>76.2</td>\n",
       "      <td>30.6</td>\n",
       "      <td>58.3</td>\n",
       "      <td>33.7</td>\n",
       "      <td>35.7</td>\n",
       "      <td>...</td>\n",
       "      <td>69.6</td>\n",
       "      <td>42.9</td>\n",
       "      <td>65.4</td>\n",
       "      <td>29.8</td>\n",
       "      <td>82.3</td>\n",
       "      <td>49.3</td>\n",
       "      <td>25.2</td>\n",
       "      <td>72.2</td>\n",
       "      <td>55.8</td>\n",
       "      <td>35.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/5</th>\n",
       "      <td>23.6</td>\n",
       "      <td>36.0</td>\n",
       "      <td>29.4</td>\n",
       "      <td>60.6</td>\n",
       "      <td>85.8</td>\n",
       "      <td>58.8</td>\n",
       "      <td>6.8</td>\n",
       "      <td>68.0</td>\n",
       "      <td>37.4</td>\n",
       "      <td>13.3</td>\n",
       "      <td>...</td>\n",
       "      <td>20.4</td>\n",
       "      <td>68.7</td>\n",
       "      <td>78.3</td>\n",
       "      <td>56.3</td>\n",
       "      <td>36.9</td>\n",
       "      <td>34.9</td>\n",
       "      <td>30.3</td>\n",
       "      <td>52.4</td>\n",
       "      <td>39.6</td>\n",
       "      <td>75.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 400 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      playful,serious  shy,bold  cheery,sorrowful  masculine,feminine  \\\n",
       "WE/1             36.4      98.8              72.3                47.3   \n",
       "WE/2             57.4      89.2              64.7                13.8   \n",
       "WE/3             26.8      57.4               9.1                92.1   \n",
       "WE/4             56.9      74.4              28.2                58.4   \n",
       "WE/5             23.6      36.0              29.4                60.6   \n",
       "\n",
       "      charming,awkward  lewd,tasteful  intellectual,physical  strict,lenient  \\\n",
       "WE/1              24.2           29.3                   75.4            64.7   \n",
       "WE/2              19.2           45.0                   67.5            38.9   \n",
       "WE/3              27.1           66.8                   44.0            74.3   \n",
       "WE/4              27.7           76.2                   30.6            58.3   \n",
       "WE/5              85.8           58.8                    6.8            68.0   \n",
       "\n",
       "      refined,rugged  trusting,suspicious  ...  timid,cocky  \\\n",
       "WE/1            82.1                 57.2  ...         93.1   \n",
       "WE/2            80.4                 54.2  ...         83.5   \n",
       "WE/3            34.0                 12.3  ...         47.4   \n",
       "WE/4            33.7                 35.7  ...         69.6   \n",
       "WE/5            37.4                 13.3  ...         20.4   \n",
       "\n",
       "      concise,long-winded  picky,always down  proactive,reactive  \\\n",
       "WE/1                 51.4               60.4                74.0   \n",
       "WE/2                 45.6               73.9                65.0   \n",
       "WE/3                 61.4               71.4                44.9   \n",
       "WE/4                 42.9               65.4                29.8   \n",
       "WE/5                 68.7               78.3                56.3   \n",
       "\n",
       "      prudish,flirtatious  innocent,jaded  touchy-feely,distant  muddy,washed  \\\n",
       "WE/1                 83.9            78.7                  70.9          33.2   \n",
       "WE/2                 93.3            81.0                  66.0          28.1   \n",
       "WE/3                 76.8            31.4                  13.3          84.4   \n",
       "WE/4                 82.3            49.3                  25.2          72.2   \n",
       "WE/5                 36.9            34.9                  30.3          52.4   \n",
       "\n",
       "      quirky,predictable  never cries,often crying  \n",
       "WE/1                31.3                      31.8  \n",
       "WE/2                41.7                      27.5  \n",
       "WE/3                12.7                      72.8  \n",
       "WE/4                55.8                      35.9  \n",
       "WE/5                39.6                      75.1  \n",
       "\n",
       "[5 rows x 400 columns]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labeled_df = pd.DataFrame(mean_df)\n",
    "labeled_df.columns = label_columns\n",
    "labeled_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>5oclock shadow.label</th>\n",
       "      <th>5oclock shadow.confidence</th>\n",
       "      <th>age.label</th>\n",
       "      <th>age.confidence</th>\n",
       "      <th>arched eyebrows.label</th>\n",
       "      <th>arched eyebrows.confidence</th>\n",
       "      <th>attractive.label</th>\n",
       "      <th>attractive.confidence</th>\n",
       "      <th>bags under eyes.label</th>\n",
       "      <th>bags under eyes.confidence</th>\n",
       "      <th>...</th>\n",
       "      <th>wearing necklace.label</th>\n",
       "      <th>wearing necklace.confidence</th>\n",
       "      <th>wearing necktie.label</th>\n",
       "      <th>wearing necktie.confidence</th>\n",
       "      <th>yaw.label</th>\n",
       "      <th>yaw.confidence</th>\n",
       "      <th>young.label</th>\n",
       "      <th>young.confidence</th>\n",
       "      <th>gender.manual</th>\n",
       "      <th>race.manual</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A1.jpg</th>\n",
       "      <td>yes</td>\n",
       "      <td>0.13</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.6</td>\n",
       "      <td>no</td>\n",
       "      <td>0.85</td>\n",
       "      <td>no</td>\n",
       "      <td>1.00</td>\n",
       "      <td>no</td>\n",
       "      <td>0.51</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>0.56</td>\n",
       "      <td>no</td>\n",
       "      <td>0.36</td>\n",
       "      <td>18.79</td>\n",
       "      <td>1.0</td>\n",
       "      <td>no</td>\n",
       "      <td>0.33</td>\n",
       "      <td>male</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A2.jpg</th>\n",
       "      <td>no</td>\n",
       "      <td>0.88</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.6</td>\n",
       "      <td>no</td>\n",
       "      <td>0.90</td>\n",
       "      <td>no</td>\n",
       "      <td>0.22</td>\n",
       "      <td>no</td>\n",
       "      <td>1.00</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>0.53</td>\n",
       "      <td>no</td>\n",
       "      <td>0.71</td>\n",
       "      <td>-0.62</td>\n",
       "      <td>1.0</td>\n",
       "      <td>yes</td>\n",
       "      <td>0.60</td>\n",
       "      <td>female</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A3.jpg</th>\n",
       "      <td>no</td>\n",
       "      <td>0.45</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.6</td>\n",
       "      <td>no</td>\n",
       "      <td>0.66</td>\n",
       "      <td>no</td>\n",
       "      <td>0.96</td>\n",
       "      <td>no</td>\n",
       "      <td>0.88</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>0.52</td>\n",
       "      <td>no</td>\n",
       "      <td>0.66</td>\n",
       "      <td>-14.97</td>\n",
       "      <td>1.0</td>\n",
       "      <td>yes</td>\n",
       "      <td>0.07</td>\n",
       "      <td>female</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A4.jpg</th>\n",
       "      <td>no</td>\n",
       "      <td>0.21</td>\n",
       "      <td>39.0</td>\n",
       "      <td>0.6</td>\n",
       "      <td>no</td>\n",
       "      <td>0.90</td>\n",
       "      <td>no</td>\n",
       "      <td>0.71</td>\n",
       "      <td>no</td>\n",
       "      <td>0.35</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>1.00</td>\n",
       "      <td>yes</td>\n",
       "      <td>0.75</td>\n",
       "      <td>38.22</td>\n",
       "      <td>1.0</td>\n",
       "      <td>no</td>\n",
       "      <td>0.33</td>\n",
       "      <td>male</td>\n",
       "      <td>white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A6.jpg</th>\n",
       "      <td>yes</td>\n",
       "      <td>0.04</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.6</td>\n",
       "      <td>no</td>\n",
       "      <td>0.50</td>\n",
       "      <td>no</td>\n",
       "      <td>1.00</td>\n",
       "      <td>no</td>\n",
       "      <td>0.17</td>\n",
       "      <td>...</td>\n",
       "      <td>no</td>\n",
       "      <td>0.60</td>\n",
       "      <td>no</td>\n",
       "      <td>0.17</td>\n",
       "      <td>2.08</td>\n",
       "      <td>1.0</td>\n",
       "      <td>no</td>\n",
       "      <td>0.14</td>\n",
       "      <td>male</td>\n",
       "      <td>black</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 90 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       5oclock shadow.label  5oclock shadow.confidence  age.label  \\\n",
       "A1.jpg                  yes                       0.13       46.0   \n",
       "A2.jpg                   no                       0.88       13.0   \n",
       "A3.jpg                   no                       0.45       13.0   \n",
       "A4.jpg                   no                       0.21       39.0   \n",
       "A6.jpg                  yes                       0.04       41.0   \n",
       "\n",
       "        age.confidence arched eyebrows.label  arched eyebrows.confidence  \\\n",
       "A1.jpg             0.6                    no                        0.85   \n",
       "A2.jpg             0.6                    no                        0.90   \n",
       "A3.jpg             0.6                    no                        0.66   \n",
       "A4.jpg             0.6                    no                        0.90   \n",
       "A6.jpg             0.6                    no                        0.50   \n",
       "\n",
       "       attractive.label  attractive.confidence bags under eyes.label  \\\n",
       "A1.jpg               no                   1.00                    no   \n",
       "A2.jpg               no                   0.22                    no   \n",
       "A3.jpg               no                   0.96                    no   \n",
       "A4.jpg               no                   0.71                    no   \n",
       "A6.jpg               no                   1.00                    no   \n",
       "\n",
       "        bags under eyes.confidence  ... wearing necklace.label  \\\n",
       "A1.jpg                        0.51  ...                     no   \n",
       "A2.jpg                        1.00  ...                     no   \n",
       "A3.jpg                        0.88  ...                     no   \n",
       "A4.jpg                        0.35  ...                     no   \n",
       "A6.jpg                        0.17  ...                     no   \n",
       "\n",
       "        wearing necklace.confidence wearing necktie.label  \\\n",
       "A1.jpg                         0.56                    no   \n",
       "A2.jpg                         0.53                    no   \n",
       "A3.jpg                         0.52                    no   \n",
       "A4.jpg                         1.00                   yes   \n",
       "A6.jpg                         0.60                    no   \n",
       "\n",
       "        wearing necktie.confidence yaw.label  yaw.confidence young.label  \\\n",
       "A1.jpg                        0.36     18.79             1.0          no   \n",
       "A2.jpg                        0.71     -0.62             1.0         yes   \n",
       "A3.jpg                        0.66    -14.97             1.0         yes   \n",
       "A4.jpg                        0.75     38.22             1.0          no   \n",
       "A6.jpg                        0.17      2.08             1.0          no   \n",
       "\n",
       "        young.confidence gender.manual  race.manual  \n",
       "A1.jpg              0.33          male        white  \n",
       "A2.jpg              0.60        female        white  \n",
       "A3.jpg              0.07        female        white  \n",
       "A4.jpg              0.33          male        white  \n",
       "A6.jpg              0.14          male        black  \n",
       "\n",
       "[5 rows x 90 columns]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "manual_labels_df = pd.read_csv('data/image_labels_manual.csv', index_col=0)\n",
    "manual_labels_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5oclock shadow.label           yes\n",
       "5oclock shadow.confidence     0.13\n",
       "age.label                       46\n",
       "age.confidence                 0.6\n",
       "arched eyebrows.label           no\n",
       "                             ...  \n",
       "yaw.confidence                   1\n",
       "young.label                     no\n",
       "young.confidence              0.33\n",
       "gender.manual                 male\n",
       "race.manual                  white\n",
       "Name: A1.jpg, Length: 90, dtype: object"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "manual_labels_df.loc['A1.jpg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>playful,serious</th>\n",
       "      <th>shy,bold</th>\n",
       "      <th>cheery,sorrowful</th>\n",
       "      <th>masculine,feminine</th>\n",
       "      <th>charming,awkward</th>\n",
       "      <th>lewd,tasteful</th>\n",
       "      <th>intellectual,physical</th>\n",
       "      <th>strict,lenient</th>\n",
       "      <th>refined,rugged</th>\n",
       "      <th>trusting,suspicious</th>\n",
       "      <th>...</th>\n",
       "      <th>picky,always down</th>\n",
       "      <th>proactive,reactive</th>\n",
       "      <th>prudish,flirtatious</th>\n",
       "      <th>innocent,jaded</th>\n",
       "      <th>touchy-feely,distant</th>\n",
       "      <th>muddy,washed</th>\n",
       "      <th>quirky,predictable</th>\n",
       "      <th>never cries,often crying</th>\n",
       "      <th>race.label</th>\n",
       "      <th>gender.label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>WE/1</th>\n",
       "      <td>36.4</td>\n",
       "      <td>98.8</td>\n",
       "      <td>72.3</td>\n",
       "      <td>47.3</td>\n",
       "      <td>24.2</td>\n",
       "      <td>29.3</td>\n",
       "      <td>75.4</td>\n",
       "      <td>64.7</td>\n",
       "      <td>82.1</td>\n",
       "      <td>57.2</td>\n",
       "      <td>...</td>\n",
       "      <td>60.4</td>\n",
       "      <td>74.0</td>\n",
       "      <td>83.9</td>\n",
       "      <td>78.7</td>\n",
       "      <td>70.9</td>\n",
       "      <td>33.2</td>\n",
       "      <td>31.3</td>\n",
       "      <td>31.8</td>\n",
       "      <td>white</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/2</th>\n",
       "      <td>57.4</td>\n",
       "      <td>89.2</td>\n",
       "      <td>64.7</td>\n",
       "      <td>13.8</td>\n",
       "      <td>19.2</td>\n",
       "      <td>45.0</td>\n",
       "      <td>67.5</td>\n",
       "      <td>38.9</td>\n",
       "      <td>80.4</td>\n",
       "      <td>54.2</td>\n",
       "      <td>...</td>\n",
       "      <td>73.9</td>\n",
       "      <td>65.0</td>\n",
       "      <td>93.3</td>\n",
       "      <td>81.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>28.1</td>\n",
       "      <td>41.7</td>\n",
       "      <td>27.5</td>\n",
       "      <td>white</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/3</th>\n",
       "      <td>26.8</td>\n",
       "      <td>57.4</td>\n",
       "      <td>9.1</td>\n",
       "      <td>92.1</td>\n",
       "      <td>27.1</td>\n",
       "      <td>66.8</td>\n",
       "      <td>44.0</td>\n",
       "      <td>74.3</td>\n",
       "      <td>34.0</td>\n",
       "      <td>12.3</td>\n",
       "      <td>...</td>\n",
       "      <td>71.4</td>\n",
       "      <td>44.9</td>\n",
       "      <td>76.8</td>\n",
       "      <td>31.4</td>\n",
       "      <td>13.3</td>\n",
       "      <td>84.4</td>\n",
       "      <td>12.7</td>\n",
       "      <td>72.8</td>\n",
       "      <td>white</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/4</th>\n",
       "      <td>56.9</td>\n",
       "      <td>74.4</td>\n",
       "      <td>28.2</td>\n",
       "      <td>58.4</td>\n",
       "      <td>27.7</td>\n",
       "      <td>76.2</td>\n",
       "      <td>30.6</td>\n",
       "      <td>58.3</td>\n",
       "      <td>33.7</td>\n",
       "      <td>35.7</td>\n",
       "      <td>...</td>\n",
       "      <td>65.4</td>\n",
       "      <td>29.8</td>\n",
       "      <td>82.3</td>\n",
       "      <td>49.3</td>\n",
       "      <td>25.2</td>\n",
       "      <td>72.2</td>\n",
       "      <td>55.8</td>\n",
       "      <td>35.9</td>\n",
       "      <td>white</td>\n",
       "      <td>female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WE/5</th>\n",
       "      <td>23.6</td>\n",
       "      <td>36.0</td>\n",
       "      <td>29.4</td>\n",
       "      <td>60.6</td>\n",
       "      <td>85.8</td>\n",
       "      <td>58.8</td>\n",
       "      <td>6.8</td>\n",
       "      <td>68.0</td>\n",
       "      <td>37.4</td>\n",
       "      <td>13.3</td>\n",
       "      <td>...</td>\n",
       "      <td>78.3</td>\n",
       "      <td>56.3</td>\n",
       "      <td>36.9</td>\n",
       "      <td>34.9</td>\n",
       "      <td>30.3</td>\n",
       "      <td>52.4</td>\n",
       "      <td>39.6</td>\n",
       "      <td>75.1</td>\n",
       "      <td>black</td>\n",
       "      <td>male</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 402 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      playful,serious  shy,bold  cheery,sorrowful  masculine,feminine  \\\n",
       "id                                                                      \n",
       "WE/1             36.4      98.8              72.3                47.3   \n",
       "WE/2             57.4      89.2              64.7                13.8   \n",
       "WE/3             26.8      57.4               9.1                92.1   \n",
       "WE/4             56.9      74.4              28.2                58.4   \n",
       "WE/5             23.6      36.0              29.4                60.6   \n",
       "\n",
       "      charming,awkward  lewd,tasteful  intellectual,physical  strict,lenient  \\\n",
       "id                                                                             \n",
       "WE/1              24.2           29.3                   75.4            64.7   \n",
       "WE/2              19.2           45.0                   67.5            38.9   \n",
       "WE/3              27.1           66.8                   44.0            74.3   \n",
       "WE/4              27.7           76.2                   30.6            58.3   \n",
       "WE/5              85.8           58.8                    6.8            68.0   \n",
       "\n",
       "      refined,rugged  trusting,suspicious  ...  picky,always down  \\\n",
       "id                                         ...                      \n",
       "WE/1            82.1                 57.2  ...               60.4   \n",
       "WE/2            80.4                 54.2  ...               73.9   \n",
       "WE/3            34.0                 12.3  ...               71.4   \n",
       "WE/4            33.7                 35.7  ...               65.4   \n",
       "WE/5            37.4                 13.3  ...               78.3   \n",
       "\n",
       "      proactive,reactive  prudish,flirtatious  innocent,jaded  \\\n",
       "id                                                              \n",
       "WE/1                74.0                 83.9            78.7   \n",
       "WE/2                65.0                 93.3            81.0   \n",
       "WE/3                44.9                 76.8            31.4   \n",
       "WE/4                29.8                 82.3            49.3   \n",
       "WE/5                56.3                 36.9            34.9   \n",
       "\n",
       "      touchy-feely,distant  muddy,washed  quirky,predictable  \\\n",
       "id                                                             \n",
       "WE/1                  70.9          33.2                31.3   \n",
       "WE/2                  66.0          28.1                41.7   \n",
       "WE/3                  13.3          84.4                12.7   \n",
       "WE/4                  25.2          72.2                55.8   \n",
       "WE/5                  30.3          52.4                39.6   \n",
       "\n",
       "      never cries,often crying  race.label  gender.label  \n",
       "id                                                        \n",
       "WE/1                      31.8       white        female  \n",
       "WE/2                      27.5       white          male  \n",
       "WE/3                      72.8       white        female  \n",
       "WE/4                      35.9       white        female  \n",
       "WE/5                      75.1       black          male  \n",
       "\n",
       "[5 rows x 402 columns]"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = list()\n",
    "for value in labeled_df.index.values:\n",
    "    \n",
    "    source_labels = manual_labels_df.loc[value.replace('/', '') + \".jpg\"]\n",
    "    if len(source_labels) == 0:\n",
    "        continue\n",
    "        \n",
    "    labels.append({\n",
    "        'id': value, \n",
    "        'race.label': source_labels['race.manual'], \n",
    "        'gender.label': source_labels['gender.manual']}\n",
    "    )\n",
    "    \n",
    "consolidated_df = pd.concat([labeled_df, pd.DataFrame(labels).set_index('id')], axis=1)\n",
    "consolidated_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_invalid_race = consolidated_df[consolidated_df['race.label'] != 'unknown']\n",
    "drop_invalid_gender = drop_invalid_race[drop_invalid_race['gender.label'] != 'unknown']\n",
    "\n",
    "drop_invalid_gender.to_csv('data/opendata-with-labels.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45.5"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "consolidated_df.loc['A/1']['empath,psychopath']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1462, 402)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drop_invalid_gender.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}