notebooks/pubmed-curate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c8acc16b-8e04-46de-8f1e-d9872242860e",
   "metadata": {},
   "source": [
    "# Pubmed Paper Classification - Curate Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56b4b774-fc15-4785-9b89-9ba2138182bc",
   "metadata": {},
   "source": [
    "## Import relevant libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "51f9801a-7c27-4c5d-a35e-5ca428ad0bd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "import pandas as pd\n",
    "from tqdm import tqdm as _tqdm\n",
    "from tqdm.notebook import tqdm\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f6193040-ee65-41d2-9de4-0958d916f0d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dt = '8.26.21'\n",
    "input_fn = '../input/pubmed-%s.xlsx' % input_dt\n",
    "efetch_fn = '../input/efetch-%s.xml' % input_dt\n",
    "failed_fn = '../input/failed-%s.pickle' % input_dt\n",
    "data_fn = '../input/abstracts-%s.pickle' % input_dt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8094373a-777d-4ec9-9bcb-db6d7d53b54a",
   "metadata": {},
   "source": [
    "## Preprocess article list and labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8eb60a0c-1194-487d-8330-6026d17a4f6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25553339</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>26663098</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27014455</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27088280</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27112773</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13997</th>\n",
       "      <td>34231533</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13998</th>\n",
       "      <td>34233515</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13999</th>\n",
       "      <td>34234854</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14000</th>\n",
       "      <td>34235035</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14001</th>\n",
       "      <td>Not in Pubmed</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14002 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                PMID  Category\n",
       "0           25553339         1\n",
       "1           26663098         1\n",
       "2           27014455         1\n",
       "3           27088280         1\n",
       "4           27112773         1\n",
       "...              ...       ...\n",
       "13997       34231533        10\n",
       "13998       34233515        10\n",
       "13999       34234854        10\n",
       "14000       34235035        10\n",
       "14001  Not in Pubmed        10\n",
       "\n",
       "[14002 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pm = pd.read_excel(input_fn)[['PMID','Title','Category']]\n",
    "orig = pm.copy()\n",
    "pm.drop(['Title'], axis=1, inplace=True)\n",
    "pm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f8e1e64a-3e92-478c-8646-a1b7f339c679",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  3,  4,  5,  6,  7,  8,  9, 10])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pm.Category.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4dabd9b7-e7ee-452a-9759-f9a979e43249",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25553339</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>26663098</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27014455</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27088280</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27112773</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13996</th>\n",
       "      <td>34231311</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13997</th>\n",
       "      <td>34231533</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13998</th>\n",
       "      <td>34233515</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13999</th>\n",
       "      <td>34234854</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14000</th>\n",
       "      <td>34235035</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13998 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PMID  Category\n",
       "0      25553339         1\n",
       "1      26663098         1\n",
       "2      27014455         1\n",
       "3      27088280         1\n",
       "4      27112773         1\n",
       "...         ...       ...\n",
       "13996  34231311        10\n",
       "13997  34231533        10\n",
       "13998  34233515        10\n",
       "13999  34234854        10\n",
       "14000  34235035        10\n",
       "\n",
       "[13998 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pm.drop_duplicates(subset='PMID', inplace=True)\n",
    "pm_filtered = pm[pd.to_numeric(pm.Category, errors='coerce').notnull()]\n",
    "pm_filtered = pm_filtered[(pm_filtered.Category >= 1) & (pm_filtered.Category <= 10)]\n",
    "pm_filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a8aa2014-d99b-4354-aeba-0cf761e63330",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAGICAYAAAA+mPioAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAx/UlEQVR4nO3de5yWdZ3/8ddHQROPeEJlTPAnioI6GSpUS5YrePp5LiEpLFp309pK02zLFU1T2y0Pv9zMxBUPgWQormuokChWYqDkAVMsUcATimZGaujn98d1zXgzzsDcMDP3DLyej8c85r6/1+H+fO/j+/5ehzsyE0mStG5br9YFSJKk2jMQSJIkA4EkSTIQSJIkDASSJAkDgSRJwkCwVouIEyMiK/7+EhG/j4gvR0S31Vjfgoi4vhXzjS1v728RsXkz00dX1LRLtXW0cJt9yvWdWMUyPy2XuXg1bu9rEXFMM+1jI6KqY3kj4oCyjgOqraOF9V1Tcf++GxF/joh5ETEuIoaswXqPiohT26LGNRER9eX9vGUVy3SPiJMj4tcR8VpEvBURT0fE1RGxT8V8MyJiRrsUvhqqfSwrXvN9qriNEyPiC6tR14KK6w2vvy9Ws57VqWt1+qjWMRCsGz4FDAGOBR4A/h/w7x1wu38HjmumfTTwlw64/RZFxEbAp8urn1mNgPQ14H2BALiK4r6utSUUdXyEos4fAbsDv4mI763mOo8Cah4IgHrgbKBVgSAiNgamAz+geP6fAAwDzgP6ltM6s2oey/8t532+ivWfCFQVCIDvAkdXuUy1TqT5ulanj2qFqr8lqkuam5lPlZfvLL+Vf5X2DwWTgc8C4xoaImJH4ABgPMULvlaOAjYDbgcOBQ4GblvVQhGxYWa+1dL0zFwELGqjGtfE25l5f8X16RHxY+Bi4FsRMSczf1Gj2jrapcD+wAGZ+duK9nuAcRHR3h9sa6rVj2VmLqEIEO2i4fmfmX9sr9tYlfbu47rMEYJ10++AzSJi25aGq1c2LBcR/xQRT0XEmxHxYER8ooXbuRYYGhE7VbR9FngGuLe5BSLimIi4PyKWlUO7P4+IDzaZp0dE/FdEvBIRb0TErUBdaztfGg28ShFK/lZeb1pLw6aPgRFxR0S8AUwqh0p3Ak6oGM69pnKZJuvpFhHfLId634yIJRExNSL6r6zA1twX1cjitKRnAC9SjHA03M42EfGTiHiyvK2FEfGziOhdMc81FPdR74o+LyinfSAiLo6IR8vH44WI+J+m/YuI7SJifEQ8Vw7ZPx8Rt0XEthXz9IiIi8rh/LfL/9+OiPXK6ScC/13OPr+ilj4t3Ifbl3X/tEkYqLxfbm7pPmurvpXPge9GxB/L58DLEXFfRHyspdtemZU8lu973UbEZyLiobL+1yPikYj453LaDODjwEcr7ssZTdY1tHzuvQbMKqetsMmgwgYR8cOIeKl8Lt3W9LEp1zm2SdsKm/xaWVdlH7tHxHlRbNZ8u/x/XkR0b+Y2/jkizi0fo9fKx7Pa94+1kiME66a+wDvAG6ux7AHAh4FvA28B3wR+GRF7Z+YTTeadCSygGKJtGNr8LHA98L7t7BHxL8CPKd7wzwU2BcYC90TEXpnZsJnhJ8DxwDkU4eYg4Get7UBE7AD8I8WHxJKIuAU4JiJ6ZuarzSwyhWKU4yLgXeDPFCMLvy/rg5V/Y5lIMSJxCTAN+AAwFNge+EMLNbb2vqhKZr4dEdOB4yKiW2Yupxh6fxP4VtmPHYDTgF9HRP/MfJNiiHgbYF/giHJ1DSMlG5b1nUcxjLslcDLw24jYPTNfKOe7jiJInQ4sBHoBBwI9yj53A+4A9ihv7xFgMHBWuc7TKIaLzwO+Q7EprGE0pqXh409QvM/dWu191ZZ9o3idfJ3idTOXYnRqEK3c7NGcFh7LFZSB43rgsrK29YD+wBblLCeX09cH/rlse73Jam4AJlBs/lvVZ8a3KPr3eWBbitf9nRExIDP/XkX3WlNXpfEUmwC/B9xHsXnl28DOwGeaqfE3FJsjtqXYlHQ9xXvbui0z/VtL/yi+/SawG8ULuSfFi+sd4JZyngPKeQ5oYdk+FW0LgLeBHSvaNgWWAtdVtI0tl+1G8WH2eNm+X9ner2L9u5TTNqH4oL26SR19y9v8Wnl9t7L+M5vM9+NyfSe24n45o5x3SHl9eHn9X5rM19CPrzazjgXA9c20j6X8Alde/2S5jn9dST0rPAatvS9Wsr5rgEUrmX5BeXu9Wpi+PrBjOc/RrV1vk+V7UOwn8vWK9jdWcT98trzNoU3av132e9smz81dWlHLN8t5d2vla2YGMKMd+nYbMLk1NazJY0mT1y3wDWBpK/p8XzPtDeu6uIW6FlRc71POOw9Yr6L9o2X7mIq2BMY2WV/D8idWUVdDHwe2sM7vlO17NbmNGU3m+0bZvkO1j8/a9ucmg3XDHyh28FsK/BdF4q92J6IG92fmwoYrWXxTbdjJpznXAv0jYl/gc+Xy85uZbwjFt6YbyuHVbuU3xoVl/UPL+fan+JYzqcnyE6vow2hgfr43hDwNeI5mNhuUWhxSboVhFG82P61imdbeF6sryv+NozQR8aUojkB5A1gOPFtO2q1VK4z4dETMKoeVlwN/pQg2lcv/Djg9Ir4aEXtGRDRZzcEUm5N+06TfdwLdKUYLOlwb9e13wKERcX5EfCwiNmir8sr/LR3Z8jugZ0RcHxGHR8QWq3Eb1Tz/b8rMdxuuZOavKUZx2nNH24bXQ9MjoBquf7xJ++1Nrj9S/l/tzXFrCwPBuuFoiqHe/sDGmfm5zFy6mut6sYW23s20k8XOjL8FxgAjKAJCcxq2I0+jCC+Vf3sCW5XTt2+hjubqep+IGEQxJD05IrYo3yA3pdgBcnBE7NrMYmuyN/NWFN/Q/lbFMq29L1bXjhTfuJcCRMRXKILiNIq92PfjvQ/fD6xqZRHxf4Ebgccphmf3p3i+LWmy/PEUQ/dnAA8DiyPi36PcP4Ci3zvx/j4/UE5fnX43hNedVjpXC9qwb9+jODLiCIpNaa9ExH9HxNarU1eFFR7LpjLzHopNKztSfLAviYhpEbFXFbdRzfO/qveHNtKw2aVpnS80md6g6X3VsOlrlc/1tZ37EKwbHs33jjJo6s3yf9NvLC29+fZqoW3xSm7/WuByim9XLX2Tf6X8fyLwWDPTG7aZN7zoewF/WkVdzWkYBfhm+dfU5yiGGiutyW+EvwxsGREbVREKWntfVK38ZvqPFCM1DducRwDTM/O0ivn6VrHaEcBTmXlixfLdafJGnJkvAacAp0TEbhSPxTkUH64/puj307x3OGhTC6qoqcEMik1M/5dipKFabdK3LLafXwRcFBHbAYcDP6TY/HD8atTV0mP5Ppl5E3BTRGxCsXnqImBqRNRVfptfiWqe/y29P8ytuP4WrX+/aY2GD/jtgMqjH7ZrMl2r4AiBnin/D2zSflgL8w+O4tBBACJi03LeZvfgLt1I8e3pwmx+pz0odvL5C8V24dnN/DXssDiLYse+ph8aI1Zy+w21bgCMLNfxiWb+5gKfbWa4tzlvARu1Yr47KYZ1qzlhS2vvi6qU/fo+xTfxypMx9aD4Jl7p882soqU+96AIe5U+S7G9vVmZ+URm/hvFkR4Nz72pFN9k32ih3y9X1EELtTS9necotnefFC2clCkijlrJKtqqb5XTX8jMqyhGZN43vTVW8li2KDPfyMzbKHbK3Z73PoRb+1xujeMqRkWIiI9SHAFU+f7wDK17v2ltXQ1HLDV9Dzih/D+jFesQjhCs8zLz+Yi4h+J45peBl4BRFHvnNudFir2Gx/LeUQYbU+wV3tJtvMoqTmKSma9HxOnA5RGxDfBLih3relNsA5yRmT/LzCci4mfAueUbz+8ottMf2oruHkbxJnhaZs5oOjEifkLxTfUA4O5VrGse8A8RcTjF0OTLmbmgmX7dHRG/AH5YBqlfUWwPHwr8b3N1tPa+WEV9G0REw7B/D4rt3aMotuWel5m3VMw7FfhmRPwbxfD8J2n+hFLzKEY7vgTMBt7MzEfK5Y+K4oyPt1HsPf8V4LWGBaM4Y+U0iv1XGvZpOZJiR9eGb+43UASR6RHxA4qjODYA/g/FUPtRmbmsrAOKb+Pjy3U9nJlvt3BffA3YtVzvFWUdb1A8x08o672lhWXbpG8RMaXsz4MUQeFDFPtM/KSF261UzWO5gog4l+Ib+t0U+8nUAf9KcW6ShiNj5gEnR8TxFN+w/7K6oZNi89st5WtpG4qdHuez4qbCicB3IuLbwP3AP1AE9aZaVVdmPhoRE4Cx5T4nv6G4b84CJpTPUbVGrfdq9K/9/mjl3tgUbxL/Q/Em9wLF9s4v0vxRBteX0/5IEQgeAj7ZZH1jy2W7VVsbxQf73RSHGC2jeDO5GtijYp4eFB/cSyne2G/lvb2ZT1zJbd5SrrdHC9M3L2/zmlX1g2J/jJnl/Nl0mSbzdqPYU/5Jiu29Syh2bNqtnH4AzR/pscr7ooV+XFOuLylGU/5CsQ18HDC4mfk3Ku/PJeW8t1Ec0bDCntsUwW8CxQdaUu5lTjHSeB7FB84yihP+fKh8vjTcLxtSfPg9Vj5mr1OEuc80qeUD5X34h/L5tbScb2zl40CxPX4xxeaAFZ6nLdwn3SmG9H9T3vbbFJsnrqLcC72cbwYVe6G3Vd8oDpm8n2KzyN+AJ8o+dW/jx/LEyvuDIgTfQbGp7S2KfSrGUbFHPcXQ+u3luhv3wmcl7x+0fJTByRSbQpaU99f/An2beYwvLWv6C8UIYsMRSCdWUVfle9MG5eP0DEUge6a83r2ZGr/YpJ4DaOb1ty7+RXmHSJKkdZj7EEiSJAOBJEkyEEiSJAwEkiQJA4EkSWIdPw/B1ltvnX369Kl1GZIkdYg5c+a8nJnbNDdtnQ4Effr0Yfbs2bUuQ5KkDhERz7Q0zU0GkiTJQCBJkgwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBurhLL72UgQMHMmDAAC655BIAjj/+eOrr66mvr6dPnz7U19c3zv/www8zZMgQBgwYwJ577smbb765wvqOOOIIBg4c2IE9kKTOYZ0+MZG6tkcffZSf/vSnPPDAA2ywwQYcfPDBHH744dx4442N85x22mlsvvnmACxfvpxRo0Zx3XXXsffee/PKK6/QvXv3xnknT57MJpts0uH9kKTOwBECdVmPP/44+++/Pz169KBbt258/OMfZ/LkyY3TM5NJkyYxcuRIAO6880722msv9t57bwC22mor1l9/fQDeeOMNfvjDH/Kd73yn4zsiSZ1AhwSCiLg6Il6KiEcr2raMiLsiYn75v2fZHhFxWUQ8FREPR8Q+FcuMLuefHxGjK9o/HBGPlMtcFhHREf1SbQ0cOJCZM2fyyiuvsGzZMm6//XYWLlzYOH3mzJn06tWLfv36AfDkk08SEQwfPpx99tmH73//+43znnXWWZx22mn06NGjw/shSZ1BR40QXAMc3KTtTGB6ZvYDppfXAQ4B+pV/JwE/hiJAAGcD+wP7AWc3hIhynn+qWK7pbWkttPvuu/PNb36TYcOGcfDBB1NfX9/4jR9gwoQJjaMDUGwyuO+++7jhhhu47777uPnmm5k+fTpz587lj3/8I0cffXQtuiFJnUKHBILMvBdY2qT5SGB8eXk8cFRF+7VZuB/YIiK2B4YDd2Xm0sx8FbgLOLictllm3p+ZCVxbsS6t5caMGcOcOXO499576dmzJ7vuuitQfPhPnjyZ448/vnHeuro6hg4dytZbb02PHj049NBDefDBB/ntb3/L7Nmz6dOnDx/72Md48sknOeCAA2rUI0mqjVruQ9ArM58vL78A9Cov9wYWVsy3qGxbWfuiZtqbFREnRcTsiJi9ZMmSNeuBau6ll14C4Nlnn2Xy5Ml85jOfAWDatGn079+furq6xnmHDx/OI488wrJly1i+fDn33HMPe+yxB1/60pd47rnnWLBgAffddx+77rorM2bMqEV3JKlmOsVRBpmZEZEddFtXAlcCDBo0qENuU+3n2GOPbTxa4PLLL2eLLbYAYOLEiStsLgDo2bMnp556Kvvuuy8RwaGHHsphhx1Wg6olqfOpZSB4MSK2z8zny2H/l8r2xcCOFfPVlW2LgQOatM8o2+uamV/rgJkzZzbbfs011zTbPmrUKEaNGtXi+vr06cOjjz7a4nRJWlvVcpPBrUDDkQKjgSkV7Z8rjzYYDPy53LRwBzAsInqWOxMOA+4op70eEYPLows+V7EuSZLUCh0yQhAREyi+3W8dEYsojha4EJgUEWOAZ4BPl7PfDhwKPAUsAz4PkJlLI+K7wO/K+c7NzIYdFU+mOJJhI+CX5Z8kSWqlKHbMXzcNGjQoZ8+eXesyJEnqEBExJzMHNTfNMxVKkiQDgSRJ6iSHHUrN+fDp19a6hBbN+Y/P1boESWpTjhBIkiQDgSRJMhBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmS6ASBICK+HhGPRcSjETEhIj4QEX0jYlZEPBURN0bEBuW8G5bXnyqn96lYz7fK9iciYnjNOiRJUhdU00AQEb2BfwUGZeZAYH1gBHARcHFm7gK8CowpFxkDvFq2X1zOR0TsUS43ADgY+K+IWL8j+yJJUldW8xECoBuwUUR0A3oAzwOfBG4qp48HjiovH1lep5x+YERE2T4xM9/KzKeBp4D9OqZ8SZK6vpoGgsxcDPwn8CxFEPgzMAd4LTOXl7MtAnqXl3sDC8tll5fzb1XZ3swyK4iIkyJidkTMXrJkSdt2SJKkLqrWmwx6Uny77wvsAGxMMeTfbjLzyswclJmDttlmm/a8KUmSuoxabzL4R+DpzFySmX8HJgMfBbYoNyEA1AGLy8uLgR0ByumbA69UtjezjCRJWoVaB4JngcER0aPcF+BAYB5wN3BcOc9oYEp5+dbyOuX0X2Vmlu0jyqMQ+gL9gAc6qA+SJHV53VY9S/vJzFkRcRPwILAceAi4EvhfYGJEnFe2jSsXGQdcFxFPAUspjiwgMx+LiEkUYWI5cEpmvtOhnZEkqQuraSAAyMyzgbObNP+JZo4SyMw3gU+1sJ7zgfPbvEBJktYBtd5kIEmSOgEDgSRJMhBIkiQDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZLoBIEgIraIiJsi4g8R8XhEDImILSPiroiYX/7vWc4bEXFZRDwVEQ9HxD4V6xldzj8/IkbXrkeSJHU9NQ8EwKXA1MzsD+wNPA6cCUzPzH7A9PI6wCFAv/LvJODHABGxJXA2sD+wH3B2Q4iQJEmrVtNAEBGbA0OBcQCZ+XZmvgYcCYwvZxsPHFVePhK4Ngv3A1tExPbAcOCuzFyama8CdwEHd1hHJEnq4mo9QtAXWAL8d0Q8FBFXRcTGQK/MfL6c5wWgV3m5N7CwYvlFZVtL7e8TESdFxOyImL1kyZI27IokSV1XrQNBN2Af4MeZ+SHgr7y3eQCAzEwg2+oGM/PKzByUmYO22WabtlqtJEldWq0DwSJgUWbOKq/fRBEQXiw3BVD+f6mcvhjYsWL5urKtpXZJktQKNQ0EmfkCsDAidiubDgTmAbcCDUcKjAamlJdvBT5XHm0wGPhzuWnhDmBYRPQsdyYcVrZJkqRW6FbrAoCvADdExAbAn4DPUwSVSRExBngG+HQ57+3AocBTwLJyXjJzaUR8F/hdOd+5mbm047ogSVLXVvNAkJlzgUHNTDqwmXkTOKWF9VwNXN2mxUmStI6o9T4EkiSpEzAQSJKk1geCiPhUC+3HtV05kiSpFqoZIRjXQvuVbVGIJEmqnVXuVBgRO5cX14uIvkBUTN4ZeLM9CpMkSR2nNUcZPEVxpsAA/thk2gvA2DauSZIkdbBVBoLMXA8gIu7JzI+3f0mSJKmjtXofAsOAJElrr1afmKjcf+B8oB7YpHJaZn6wbcuSJEkdqZozFf6MYh+C0yhOGyxJktYS1QSCAcBHM/Pd9ipGkiTVRjXnIbgX+FB7FSJJkmqnmhGCBcDUiLiZ4nDDRpn5721ZlCRJ6ljVBIKNgduA7sCO7VOOJEmqhVYHgsz8fHsWIkmSaqeaww53bmlaZv6pbcqRJEm1UM0mg8pTGDfI8v/6bVaRJEnqcNVsMljhiISI2A44G5jZ1kVJkqSOVc1hhyvIzBeArwEXtFk1kiSpJlY7EJR2A3q0RSGSJKl2qtmpcCbv7TMARRAYAJzb1kVJkqSOVc1OhVc1uf5X4PeZOb8N65EkSTVQzU6F49uzEEmSVDut3ocgIrpHxDkR8aeIeLP8f05EbNCeBUqSpPZXzSaD7wP7Af8CPAPsBJwFbAZ8ve1LkyRJHaWaQPApYO/MfKW8/kREPAj8HgOBJEldWjWHHUaV7ZIkqYuoJhD8HPifiBgeEbtHxMHALWW7JEnqwqrZZHAG8B3gcmAHYDEwATivHeqSJEkdaJUjBBHx0Yi4KDPfzsx/z8xdMrNHZvYDNgT2af8yJUlSe2rNJoN/A+5tYdrdwLfbrhxJklQLrQkE9cDUFqZNAz7cZtVIkqSaaE0g2Axo6eRD3YFN264cSZJUC60JBH8AhrUwbVg5XZIkdWGtOcrgYuAnEbE+cEtmvhsR6wFHURxxcGo71idJkjrAKgNBZv4sIrYDxgMbRsTLwNbAW8DZmTmhnWuUJEntrFXnIcjMH0bEVcAQYCvgFeC3mfl6exYnSZI6RjU/f/w6cEc71iJJkmqkmlMXS5KktZSBQJIkGQgkSZKBQJIkYSCQJEkYCCRJEgYCSZKEgUCSJGEgkCRJGAgkSRIGAkmShIFAkiRhIJAkSRgIJEkSBgJJkkQnCQQRsX5EPBQRt5XX+0bErIh4KiJujIgNyvYNy+tPldP7VKzjW2X7ExExvEZdkSSpS+oUgQD4KvB4xfWLgIszcxfgVWBM2T4GeLVsv7icj4jYAxgBDAAOBv4rItbvoNolSeryah4IIqIOOAy4qrwewCeBm8pZxgNHlZePLK9TTj+wnP9IYGJmvpWZTwNPAft1SAckSVoL1DwQAJcAZwDvlte3Al7LzOXl9UVA7/Jyb2AhQDn9z+X8je3NLCNJklahpoEgIg4HXsrMOR14mydFxOyImL1kyZKOullJkjq1Wo8QfBQ4IiIWABMpNhVcCmwREd3KeeqAxeXlxcCOAOX0zYFXKtubWWYFmXllZg7KzEHbbLNN2/ZGkqQuqqaBIDO/lZl1mdmHYqfAX2XmCcDdwHHlbKOBKeXlW8vrlNN/lZlZto8oj0LoC/QDHuigbkiS1OV1W/UsNfFNYGJEnAc8BIwr28cB10XEU8BSihBBZj4WEZOAecBy4JTMfKfjy5YkqWvqNIEgM2cAM8rLf6KZowQy803gUy0sfz5wfvtVKEnS2qvW+xBIkqROwEAgSZIMBJIkyUAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEmSMBBIkiQMBJIkCQOBJEnCQCBJkjAQSJIkDASSJAkDgSRJwkAgSZIwEEiSJAwEkiQJA4EkScJAIEkALFy4kE984hPsscceDBgwgEsvvRSApUuXctBBB9GvXz8OOuggXn31VQBuuOEG9tprL/bcc08+8pGP8Pvf/75xXV/4whfYdtttGThwYE36Iq0OA4EkAd26deMHP/gB8+bN4/777+fyyy9n3rx5XHjhhRx44IHMnz+fAw88kAsvvBCAvn37cs899/DII49w1llncdJJJzWu68QTT2Tq1Km16oq0WgwEkgRsv/327LPPPgBsuumm7L777ixevJgpU6YwevRoAEaPHs0tt9wCwEc+8hF69uwJwODBg1m0aFHjuoYOHcqWW27ZsR2Q1pCBQJKaWLBgAQ899BD7778/L774Ittvvz0A2223HS+++OL75h83bhyHHHJIR5cptalutS5AkjqTN954g2OPPZZLLrmEzTbbbIVpEUFErNB29913M27cOO67776OLFNqc44QSFLp73//O8ceeywnnHACxxxzDAC9evXi+eefB+D5559n2223bZz/4Ycf5otf/CJTpkxhq622qknNUlsxEEgSkJmMGTOG3XffnVNPPbWx/YgjjmD8+PEAjB8/niOPPBKAZ599lmOOOYbrrruOXXfdtSY1S23JQCBJwK9//Wuuu+46fvWrX1FfX099fT233347Z555JnfddRf9+vVj2rRpnHnmmQCce+65vPLKK5x88snU19czaNCgxnWNHDmSIUOG8MQTT1BXV8e4ceNq1S2p1SIza11DzQwaNChnz55d6zLUgg+ffm2tS2jRnP/4XK1LkKSqRcSczBzU3DRHCCRJUm0DQUTsGBF3R8S8iHgsIr5atm8ZEXdFxPzyf8+yPSLisoh4KiIejoh9KtY1upx/fkSMrlWfJElq0NxZK48//vjGzVJ9+vShvr6+cdoFF1zALrvswm677cYdd9zR2D516lR22203dtlll8aTY7W1Wo8QLAdOy8w9gMHAKRGxB3AmMD0z+wHTy+sAhwD9yr+TgB9DESCAs4H9gf2AsxtChCRJtdLcWStvvPFG5s6dy9y5czn22GMbj2iZN28eEydO5LHHHmPq1KmcfPLJvPPOO7zzzjuccsop/PKXv2TevHlMmDCBefPmtXmtNQ0Emfl8Zj5YXv4L8DjQGzgSGF/ONh44qrx8JHBtFu4HtoiI7YHhwF2ZuTQzXwXuAg7uuJ5IkvR+KztrZWYyadIkRo4cCcCUKVMYMWIEG264IX379mWXXXbhgQce4IEHHmCXXXZh5513ZoMNNmDEiBFMmTKlzWvtNCcmiog+wIeAWUCvzHy+nPQC0Ku83BtYWLHYorKtpXZJauSOqupMZs6cSa9evejXrx8AixcvZvDgwY3T6+rqWLx4MQA77rjjCu2zZs1q83pqvckAgIjYBPgF8LXMfL1yWhaHQbTZoRARcVJEzI6I2UuWLGmr1UqSVJUJEyY0jg50BjUfIYiI7hRh4IbMnFw2vxgR22fm8+UmgZfK9sXAjhWL15Vti4EDmrTPaO72MvNK4EooDjtso25IktRqy5cvZ/LkycyZM6exrXfv3ixc+N5g96JFi+jduxjsbqm9LdX6KIMAxgGPZ+YPKybdCjQcKTAamFLR/rnyaIPBwJ/LTQt3AMMiome5M+Gwsk2SpE5n2rRp9O/fn7q6usa2I444gokTJ/LWW2/x9NNPM3/+fPbbbz/23Xdf5s+fz9NPP83bb7/NxIkTOeKII9q8plqPEHwU+CzwSETMLdv+DbgQmBQRY4BngE+X024HDgWeApYBnwfIzKUR8V3gd+V852bm0g7pgSRJLRg5ciQzZszg5Zdfpq6ujnPOOYcxY8YwceLE920uGDBgAJ/+9KfZY4896NatG5dffjnrr78+AD/60Y8YPnw477zzDl/4whcYMGBAm9fqmQo9U2Gn5Q5gams+p7Su80yFkiRppQwEkiTJQCBJkgwEkiSJ2h9lIElSl9SZd1KF6ndUdYRAkiQZCCRJkoFAkiRhIJAkSRgIJEkSBgJJkoSBQJIkYSCQJEkYCCRJEgYCSZKEgUCSJGEgkCRJGAgkSRIGAkmShIFAkiRhIJAkSRgIJEkSBgJJkoSBQJIkYSCQJEkYCCRJEgYCSZKEgUCSJGEgkCRJGAgkSRIGAkmShIFAkiRhIJAkSRgIJEkSBgJJkoSBQJIkYSCQJEkYCCRJEgYCSZKEgWCNfOELX2Dbbbdl4MCBjW1nnXUWe+21F/X19QwbNoznnnsOgD/84Q8MGTKEDTfckP/8z/+sVcmS1OU09157+umn079/f/baay+OPvpoXnvttcZpDz/8MEOGDGHAgAHsueeevPnmmzWouusxEKyBE088kalTp67Qdvrpp/Pwww8zd+5cDj/8cM4991wAttxySy677DK+8Y1v1KLUlWruxfbzn/+cAQMGsN566zF79uz3LfPss8+yySabGG6kTq6a1/cNN9xAfX194996663H3Llza1D1ipp7rz3ooIN49NFHefjhh9l111254IILAFi+fDmjRo3iiiuu4LHHHmPGjBl07969FmV3OQaCNTB06FC23HLLFdo222yzxst//etfiQgAtt12W/bdd99O+cRs7sU2cOBAJk+ezNChQ5td5tRTT+WQQw7piPIkrYFqXt8nnHACc+fOZe7cuVx33XX07duX+vr6Dqy2ec291w4bNoxu3boBMHjwYBYtWgTAnXfeyV577cXee+8NwFZbbcX666/fsQV3Ud1qXcDa6Nvf/jbXXnstm2++OXfffXety1mloUOHsmDBghXadt999xbnv+WWW+jbty8bb7xxO1cmaU1V+/puMGHCBEaMGNFOVbWtq6++muOPPx6AJ598kohg+PDhLFmyhBEjRnDGGWfUuMKuwRGCdnD++eezcOFCTjjhBH70ox/Vupw29cYbb3DRRRdx9tln17qUtV5zQ73HH39843Bunz59OsW3t2pcfPHFDBgwgIEDBzJy5Ei37XZiN954IyNHjqx1Gat0/vnn061bN0444QSg2GRw3333ccMNN3Dfffdx8803M3369BpX2TUYCNrRCSecwC9+8Ytal9Gmxo4dy9e//nU22WSTWpey1mtuqPfGG29sHNI99thjOeaYY2pUXfUWL17MZZddxuzZs3n00Ud55513mDhxYq3LUjNmzZpFjx49VgijndE111zDbbfdxg033NC4ebauro6hQ4ey9dZb06NHDw499FAefPDBGlfaNRgI2tj8+fMbL0+ZMoX+/fvXsJq2N2vWLM444wz69OnDJZdcwve+9721bhSks2huu2mDzGTSpEld4htcpeXLl/O3v/2N5cuXs2zZMnbYYYdal6RmTJw4sdM/t6ZOncr3v/99br31Vnr06NHYPnz4cB555BGWLVvG8uXLueeee9hjjz1qWGnX4T4Ea2DkyJHMmDGDl19+mbq6Os455xxuv/12nnjiCdZbbz122mknrrjiCgBeeOEFBg0axOuvv856663HJZdcwrx581bYCbErmDlzZuPlsWPHsskmm/DlL3+5hhWtm2bOnEmvXr3o169frUtptd69e/ONb3yDD37wg2y00UYMGzaMYcOG1bosNfHuu+8yadKkFV7rtdbce+0FF1zAW2+9xUEHHQQUOxZeccUV9OzZk1NPPZV9992XiODQQw/lsMMOq3EPugYDwRqYMGHC+9rGjBnT7Lzbbbdd416wnU1zL7Ytt9ySr3zlKyxZsoTDDjuM+vp67rjjjlqXqtKECRM6/Te4pl599VWmTJnC008/zRZbbMGnPvUprr/+ekaNGlXr0tZq1b6+7733XnbccUd23nnnGlf+nmreawFGjRrl82o1GAjU7IsN4Oijj17pcmPHjm2HarQqy5cvZ/LkycyZM6fWpVRl2rRp9O3bl2222QaAY445ht/85je+cbezal/fBxxwAPfff397lqROyn0IpC5m2rRp9O/fn7q6ulqXUpUPfvCD3H///SxbtozMZPr06a06/E1SxzAQSJ3UyJEjGTJkCE888QR1dXWMGzcO6Bo7fDVn//3357jjjmOfffZhzz335N133+Wkk06qdVmSSm4ykDqploZ6r7nmmo4tpA2dc845nHPOObUuQ1IzHCGQJElr1whBRBwMXAqsD1yVmRdWu44Pn35tm9fVVub8x+daNV9n7gO0vh+S3m9teX135n6sq+9Ra80IQUSsD1wOHALsAYyMCM9GIUlSK6w1gQDYD3gqM/+UmW8DE4Eja1yTJEldwtq0yaA3sLDi+iJg/xrVIgGde1gU1t2hUUnvF5lZ6xraREQcBxycmV8sr38W2D8zv9xkvpOAhmOddgOeaMeytgZebsf1dxT70XmsDX2AtaMfa0MfwH50Jh3Rh50yc5vmJqxNIwSLgR0rrteVbSvIzCuBKzuioIiYnZmDOuK22pP96DzWhj7A2tGPtaEPYD86k1r3YW3ah+B3QL+I6BsRGwAjgFtrXJMkSV3CWjNCkJnLI+LLwB0Uhx1enZmP1bgsSZK6hLUmEABk5u3A7bWuo0KHbJroAPaj81gb+gBrRz/Whj6A/ehMatqHtWanQkmStPrWpn0IJEnSajIQtIOIuDoiXoqIR2tdy5qIiA9ExAMR8fuIeCwiuuyv0kTE+hHxUETcVutaVldELIiIRyJibkTMrnU9qyMitoiImyLiDxHxeEQMqXVN1YqI3crHoOHv9Yj4Wq3rqlZEfL18XT8aERMi4gO1rml1RMRXyz481pUeh+Y+JyJiy4i4KyLml/97dmRNBoL2cQ1wcK2LaANvAZ/MzL2BeuDgiBhc25JW21eBx2tdRBv4RGbWd+HDqy4FpmZmf2BvuuBjkplPlI9BPfBhYBlwc22rqk5E9Ab+FRiUmQMpdsQeUduqqhcRA4F/ojhT7d7A4RGxS22rarVreP/nxJnA9MzsB0wvr3cYA0E7yMx7gaW1rmNNZeGN8mr38q/L7XQSEXXAYcBVta5lXRYRmwNDgXEAmfl2Zr5W06LW3IHAHzPzmVoXshq6ARtFRDegB/BcjetZHbsDszJzWWYuB+4BjqlxTa3SwufEkcD48vJ44KiOrMlAoJUqh9rnAi8Bd2XmrBqXtDouAc4A3q1xHWsqgTsjYk55xs2upi+wBPjvcvPNVRGxca2LWkMjgAm1LqJambkY+E/gWeB54M+ZeWdtq1otjwL/EBFbRUQP4FBWPEFdV9MrM58vL78A9OrIGzcQaKUy851yaLQO2K8cousyIuJw4KXMnFPrWtrAxzJzH4pf9DwlIobWuqAqdQP2AX6cmR8C/koHD4m2pfIEaEcAP691LdUqt00fSRHSdgA2johRta2qepn5OHARcCcwFZgLvFPLmtpKFocAduiIrIFArVIO7d5N19s34qPAERGxgOIXMD8ZEdfXtqTVU36rIzNfothmvV9tK6raImBRxSjTTRQBoas6BHgwM1+sdSGr4R+BpzNzSWb+HZgMfKTGNa2WzByXmR/OzKHAq8CTta5pDbwYEdsDlP9f6sgbNxCoRRGxTURsUV7eCDgI+ENNi6pSZn4rM+sysw/F8O6vMrPLfROKiI0jYtOGy8AwiuHSLiMzXwAWRsRuZdOBwLwalrSmRtIFNxeUngUGR0SPiAiKx6LL7eAJEBHblv8/SLH/wM9qW9EauRUYXV4eDUzpyBtfq85U2FlExATgAGDriFgEnJ2Z42pb1WrZHhgfEetThMdJmdllD9vr4noBNxfv3XQDfpaZU2tb0mr5CnBDOdz+J+DzNa5ntZSh7CDgn2tdy+rIzFkRcRPwILAceIiue6a/X0TEVsDfgVO6yo6qzX1OABcCkyJiDPAM8OkOrckzFUqSJDcZSJIkA4EkSTIQSJIkDASSJAkDgSRJwkAgSZIwEEhqIiI+ExGzI+KNiHg+In4ZER9rxXLZhX5pTlITBgJJjSLiVIofg/oexcmQPgj8F8V57zul8tf6JK0hA4EkoPHnic+lONvb5Mz8a2b+PTP/JzNPj4j9IuK3EfFaOXLwo/KMg0TEveVqfl+OLBxfth8eEXPLZX4TEXtV3N4+5a8e/iUifh4RN0bEeRXT/ykinoqIpRFxa0TsUDEtI+KUiJgPzI+IyyPiB036c2tEfL397jFp7WIgkNRgCPABih9Oas47wNeBrct5DwROBih/WAZg78zcJDNvjIgPAVdTnN53K+AnwK0RsWEZJG4GrgG2pPhNgKMbbigiPglcQHHq1u0pTuM6sUk9RwH7A3tQ/Hb8yIhYr1x+a4of8OnK57WXOpSBQFKDrYCXM3N5cxMzc05m3p+ZyzNzAcUH/MdXsr6TgJ9k5qzyZ7THA28Bg8u/bsBl5SjEZOCBimVPAK7OzAcz8y3gW8CQiOhTMc8Fmbk0M/+WmQ8Af6YIKVD8kNWMLvpLhFJNGAgkNXiF4odWmt0mHxG7RsRtEfFCRLxOsZ/B1itZ307AaeXmgtci4jVgR2CH8m9xrvhjKgsrLu9AMSoAQGa+UdbXu4X5oRglaPgly1HAdSupTVITBgJJDX5L8Q3+qBam/5ji56/7ZeZmwL8BsZL1LQTOz8wtKv56ZOYE4Hmgd/nTuw12rLj8HEWgABp/XXArYHHFPE1/me164MiI2BvYHbhlJbVJasJAIAmAzPwz8O/A5RFxVET0iIjuEXFIRHwf2BR4HXgjIvoDX2qyiheBnSuu/xT4l4jYPwobR8RhEbEpRfh4B/hyRHSLiCOB/SqWnQB8PiLqI2JDitGIWeWmipbqXwT8jmJk4BeZ+bfVvzekdY+BQFKjzPwBcCrwHWAJxbf8L1N82/4G8BngLxQf9jc2WXwsML7cPPDpzJwN/BPwI+BV4CngxPJ23gaOAcYAr1EM8d9GMUJBZk4DzgJ+QTGa8H8o9gtYlfHAnri5QKparLgJT5JqIyJmAVdk5n+vwTqGUmw62Cl9c5Oq4giBpJqIiI9HxHblJoPRwF7A1DVYX3fgq8BVhgGpep7hS1Kt7AZMAjYG/gQcl5nPr86KImJ3YDbwe+DzbVahtA5xk4EkSXKTgSRJMhBIkiQMBJIkCQOBJEnCQCBJkjAQSJIk4P8DG0/B0ArakdAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "counts = pm_filtered.Category.value_counts().sort_index()\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "splot=sns.barplot(x=counts.index,y=counts, color='tab:blue')\n",
    "for p in splot.patches:\n",
    "    splot.annotate(format(p.get_height(), '.0f'), \n",
    "                   (p.get_x() + p.get_width() / 2., p.get_height()), \n",
    "                   ha = 'center', va = 'center', \n",
    "                   xytext = (0, 9), \n",
    "                   textcoords = 'offset points')\n",
    "plt.ylim(0,10500)\n",
    "plt.title('PubMed Article Dataset Class Distribution', size=16)\n",
    "plt.xlabel('Category', size=12)\n",
    "plt.ylabel('Count', size=12);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a91592a-d12c-40a8-bf61-a7ba78380ac9",
   "metadata": {},
   "source": [
    "## Fetch abstracts (don't need to run unless new input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6e25ec86-73fd-4d78-8e80-6dc339bf922a",
   "metadata": {},
   "outputs": [],
   "source": [
    "### run if you don't have efetch installed already\n",
    "# !sh -c \"$(curl -fsSL ftp://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)\"\n",
    "\n",
    "### this removes the efetch output file if present\n",
    "# if os.path.isfile(efetch_fn):\n",
    "#     os.remove(efetch_fn)\n",
    "\n",
    "if not os.path.isfile(efetch_fn):\n",
    "    tmp = ','.join([str(i) for i in pm_filtered.PMID.tolist()])\n",
    "    !efetch -db pubmed -id {tmp} -format xml > {efetch_fn}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5eaedca8-4a52-46d1-89d4-a81a62bc5f64",
   "metadata": {},
   "source": [
    "## Parse efetch output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "34e792c4-7afe-48d5-a7a2-bfd0c083ac97",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(efetch_fn) as f:\n",
    "    soup = BeautifulSoup(f, 'lxml-xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1f996019-0b66-4a32-8938-00023db66b3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "284 response from efetch but no abstract\n",
      "291 failed total\n"
     ]
    }
   ],
   "source": [
    "abstracts = {}\n",
    "failed = []\n",
    "\n",
    "for node in soup('PubmedArticle'):\n",
    "    pmid = int(node.find('PMID').text)\n",
    "    if node('AbstractText'):\n",
    "        abstracts[pmid] = node.find('AbstractText').text\n",
    "    else:\n",
    "        failed.append(pmid)\n",
    "\n",
    "print(len(failed), 'response from efetch but no abstract')\n",
    "\n",
    "for pmid in pm_filtered.PMID.tolist():\n",
    "    if pmid not in abstracts and pmid not in failed:\n",
    "        failed.append(pmid)\n",
    "\n",
    "print(len(failed), 'failed total')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9e91e77d-f11f-469a-a10b-2761e39ead75",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25553339</td>\n",
       "      <td>Cerebellar ataxia is a progressive neuro-degen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>26663098</td>\n",
       "      <td>Facial analysis systems are becoming available...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27014455</td>\n",
       "      <td>Patients with Williams-Beuren Syndrome can be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27112773</td>\n",
       "      <td>The genetic basis of numerous intellectual dis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27356087</td>\n",
       "      <td>We report four individuals from two unrelated ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13702</th>\n",
       "      <td>34231311</td>\n",
       "      <td>Deep learning (DL) has shown rapid advancement...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13703</th>\n",
       "      <td>34231533</td>\n",
       "      <td>Systemic retinal biomarkers are biomarkers ide...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13704</th>\n",
       "      <td>34233515</td>\n",
       "      <td>In the last few years, artificial intelligence...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13705</th>\n",
       "      <td>34234854</td>\n",
       "      <td>Despite the significant progress in diagnosis ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13706</th>\n",
       "      <td>34235035</td>\n",
       "      <td>Machine learning (ML) is a set of models and m...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13707 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PMID                                           Abstract\n",
       "0      25553339  Cerebellar ataxia is a progressive neuro-degen...\n",
       "1      26663098  Facial analysis systems are becoming available...\n",
       "2      27014455  Patients with Williams-Beuren Syndrome can be ...\n",
       "3      27112773  The genetic basis of numerous intellectual dis...\n",
       "4      27356087  We report four individuals from two unrelated ...\n",
       "...         ...                                                ...\n",
       "13702  34231311  Deep learning (DL) has shown rapid advancement...\n",
       "13703  34231533  Systemic retinal biomarkers are biomarkers ide...\n",
       "13704  34233515  In the last few years, artificial intelligence...\n",
       "13705  34234854  Despite the significant progress in diagnosis ...\n",
       "13706  34235035  Machine learning (ML) is a set of models and m...\n",
       "\n",
       "[13707 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pm_abstracts = pd.DataFrame(list(abstracts.items()), columns=['PMID', 'Abstract'])\n",
    "pm_abstracts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a72e3c22-b22e-4446-95c6-46fac69a1131",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Drop invalid abstracts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e42b400d-e002-4c77-a7a2-cbdf39ad1cd5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(13701, 2)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drop = ['A correction to this article has been published and is linked from the HTML and PDF versions of this paper. The error has not been fixed in the paper.',\n",
    "        'An amendment to this paper has been published and can be accessed via a link at the top of the paper.']\n",
    "print(pm_abstracts[pm_abstracts.Abstract.isin(drop)].shape)\n",
    "pm_abstracts.drop(pm_abstracts[pm_abstracts.Abstract.isin(drop)].index, inplace=True)\n",
    "pm_abstracts.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccab9907-8378-4719-8f96-c6c8cc260d9e",
   "metadata": {},
   "source": [
    "## Save files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "20a87c23-7bab-4353-ac50-5265f0cb09c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27088280</td>\n",
       "      <td>Microtia in Cornelia de Lange syndrome: a case...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>32529320</td>\n",
       "      <td>Further delineation of the neurodevelopmental ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>Not in Pubmed</td>\n",
       "      <td>DeepGestalt analysis of the SETD5-associated i...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>Not in Pubmed</td>\n",
       "      <td>A case of Noonan syndrome diagnosed using the ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>Not in Pubmed</td>\n",
       "      <td>Integrated facial analysis and targeted sequen...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13952</th>\n",
       "      <td>34126753</td>\n",
       "      <td>Promise and Frustration: Machine Learning in C...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13959</th>\n",
       "      <td>34138986</td>\n",
       "      <td>Putting the Pieces Together: Deep Learning for...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13963</th>\n",
       "      <td>34154724</td>\n",
       "      <td>Reducing Blindness Resulting from Retinopathy ...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13993</th>\n",
       "      <td>34224062</td>\n",
       "      <td>Letter to the Editor: Quantitative evaluation ...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14001</th>\n",
       "      <td>Not in Pubmed</td>\n",
       "      <td>Down Syndrome Face Recognition: A Review</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>301 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                PMID                                              Title  \\\n",
       "3           27088280  Microtia in Cornelia de Lange syndrome: a case...   \n",
       "67          32529320  Further delineation of the neurodevelopmental ...   \n",
       "130    Not in Pubmed  DeepGestalt analysis of the SETD5-associated i...   \n",
       "131    Not in Pubmed  A case of Noonan syndrome diagnosed using the ...   \n",
       "132    Not in Pubmed  Integrated facial analysis and targeted sequen...   \n",
       "...              ...                                                ...   \n",
       "13952       34126753  Promise and Frustration: Machine Learning in C...   \n",
       "13959       34138986  Putting the Pieces Together: Deep Learning for...   \n",
       "13963       34154724  Reducing Blindness Resulting from Retinopathy ...   \n",
       "13993       34224062  Letter to the Editor: Quantitative evaluation ...   \n",
       "14001  Not in Pubmed           Down Syndrome Face Recognition: A Review   \n",
       "\n",
       "       Category  \n",
       "3             1  \n",
       "67            1  \n",
       "130           1  \n",
       "131           1  \n",
       "132           1  \n",
       "...         ...  \n",
       "13952        10  \n",
       "13959        10  \n",
       "13963        10  \n",
       "13993        10  \n",
       "14001        10  \n",
       "\n",
       "[301 rows x 3 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp = pm_abstracts.PMID.tolist()\n",
    "failed = []\n",
    "for n, row in orig.iterrows():\n",
    "    if row.PMID not in tmp:\n",
    "        failed.append(row)\n",
    "\n",
    "failed = pd.DataFrame(failed)\n",
    "\n",
    "if os.path.isfile(failed_fn):\n",
    "    os.remove(failed_fn)\n",
    "\n",
    "failed.to_pickle(failed_fn)\n",
    "failed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "11d4e7ee-6210-46d2-afb8-a04fae454e8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[33928255, 34031651, 31610898, 30951265, 34013251]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(13696, 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(pm_abstracts[pm_abstracts.Abstract.duplicated()].PMID.tolist())\n",
    "pm_abstracts.drop(pm_abstracts[pm_abstracts.Abstract.duplicated()].index, inplace=True)\n",
    "pm_abstracts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1d4eac6a-ef68-4b58-8dda-508e9544095d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PMID</th>\n",
       "      <th>Category</th>\n",
       "      <th>Abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25553339</td>\n",
       "      <td>1</td>\n",
       "      <td>Cerebellar ataxia is a progressive neuro-degen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>26663098</td>\n",
       "      <td>1</td>\n",
       "      <td>Facial analysis systems are becoming available...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27014455</td>\n",
       "      <td>1</td>\n",
       "      <td>Patients with Williams-Beuren Syndrome can be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>27112773</td>\n",
       "      <td>1</td>\n",
       "      <td>The genetic basis of numerous intellectual dis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>27356087</td>\n",
       "      <td>1</td>\n",
       "      <td>We report four individuals from two unrelated ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13691</th>\n",
       "      <td>34231311</td>\n",
       "      <td>10</td>\n",
       "      <td>Deep learning (DL) has shown rapid advancement...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13692</th>\n",
       "      <td>34231533</td>\n",
       "      <td>10</td>\n",
       "      <td>Systemic retinal biomarkers are biomarkers ide...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13693</th>\n",
       "      <td>34233515</td>\n",
       "      <td>10</td>\n",
       "      <td>In the last few years, artificial intelligence...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13694</th>\n",
       "      <td>34234854</td>\n",
       "      <td>10</td>\n",
       "      <td>Despite the significant progress in diagnosis ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13695</th>\n",
       "      <td>34235035</td>\n",
       "      <td>10</td>\n",
       "      <td>Machine learning (ML) is a set of models and m...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13696 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PMID  Category                                           Abstract\n",
       "0      25553339         1  Cerebellar ataxia is a progressive neuro-degen...\n",
       "1      26663098         1  Facial analysis systems are becoming available...\n",
       "2      27014455         1  Patients with Williams-Beuren Syndrome can be ...\n",
       "3      27112773         1  The genetic basis of numerous intellectual dis...\n",
       "4      27356087         1  We report four individuals from two unrelated ...\n",
       "...         ...       ...                                                ...\n",
       "13691  34231311        10  Deep learning (DL) has shown rapid advancement...\n",
       "13692  34231533        10  Systemic retinal biomarkers are biomarkers ide...\n",
       "13693  34233515        10  In the last few years, artificial intelligence...\n",
       "13694  34234854        10  Despite the significant progress in diagnosis ...\n",
       "13695  34235035        10  Machine learning (ML) is a set of models and m...\n",
       "\n",
       "[13696 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pm_data = pm_filtered.merge(pm_abstracts, on='PMID')\n",
    "\n",
    "if os.path.isfile(data_fn):\n",
    "    os.remove(data_fn)\n",
    "\n",
    "pm_data.to_pickle(data_fn)\n",
    "pm_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0c16651-4714-4cd2-801f-818daf1908cd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}