HPO extraction tool.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read dictionary\n",
    "The first 3 lines are comments\n",
    "- Downloaded from: https://hpo.jax.org/app/download/annotation \n",
    "- http://compbio.charite.de/jenkins/job/hpo.annotations/lastStableBuild/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nr of HPO codes: 199197\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\levrex\\Anaconda3\\envs\\my_env\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3063: DtypeWarning: Columns (6,8,9) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#DatabaseID</th>\n",
       "      <th>DiseaseName</th>\n",
       "      <th>Qualifier</th>\n",
       "      <th>HPO_ID</th>\n",
       "      <th>Reference</th>\n",
       "      <th>Evidence</th>\n",
       "      <th>Onset</th>\n",
       "      <th>Frequency</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Modifier</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Biocuration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OMIM:210100</td>\n",
       "      <td>BETA-AMINOISOBUTYRIC ACID, URINARY EXCRETION OF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HP:0000007</td>\n",
       "      <td>OMIM:210100</td>\n",
       "      <td>IEA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I</td>\n",
       "      <td>HPO:iea[2009-02-17]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>OMIM:210100</td>\n",
       "      <td>BETA-AMINOISOBUTYRIC ACID, URINARY EXCRETION OF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HP:0003355</td>\n",
       "      <td>OMIM:210100</td>\n",
       "      <td>IEA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P</td>\n",
       "      <td>HPO:skoehler[2009-02-17]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>OMIM:163600</td>\n",
       "      <td>NIPPLES INVERTED</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HP:0000006</td>\n",
       "      <td>OMIM:163600</td>\n",
       "      <td>IEA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I</td>\n",
       "      <td>HPO:iea[2009-02-17]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OMIM:163600</td>\n",
       "      <td>NIPPLES INVERTED</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HP:0003186</td>\n",
       "      <td>OMIM:163600</td>\n",
       "      <td>IEA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P</td>\n",
       "      <td>HPO:iea[2009-02-17]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OMIM:615763</td>\n",
       "      <td>#615763 CORTICAL DYSPLASIA, COMPLEX, WITH OTHE...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HP:0002365</td>\n",
       "      <td>OMIM:615763</td>\n",
       "      <td>TAS</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HP:0040283</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P</td>\n",
       "      <td>HPO:skoehler[2014-08-24]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   #DatabaseID                                        DiseaseName Qualifier  \\\n",
       "0  OMIM:210100    BETA-AMINOISOBUTYRIC ACID, URINARY EXCRETION OF       NaN   \n",
       "1  OMIM:210100    BETA-AMINOISOBUTYRIC ACID, URINARY EXCRETION OF       NaN   \n",
       "2  OMIM:163600                                   NIPPLES INVERTED       NaN   \n",
       "3  OMIM:163600                                   NIPPLES INVERTED       NaN   \n",
       "4  OMIM:615763  #615763 CORTICAL DYSPLASIA, COMPLEX, WITH OTHE...       NaN   \n",
       "\n",
       "       HPO_ID    Reference Evidence Onset   Frequency  Sex Modifier Aspect  \\\n",
       "0  HP:0000007  OMIM:210100      IEA   NaN         NaN  NaN      NaN      I   \n",
       "1  HP:0003355  OMIM:210100      IEA   NaN         NaN  NaN      NaN      P   \n",
       "2  HP:0000006  OMIM:163600      IEA   NaN         NaN  NaN      NaN      I   \n",
       "3  HP:0003186  OMIM:163600      IEA   NaN         NaN  NaN      NaN      P   \n",
       "4  HP:0002365  OMIM:615763      TAS   NaN  HP:0040283  NaN      NaN      P   \n",
       "\n",
       "                Biocuration  \n",
       "0       HPO:iea[2009-02-17]  \n",
       "1  HPO:skoehler[2009-02-17]  \n",
       "2       HPO:iea[2009-02-17]  \n",
       "3       HPO:iea[2009-02-17]  \n",
       "4  HPO:skoehler[2014-08-24]  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hpo2 = pd.read_table(r'hpo/phenotype.hpoa', sep='\\t', header=4, encoding = \"ISO-8859-1\")\n",
    "print('Nr of HPO codes:', len(df_hpo2))\n",
    "df_hpo2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16950"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_hpo2['DiseaseName'].unique()) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.0"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hpo2['HPO_ID'].value_counts().median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([8.843e+03, 1.580e+02, 4.300e+01, 1.900e+01, 8.000e+00, 5.000e+00,\n",
       "        3.000e+00, 2.000e+00, 0.000e+00, 1.000e+00, 1.000e+00, 0.000e+00,\n",
       "        1.000e+00, 2.000e+00, 0.000e+00, 0.000e+00, 0.000e+00, 0.000e+00,\n",
       "        0.000e+00, 0.000e+00, 0.000e+00, 1.000e+00, 0.000e+00, 0.000e+00,\n",
       "        1.000e+00]),\n",
       " array([1.00000e+00, 1.52160e+02, 3.03320e+02, 4.54480e+02, 6.05640e+02,\n",
       "        7.56800e+02, 9.07960e+02, 1.05912e+03, 1.21028e+03, 1.36144e+03,\n",
       "        1.51260e+03, 1.66376e+03, 1.81492e+03, 1.96608e+03, 2.11724e+03,\n",
       "        2.26840e+03, 2.41956e+03, 2.57072e+03, 2.72188e+03, 2.87304e+03,\n",
       "        3.02420e+03, 3.17536e+03, 3.32652e+03, 3.47768e+03, 3.62884e+03,\n",
       "        3.78000e+03]),\n",
       " <a list of 25 Patch objects>)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAPZUlEQVR4nO3cf6zd9V3H8efLlgEbawZyIbVtbGcatRDdoMHOmf3DIh1bLP5BUpNJY0iaEKab0ZjiEjf/aMKMLkoUkrpNii4jzTZD44KO1C3GhICXASulq3QDoaPSO81c5x9ssLd/nA/zpL0/zuX2nnvo5/lIvjnf8z6f7znv82nv637v5/xIVSFJ6sNPrHQDkqTxMfQlqSOGviR1xNCXpI4Y+pLUkdUr3cBCLr/88tq4ceNKtyFJbyiPPfbYd6pq6sz6xIf+xo0bmZ6eXuk2JOkNJcl/zFZ3eUeSOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjoy8Z/IXYqNe760qPHP3fn+ZepEkiaDZ/qS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1ZKTQT/K7SY4keSrJ55JclOSyJA8leaZdXjo0/o4kx5McS3LDUP3aJIfbbXclyXI8KUnS7BYM/STrgN8BtlbV1cAqYCewBzhUVZuBQ+06Sba0268CtgN3J1nV7u4eYDewuW3bz+mzkSTNa9TlndXAxUlWA28GXgR2APvb7fuBm9r+DuD+qnq5qp4FjgPXJVkLrKmqh6uqgPuGjpEkjcGCoV9V3wb+FHgeOAn8T1V9Gbiyqk62MSeBK9oh64AXhu7iRKuta/tn1s+SZHeS6STTMzMzi3tGkqQ5jbK8cymDs/dNwE8Bb0nywfkOmaVW89TPLlbtq6qtVbV1ampqoRYlSSMaZXnnvcCzVTVTVT8Evgj8MvBSW7KhXZ5q408AG4aOX89gOehE2z+zLkkak1FC/3lgW5I3t3fbXA8cBQ4Cu9qYXcADbf8gsDPJhUk2MXjB9tG2BHQ6ybZ2P7cMHSNJGoPVCw2oqkeSfB74GvAK8DiwD7gEOJDkVga/GG5u448kOQA83cbfXlWvtru7DbgXuBh4sG2SpDFZMPQBqupjwMfOKL/M4Kx/tvF7gb2z1KeBqxfZoyTpHPETuZLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpIyOFfpK3Jfl8km8kOZrkXUkuS/JQkmfa5aVD4+9IcjzJsSQ3DNWvTXK43XZXkizHk5IkzW7UM/2/AP6xqn4O+EXgKLAHOFRVm4FD7TpJtgA7gauA7cDdSVa1+7kH2A1sbtv2c/Q8JEkjWDD0k6wB3gN8GqCqflBV3wV2APvbsP3ATW1/B3B/Vb1cVc8Cx4HrkqwF1lTVw1VVwH1Dx0iSxmCUM/23AzPA3yR5PMmnkrwFuLKqTgK0yyva+HXAC0PHn2i1dW3/zPpZkuxOMp1kemZmZlFPSJI0t1FCfzVwDXBPVb0T+F/aUs4cZlunr3nqZxer9lXV1qraOjU1NUKLkqRRjBL6J4ATVfVIu/55Br8EXmpLNrTLU0PjNwwdvx54sdXXz1KXJI3JgqFfVf8JvJDkZ1vpeuBp4CCwq9V2AQ+0/YPAziQXJtnE4AXbR9sS0Okk29q7dm4ZOkaSNAarRxz328Bnk7wJ+BbwWwx+YRxIcivwPHAzQFUdSXKAwS+GV4Dbq+rVdj+3AfcCFwMPtk2SNCYjhX5VPQFsneWm6+cYvxfYO0t9Grh6MQ1Kks4dP5ErSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjoycugnWZXk8ST/0K5fluShJM+0y0uHxt6R5HiSY0luGKpfm+Rwu+2uJDm3T0eSNJ/FnOl/GDg6dH0PcKiqNgOH2nWSbAF2AlcB24G7k6xqx9wD7AY2t237krqXJC3KSKGfZD3wfuBTQ+UdwP62vx+4aah+f1W9XFXPAseB65KsBdZU1cNVVcB9Q8dIksZg1DP9Pwf+APjRUO3KqjoJ0C6vaPV1wAtD40602rq2f2b9LEl2J5lOMj0zMzNii5KkhSwY+kk+AJyqqsdGvM/Z1ulrnvrZxap9VbW1qrZOTU2N+LCSpIWsHmHMu4FfS3IjcBGwJsnfAS8lWVtVJ9vSzak2/gSwYej49cCLrb5+lrokaUwWPNOvqjuqan1VbWTwAu0/V9UHgYPArjZsF/BA2z8I7ExyYZJNDF6wfbQtAZ1Osq29a+eWoWMkSWMwypn+XO4EDiS5FXgeuBmgqo4kOQA8DbwC3F5Vr7ZjbgPuBS4GHmybJGlMFhX6VfVV4Ktt/7+A6+cYtxfYO0t9Grh6sU1Kks4NP5ErSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjqyYOgn2ZDkK0mOJjmS5MOtflmSh5I80y4vHTrmjiTHkxxLcsNQ/dokh9ttdyXJ8jwtSdJsRjnTfwX4var6eWAbcHuSLcAe4FBVbQYOteu023YCVwHbgbuTrGr3dQ+wG9jctu3n8LlIkhawYOhX1cmq+lrbPw0cBdYBO4D9bdh+4Ka2vwO4v6perqpngePAdUnWAmuq6uGqKuC+oWMkSWOwqDX9JBuBdwKPAFdW1UkY/GIArmjD1gEvDB12otXWtf0z67M9zu4k00mmZ2ZmFtOiJGkeI4d+kkuALwAfqarvzTd0llrNUz+7WLWvqrZW1dapqalRW5QkLWCk0E9yAYPA/2xVfbGVX2pLNrTLU61+AtgwdPh64MVWXz9LXZI0JqO8eyfAp4GjVfXJoZsOArva/i7ggaH6ziQXJtnE4AXbR9sS0Okk29p93jJ0jCRpDFaPMObdwG8Ch5M80Wp/CNwJHEhyK/A8cDNAVR1JcgB4msE7f26vqlfbcbcB9wIXAw+2TZI0JguGflX9K7OvxwNcP8cxe4G9s9SngasX06Ak6dzxE7mS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdWb3SDUySjXu+tKjxz935/mXqRJKWx9jP9JNsT3IsyfEke8b9+JLUs7GGfpJVwF8B7wO2AL+RZMs4e5Ckno17eec64HhVfQsgyf3ADuDpMfdxTix2Oej1cAlJ0rk07tBfB7wwdP0E8EtnDkqyG9jdrn4/ybHX+XiXA995nceOw4L95RNj6mR2kz5/MPk92t/STHp/MLk9/vRsxXGHfmap1VmFqn3AviU/WDJdVVuXej/Lxf6WbtJ7tL+lmfT+4I3R47Bxv5B7AtgwdH098OKYe5Ckbo079P8N2JxkU5I3ATuBg2PuQZK6Ndblnap6JcmHgH8CVgGfqaojy/iQS14iWmb2t3ST3qP9Lc2k9wdvjB5/LFVnLalLks5Tfg2DJHXE0JekjpyXoT8pX/WQ5Lkkh5M8kWS61S5L8lCSZ9rlpUPj72g9H0tywzL19Jkkp5I8NVRbdE9Jrm3P7XiSu5LM9nbcc9Xfx5N8u83jE0luXMH+NiT5SpKjSY4k+XCrT8QcztPfRMxhkouSPJrkydbfH7f6RMzfAj1OxBwuWVWdVxuDF4i/CbwdeBPwJLBlhXp5Drj8jNqfAHva/h7gE21/S+v1QmBTew6rlqGn9wDXAE8tpSfgUeBdDD578SDwvmXs7+PA788ydiX6Wwtc0/bfCvx762Mi5nCe/iZiDtt9XdL2LwAeAbZNyvwt0ONEzOFSt/PxTP/HX/VQVT8AXvuqh0mxA9jf9vcDNw3V76+ql6vqWeA4g+dyTlXVvwD/vZSekqwF1lTVwzX4n33f0DHL0d9cVqK/k1X1tbZ/GjjK4JPmEzGH8/Q3l3H3V1X1/Xb1grYVEzJ/C/Q4l7H3uBTnY+jP9lUP8/2nX04FfDnJYxl8tQTAlVV1EgY/oMAVrb6SfS+2p3Vt/8z6cvpQkq+35Z/X/vRf0f6SbATeyeBMcOLm8Iz+YELmMMmqJE8Ap4CHqmri5m+OHmFC5nApzsfQH+mrHsbk3VV1DYNvFb09yXvmGTtJfb9mrp7G3es9wM8A7wBOAn/W6ivWX5JLgC8AH6mq7803dI5elrXHWfqbmDmsqler6h0MPpF/XZKr5xm+IvM3R48TM4dLcT6G/sR81UNVvdguTwF/z2C55qX2Zx/t8lQbvpJ9L7anE23/zPqyqKqX2g/hj4C/5v+XvVakvyQXMAjUz1bVF1t5YuZwtv4mbQ5bT98FvgpsZ4Lmb64eJ3EOX4/zMfQn4qsekrwlyVtf2wd+FXiq9bKrDdsFPND2DwI7k1yYZBOwmcGLQOOwqJ7an9+nk2xr70a4ZeiYc+61MGh+ncE8rkh/7f4+DRytqk8O3TQRczhXf5Myh0mmkryt7V8MvBf4BhMyf/P1OClzuGQr/UrycmzAjQzetfBN4KMr1MPbGbyi/yRw5LU+gJ8EDgHPtMvLho75aOv5GMv0Kj/wOQZ/mv6QwZnIra+nJ2Arg//03wT+kvbp7mXq72+Bw8DXGfyArV3B/n6FwZ/oXweeaNuNkzKH8/Q3EXMI/ALweOvjKeCPXu/PxTL+G8/V40TM4VI3v4ZBkjpyPi7vSJLmYOhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0JekjvwfBCBiR0gsvrcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.hist(df_hpo2['HPO_ID'].value_counts(), bins=25)#.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx\n",
    "import obonet\n",
    "import DeepPhenotyping_functions as func\n",
    "\n",
    "url = 'phenopy_mod/.phenopy/data/hp.obo'\n",
    "graph = obonet.read_obo(url)\n",
    "hpo_to_umls, umls_to_hpo = func.HPO_to_UMLS(graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'High-frequency hearing impairment',\n",
       " 'alt_id': ['HP:0008522', 'HP:0008584', 'HP:0008597'],\n",
       " 'def': '\"A type of hearing impairment affecting primarily the higher frequencies of sound (3,000 to 6,000 Hz).\" [HPO:probinson]',\n",
       " 'comment': 'High frequency hearing impairment often involves loss of ability to hear consonants such as s, f, t, and z, even though vowels can be heard normally.',\n",
       " 'synonym': ['\"Hearing loss, high-frequency\" EXACT layperson []',\n",
       "  '\"High frequency hearing loss\" EXACT layperson []',\n",
       "  '\"High-frequency deafness\" EXACT []',\n",
       "  '\"Progressive high frequency hearing loss\" EXACT layperson []',\n",
       "  '\"Progressive high-frequency hearing loss\" EXACT []'],\n",
       " 'xref': ['MSH:D006316',\n",
       "  'SNOMEDCT_US:232326009',\n",
       "  'SNOMEDCT_US:48758008',\n",
       "  'UMLS:C0018780'],\n",
       " 'is_a': ['HP:0000365'],\n",
       " 'created_by': 'peter',\n",
       " 'creation_date': '2008-03-25T05:03:00Z'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#graph.get('HP:0000002') # HP:0000002\n",
    "graph.nodes['HP:0005101']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## read genes to calculate occurence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#Format: HPO-id</th>\n",
       "      <th>HPO label</th>\n",
       "      <th>entrez-gene-id</th>\n",
       "      <th>entrez-gene-symbol</th>\n",
       "      <th>Additional Info from G-D source</th>\n",
       "      <th>G-D source</th>\n",
       "      <th>disease-ID for link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HP:0000002</td>\n",
       "      <td>Abnormality of body height</td>\n",
       "      <td>26229</td>\n",
       "      <td>B3GAT3</td>\n",
       "      <td>-</td>\n",
       "      <td>mim2gene</td>\n",
       "      <td>OMIM:245600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HP:0000002</td>\n",
       "      <td>Abnormality of body height</td>\n",
       "      <td>2892</td>\n",
       "      <td>GRIA3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>orphadata</td>\n",
       "      <td>ORPHA:364028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HP:0000002</td>\n",
       "      <td>Abnormality of body height</td>\n",
       "      <td>64374</td>\n",
       "      <td>SIL1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>orphadata</td>\n",
       "      <td>ORPHA:559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HP:0000002</td>\n",
       "      <td>Abnormality of body height</td>\n",
       "      <td>10588</td>\n",
       "      <td>MTHFS</td>\n",
       "      <td>-</td>\n",
       "      <td>mim2gene</td>\n",
       "      <td>OMIM:618367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HP:0000002</td>\n",
       "      <td>Abnormality of body height</td>\n",
       "      <td>51633</td>\n",
       "      <td>OTUD6B</td>\n",
       "      <td>-</td>\n",
       "      <td>mim2gene</td>\n",
       "      <td>OMIM:617452</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  #Format: HPO-id                   HPO label  entrez-gene-id  \\\n",
       "0      HP:0000002  Abnormality of body height           26229   \n",
       "1      HP:0000002  Abnormality of body height            2892   \n",
       "2      HP:0000002  Abnormality of body height           64374   \n",
       "3      HP:0000002  Abnormality of body height           10588   \n",
       "4      HP:0000002  Abnormality of body height           51633   \n",
       "\n",
       "  entrez-gene-symbol Additional Info from G-D source G-D source  \\\n",
       "0             B3GAT3                               -   mim2gene   \n",
       "1              GRIA3                             NaN  orphadata   \n",
       "2               SIL1                             NaN  orphadata   \n",
       "3              MTHFS                               -   mim2gene   \n",
       "4             OTUD6B                               -   mim2gene   \n",
       "\n",
       "  disease-ID for link  \n",
       "0         OMIM:245600  \n",
       "1        ORPHA:364028  \n",
       "2           ORPHA:559  \n",
       "3         OMIM:618367  \n",
       "4         OMIM:617452  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df_hpo = pd.read_table(r'hpo/util/annotation/phenotype_to_genes.txt', sep='\\t|<tab>', engine='python') # phenotype_to_genes.txt\n",
    "df_hpo.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nr of genes:  841750\n"
     ]
    }
   ],
   "source": [
    "print('Nr of genes: ', len(df_hpo))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['-', nan, 'question', 'susceptibility', 'susceptibility; question',\n",
       "       'susceptibility; modifier', 'susceptibility; modifier; question',\n",
       "       'susceptibility; QTL 1', 'susceptibility; somatic'], dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hpo['Additional Info from G-D source'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = sns.boxplot(x=\"score\", y=\"score_ent\", data=df3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([8.442e+03, 3.010e+02, 9.300e+01, 5.100e+01, 4.200e+01, 2.900e+01,\n",
       "        2.400e+01, 1.600e+01, 1.100e+01, 1.500e+01, 4.000e+00, 8.000e+00,\n",
       "        0.000e+00, 6.000e+00, 7.000e+00, 2.000e+00, 2.000e+00, 1.000e+00,\n",
       "        2.000e+00, 3.000e+00, 0.000e+00, 0.000e+00, 1.000e+00, 0.000e+00,\n",
       "        1.000e+00]),\n",
       " array([1.00000e+00, 2.75040e+02, 5.49080e+02, 8.23120e+02, 1.09716e+03,\n",
       "        1.37120e+03, 1.64524e+03, 1.91928e+03, 2.19332e+03, 2.46736e+03,\n",
       "        2.74140e+03, 3.01544e+03, 3.28948e+03, 3.56352e+03, 3.83756e+03,\n",
       "        4.11160e+03, 4.38564e+03, 4.65968e+03, 4.93372e+03, 5.20776e+03,\n",
       "        5.48180e+03, 5.75584e+03, 6.02988e+03, 6.30392e+03, 6.57796e+03,\n",
       "        6.85200e+03]),\n",
       " <a list of 25 Patch objects>)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD4CAYAAAAKA1qZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAVS0lEQVR4nO3dcYyU953f8ffnICbYCTU+L4juokIklNa2GideUVJX0TXkziQ5Bf9jiUipUeWKynLbpK10gp7U6P5A8lXV6Wq1toqSnLEuF8T5khrl5HMQd1HVyjLZ2M7Z2KYmwTVbCGxdpeHuJHrmvv1jfpbnloGdxd6dhef9kkbPM9/n95v5Llr02fk9M/OkqpAkddMvjLoBSdLoGAKS1GGGgCR1mCEgSR1mCEhShy0fdQNzufXWW2vDhg2jbkOSrhm33norzzzzzDNVtW2usUs+BDZs2MDU1NSo25Cka0qSW4cZ53KQJHWYISBJHWYISFKHGQKS1GGGgCR1mCEgSR1mCEhShxkCktRhhoAkddiS/8Twe7Fh9x/Oa/wbD39+gTqRpKXJVwKS1GGGgCR1mCEgSR1mCEhShxkCktRhhoAkddhQIZDkXyY5luTlJN9K8sEktyQ5nOT1tl3dN35PkhNJjie5p69+V5KX2rFHkmQhfihJ0nDmDIEk48C/ACar6g5gGbAD2A0cqapNwJF2nyS3teO3A9uAR5Msaw/3GLAL2NRuc176TJK0cIZdDloOrEyyHLgROA1sB/a34/uBe9v+duBAVV2oqpPACWBzknXAqqp6tqoKeKJvjiRpBOYMgar6X8C/B94EzgD/t6q+B6ytqjNtzBlgTZsyDpzqe4jpVhtv+7PrkqQRGWY5aDW9v+43An8TuCnJl640ZUCtrlAf9Jy7kkwlmZqZmZmrRUnSVRpmOegzwMmqmqmqvwS+Dfx94Gxb4qFtz7Xx08D6vvkT9JaPptv+7PolqmpfVU1W1eTY2Nh8fh5J0jwMEwJvAluS3NjezbMVeBU4BOxsY3YCT7X9Q8COJCuSbKR3AvhoWzI6n2RLe5z7++ZIkkZgzm8RrarnkjwJPA+8DbwA7AM+BBxM8gC9oLivjT+W5CDwShv/UFVdbA/3IPA4sBJ4ut0kSSMy1FdJV9VXga/OKl+g96pg0Pi9wN4B9Sngjnn2KElaIH5iWJI6zBCQpA4zBCSpwwwBSeowQ0CSOswQkKQOMwQkqcMMAUnqMENAkjrMEJCkDjMEJKnDDAFJ6jBDQJI6zBCQpA4zBCSpwwwBSeqwYS40/9EkL/bdfp7kK0luSXI4yettu7pvzp4kJ5IcT3JPX/2uJC+1Y4+0y0xKkkZkzhCoquNVdWdV3QncBfwF8B1gN3CkqjYBR9p9ktwG7ABuB7YBjyZZ1h7uMWAXvesOb2rHJUkjMt/loK3Aj6vqfwLbgf2tvh+4t+1vBw5U1YWqOgmcADYnWQesqqpnq6qAJ/rmSJJGYL4hsAP4VttfW1VnANp2TauPA6f65ky32njbn12/RJJdSaaSTM3MzMyzRUnSsIYOgSQ3AF8Afn+uoQNqdYX6pcWqfVU1WVWTY2Njw7YoSZqn+bwS+CzwfFWdbffPtiUe2vZcq08D6/vmTQCnW31iQF2SNCLzCYEv8u5SEMAhYGfb3wk81VffkWRFko30TgAfbUtG55Nsae8Kur9vjiRpBJYPMyjJjcAvA/+0r/wwcDDJA8CbwH0AVXUsyUHgFeBt4KGqutjmPAg8DqwEnm43SdKIDBUCVfUXwC/Oqr1F791Cg8bvBfYOqE8Bd8y/TUnSQvATw5LUYYaAJHWYISBJHWYISFKHGQKS1GGGgCR1mCEgSR1mCEhShxkCktRhhoAkdZghIEkdZghIUocZApLUYYaAJHWYISBJHTZUCCS5OcmTSV5L8mqSTya5JcnhJK+37eq+8XuSnEhyPMk9ffW7krzUjj3SrjAmSRqRYV8J/Afgj6rqbwMfA14FdgNHqmoTcKTdJ8ltwA7gdmAb8GiSZe1xHgN20bvk5KZ2XJI0InOGQJJVwKeArwNU1f+rqp8B24H9bdh+4N62vx04UFUXquokcALY3C5Gv6qqnq2qAp7omyNJGoFhXgl8BJgBfifJC0m+luQmYG27eDxtu6aNHwdO9c2fbrXxtj+7LkkakWFCYDnwCeCxqvo48Oe0pZ/LGLTOX1eoX/oAya4kU0mmZmZmhmhRknQ1hgmBaWC6qp5r95+kFwpn2xIPbXuub/z6vvkTwOlWnxhQv0RV7auqyaqaHBsbG/ZnkSTN05whUFU/BU4l+WgrbQVeAQ4BO1ttJ/BU2z8E7EiyIslGeieAj7Ylo/NJtrR3Bd3fN0eSNALLhxz3z4FvJrkB+Anwj+kFyMEkDwBvAvcBVNWxJAfpBcXbwENVdbE9zoPA48BK4Ol2kySNyFAhUFUvApMDDm29zPi9wN4B9Sngjvk0KElaOH5iWJI6zBCQpA4zBCSpwwwBSeowQ0CSOswQkKQOMwQkqcMMAUnqMENAkjrMEJCkDjMEJKnDDAFJ6jBDQJI6zBCQpA4zBCSpwwwBSeqwoUIgyRtJXkryYpKpVrslyeEkr7ft6r7xe5KcSHI8yT199bva45xI8ki7zKQkaUTm80rgH1bVnVX1zhXGdgNHqmoTcKTdJ8ltwA7gdmAb8GiSZW3OY8Auetcd3tSOS5JG5L0sB20H9rf9/cC9ffUDVXWhqk4CJ4DNSdYBq6rq2aoq4Im+OZKkERg2BAr4XpIfJtnVamur6gxA265p9XHgVN/c6VYbb/uz65dIsivJVJKpmZmZIVuUJM3XUBeaB+6uqtNJ1gCHk7x2hbGD1vnrCvVLi1X7gH0Ak5OTA8dIkt67oV4JVNXptj0HfAfYDJxtSzy07bk2fBpY3zd9Ajjd6hMD6pKkEZkzBJLclOTD7+wDvwK8DBwCdrZhO4Gn2v4hYEeSFUk20jsBfLQtGZ1PsqW9K+j+vjmSpBEYZjloLfCd9m7O5cDvVdUfJfkBcDDJA8CbwH0AVXUsyUHgFeBt4KGqutge60HgcWAl8HS7SZJGZM4QqKqfAB8bUH8L2HqZOXuBvQPqU8Ad829TkrQQ/MSwJHWYISBJHWYISFKHGQKS1GGGgCR1mCEgSR1mCEhShxkCktRhhoAkdZghIEkdZghIUocZApLUYYaAJHWYISBJHWYISFKHGQKS1GFDh0CSZUleSPLddv+WJIeTvN62q/vG7klyIsnxJPf01e9K8lI79ki7zKQkaUTm80rgy8Crffd3A0eqahNwpN0nyW3ADuB2YBvwaJJlbc5jwC561x3e1I5LkkZkqBBIMgF8HvhaX3k7sL/t7wfu7asfqKoLVXUSOAFsTrIOWFVVz1ZVAU/0zZEkjcCwrwR+G/g14K/6amur6gxA265p9XHgVN+46VYbb/uz65dIsivJVJKpmZmZIVuUJM3XnCGQ5FeBc1X1wyEfc9A6f12hfmmxal9VTVbV5NjY2JBPK0mar+VDjLkb+EKSzwEfBFYl+V3gbJJ1VXWmLfWca+OngfV98yeA060+MaAuSRqROV8JVNWeqpqoqg30Tvj+cVV9CTgE7GzDdgJPtf1DwI4kK5JspHcC+GhbMjqfZEt7V9D9fXMkSSMwzCuBy3kYOJjkAeBN4D6AqjqW5CDwCvA28FBVXWxzHgQeB1YCT7ebJGlE5hUCVfV94Ptt/y1g62XG7QX2DqhPAXfMt0lJ0sLwE8OS1GGGgCR1mCEgSR1mCEhShxkCktRhhoAkdZghIEkdZghIUocZApLUYYaAJHWYISBJHWYISFKHGQKS1GGGgCR1mCEgSR02zDWGP5jkaJIfJTmW5Dda/ZYkh5O83rar++bsSXIiyfEk9/TV70ryUjv2SLvCmCRpRIZ5JXAB+HRVfQy4E9iWZAuwGzhSVZuAI+0+SW6jdxnK24FtwKNJlrXHegzYRe+Sk5vacUnSiAxzjeGqqj9rdz/QbgVsB/a3+n7g3ra/HThQVReq6iRwAtjcLka/qqqeraoCnuibI0kagaHOCSRZluRF4BxwuKqeA9a2i8fTtmva8HHgVN/06VYbb/uz64Oeb1eSqSRTMzMz8/l5JEnzMFQIVNXFqroTmKD3V/2VrhM8aJ2/rlAf9Hz7qmqyqibHxsaGaVGSdBXm9e6gqvoZvQvNbwPOtiUe2vZcGzYNrO+bNgGcbvWJAXVJ0ogM8+6gsSQ3t/2VwGeA14BDwM42bCfwVNs/BOxIsiLJRnongI+2JaPzSba0dwXd3zdHkjQCy4cYsw7Y397h8wvAwar6bpJngYNJHgDeBO4DqKpjSQ4CrwBvAw9V1cX2WA8CjwMrgafbTZI0InOGQFX9KfDxAfW3gK2XmbMX2DugPgVc6XyCJGkR+YlhSeowQ0CSOswQkKQOMwQkqcMMAUnqMENAkjrMEJCkDjMEJKnDDAFJ6jBDQJI6zBCQpA4zBCSpwwwBSeowQ0CSOswQkKQOMwQkqcOGubzk+iR/kuTVJMeSfLnVb0lyOMnrbbu6b86eJCeSHE9yT1/9riQvtWOPtMtMSpJGZJhXAm8D/7qq/g6wBXgoyW3AbuBIVW0CjrT7tGM7gNvpXZD+0XZpSoDHgF30rju8qR2XJI3InCFQVWeq6vm2fx54FRgHtgP727D9wL1tfztwoKouVNVJ4ASwOck6YFVVPVtVBTzRN0eSNALzOieQZAO96w0/B6ytqjPQCwpgTRs2DpzqmzbdauNtf3Z90PPsSjKVZGpmZmY+LUqS5mHoEEjyIeAPgK9U1c+vNHRAra5Qv7RYta+qJqtqcmxsbNgWJUnzNFQIJPkAvQD4ZlV9u5XPtiUe2vZcq08D6/umTwCnW31iQF2SNCLDvDsowNeBV6vqt/oOHQJ2tv2dwFN99R1JViTZSO8E8NG2ZHQ+yZb2mPf3zZEkjcDyIcbcDfwj4KUkL7bavwEeBg4meQB4E7gPoKqOJTkIvELvnUUPVdXFNu9B4HFgJfB0u0mSRmTOEKiq/8bg9XyArZeZsxfYO6A+BdwxnwYlSQvHTwxLUocZApLUYYaAJHWYISBJHWYISFKHGQKS1GGGgCR1mCEgSR1mCEhShxkCktRhhoAkdZghIEkdZghIUocZApLUYYaAJHXYMFcW+0aSc0le7qvdkuRwktfbdnXfsT1JTiQ5nuSevvpdSV5qxx5pVxeTJI3QMK8EHge2zartBo5U1SbgSLtPktuAHcDtbc6jSZa1OY8Bu+hdbnLTgMeUJC2yOUOgqv4r8H9mlbcD+9v+fuDevvqBqrpQVSeBE8DmdiH6VVX1bFUV8ETfHEnSiFztOYG17cLxtO2aVh8HTvWNm2618bY/uz5Qkl1JppJMzczMXGWLkqS5vN8nhget89cV6gNV1b6qmqyqybGxsfetOUnSX3e1IXC2LfHQtudafRpY3zduAjjd6hMD6pKkEbraEDgE7Gz7O4Gn+uo7kqxIspHeCeCjbcnofJIt7V1B9/fNkSSNyPK5BiT5FvBLwK1JpoGvAg8DB5M8ALwJ3AdQVceSHAReAd4GHqqqi+2hHqT3TqOVwNPtJkkaoTlDoKq+eJlDWy8zfi+wd0B9CrhjXt1JkhaUnxiWpA4zBCSpwwwBSeowQ0CSOswQkKQOMwQkqcMMAUnqMENAkjrMEJCkDjMEJKnDDAFJ6rA5vzuoSzbs/sN5jX/j4c8vUCeStDh8JSBJHWYISFKHGQKS1GGGgCR12KKHQJJtSY4nOZFk92I/vyTpXYv67qAky4D/BPwyvYvP/yDJoap6ZTH7eL/4biJJ17rFfovoZuBEVf0EIMkBYDu9axJf9+YbGteD+Qbf1fwbGa7S1VvsEBgHTvXdnwb+3uxBSXYBu9rdP0ty/Cqf71bgf1/l3FG41vqFOXrOby58A/N8juvu33gJst+FN1fPQ/88ix0CGVCrSwpV+4B97/nJkqmqmnyvj7NYrrV+4drr+VrrF669nu134b2fPS/2ieFpYH3f/Qng9CL3IElqFjsEfgBsSrIxyQ3ADuDQIvcgSWoWdTmoqt5O8s+AZ4BlwDeq6tgCPuV7XlJaZNdav3Dt9Xyt9QvXXs/2u/Det55TdcmSvCSpI/zEsCR1mCEgSR12XYbAUvpqiiTfSHIuyct9tVuSHE7yetuu7ju2p/V9PMk9ffW7krzUjj2SZNDbbd+Pftcn+ZMkryY5luTLS7nnJB9McjTJj1q/v7GU++17rmVJXkjy3Wuk3zfac72YZGqp95zk5iRPJnmt/S5/con3+9H2b/vO7edJvrIoPVfVdXWjd8L5x8BHgBuAHwG3jbCfTwGfAF7uq/07YHfb3w38Ztu/rfW7AtjYfo5l7dhR4JP0PmvxNPDZBep3HfCJtv9h4H+0vpZkz+2xP9T2PwA8B2xZqv329f2vgN8DvrvUfyfac70B3DqrtmR7BvYD/6Tt3wDcvJT7ndX7MuCnwN9ajJ4X9IcZxa398M/03d8D7BlxTxv46yFwHFjX9tcBxwf1Su9dVJ9sY17rq38R+M+L1PtT9L7racn3DNwIPE/vU+hLtl96n485Anyad0NgyfbbHv8NLg2BJdkzsAo4SXvjy1Lvd0D/vwL898Xq+XpcDhr01RTjI+rlctZW1RmAtl3T6pfrfbztz64vqCQbgI/T++t6yfbcllZeBM4Bh6tqSfcL/Dbwa8Bf9dWWcr/Q+2T/95L8ML2vdVnKPX8EmAF+py25fS3JTUu439l2AN9q+wve8/UYAkN9NcUSdbneF/1nSvIh4A+Ar1TVz680dEBtUXuuqotVdSe9v7A3J7njCsNH2m+SXwXOVdUPh50yoDaK34m7q+oTwGeBh5J86gpjR93zcnpLsI9V1ceBP6e3lHI5o+733UZ6H6L9AvD7cw0dULuqnq/HELgWvpribJJ1AG17rtUv1/t0259dXxBJPkAvAL5ZVd++FnoGqKqfAd8Hti3hfu8GvpDkDeAA8Okkv7uE+wWgqk637TngO/S+EXip9jwNTLdXhABP0guFpdpvv88Cz1fV2XZ/wXu+HkPgWvhqikPAzra/k966+zv1HUlWJNkIbAKOtpeB55NsaWf67++b875qj/914NWq+q2l3nOSsSQ3t/2VwGeA15Zqv1W1p6omqmoDvd/NP66qLy3VfgGS3JTkw+/s01uzfnmp9lxVPwVOJfloK22l93X1S7LfWb7Iu0tB7/S2sD0v9EmOUdyAz9F7V8uPgV8fcS/fAs4Af0kvpR8AfpHeicHX2/aWvvG/3vo+Tt9ZfWCS3n+8HwP/kVknvd7Hfv8BvZePfwq82G6fW6o9A38XeKH1+zLwb1t9SfY7q/df4t0Tw0u2X3pr7D9qt2Pv/J9a4j3fCUy134v/Aqxeyv2257oReAv4G321Be/Zr42QpA67HpeDJElDMgQkqcMMAUnqMENAkjrMEJCkDjMEJKnDDAFJ6rD/D0CRY2x+i4hDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.hist(df_hpo['#Format: HPO-id'].value_counts(), bins=25)#.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "92.89813486370157"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hpo['#Format: HPO-id'].value_counts().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hpo['#Format: HPO-id'].value_counts().median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "#Format: HPO-id\n",
       "HP:0000707    3276\n",
       "Name: entrez-gene-id, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hpo.groupby('#Format: HPO-id')['entrez-gene-id'].nunique()[df_hpo.groupby('#Format: HPO-id')['entrez-gene-id'].nunique() == df_hpo.groupby('#Format: HPO-id')['entrez-gene-id'].nunique().max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7.0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hpo.groupby('#Format: HPO-id')['entrez-gene-id'].nunique().median()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "64.07758525549056"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hpo.groupby('#Format: HPO-id')['entrez-gene-id'].nunique().mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting data from scientific article"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "import nltk\n",
    "\n",
    "user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'\n",
    "\n",
    "\n",
    "url = \"https://www.cell.com/ajhg/fulltext/S0002-9297(17)30459-7\"\n",
    "#cookies = {'required_cookie': required_value}\n",
    "headers={'User-Agent':user_agent,} \n",
    "cookieProcessor = urllib.request.HTTPCookieProcessor()\n",
    "opener = urllib.request.build_opener(cookieProcessor)\n",
    "\n",
    "request=urllib.request.Request(url,None, headers) #The assembled request\n",
    "response = opener.open(request,timeout=100)\n",
    "data = response.read() # The data u need"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find any tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.2.0\n"
     ]
    }
   ],
   "source": [
    "from PIL import __version__ as vers\n",
    "print(vers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "soup = BeautifulSoup(data)\n",
    "\n",
    "table = soup.find('table') # what if there are more tables?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def tableDataText(table):    \n",
    "    \"\"\"Parses a html segment started with tag <table> followed \n",
    "    by multiple <tr> (table rows) and inner <td> (table data) tags. \n",
    "    It returns a list of rows with inner columns. \n",
    "    Accepts only one <th> (table header/data) in the first row.\n",
    "    \"\"\"\n",
    "    def rowgetDataText(tr, coltag='td'): # td (data) or th (header)       \n",
    "        return [td.get_text(strip=True) for td in tr.find_all(coltag)]  \n",
    "    rows = []\n",
    "    trs = table.find_all('tr')\n",
    "    headerow = rowgetDataText(trs[0], 'th')\n",
    "    if headerow: # if there is a header row include first\n",
    "        rows.append(headerow)\n",
    "        trs = trs[1:]\n",
    "    for tr in trs: # for every table row\n",
    "        rows.append(rowgetDataText(tr, 'td') ) # data row       \n",
    "    return rows\n",
    "\n",
    "def parseTable(table, remove_inc=True):\n",
    "    \"\"\"\n",
    "    Convert Table to pandas Dataframe\n",
    "    \n",
    "    Input:\n",
    "        table = html table from article\n",
    "        remove_inc = remove rows with incosistent length\n",
    "            (be careful: these can be helpful to categorize table)\n",
    "    \"\"\"\n",
    "    list_table = tableDataText(table)\n",
    "    dftable = pd.DataFrame(list_table[1:], columns=list_table[0])\n",
    "    if remove_inc:\n",
    "        med = np.median(dftable.isnull().sum(axis=1).values)\n",
    "        dftable = dftable.dropna(thresh=len(dftable.columns)-med)\n",
    "    return dftable\n",
    "\n",
    "raw_table = parseTable(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hpo_column_recognition(): \n",
    "    for col in columns: \n",
    "        # each column is checked seperately\n",
    "        if col == hpo:\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Case ID</th>\n",
       "      <th>Inheritance</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age (Years)</th>\n",
       "      <th>Prenatal and Neonatal History</th>\n",
       "      <th>PN Growth Retardation</th>\n",
       "      <th>Microcephaly</th>\n",
       "      <th>Motor Delay</th>\n",
       "      <th>Speech Delay</th>\n",
       "      <th>DD/ ID</th>\n",
       "      <th>Behavioral, Psychiatric, and Neurological Features</th>\n",
       "      <th>Malformations and Physical Anomalies</th>\n",
       "      <th>Additional Comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I</td>\n",
       "      <td>DN</td>\n",
       "      <td>M</td>\n",
       "      <td>4</td>\n",
       "      <td>SGA and feeding difficulties</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>mod</td>\n",
       "      <td>possible absence and focal seizures</td>\n",
       "      <td>VSD with tortuous aortic arch, horseshoe kidne...</td>\n",
       "      <td>early-onset hypothyroidism, limitation of join...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>II</td>\n",
       "      <td>DN</td>\n",
       "      <td>M</td>\n",
       "      <td>7</td>\n",
       "      <td>SGA and feeding difficulties</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>mild</td>\n",
       "      <td>sociable, empathetic, hand flapping tendency a...</td>\n",
       "      <td>Rt pelvic kidney, Rt inguinal hernia and scoli...</td>\n",
       "      <td>GOR, asthma and allergies.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>III</td>\n",
       "      <td>DN</td>\n",
       "      <td>M</td>\n",
       "      <td>7</td>\n",
       "      <td>SGA, polycythaemia, jaundice and hypoglycaemia...</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Mild</td>\n",
       "      <td>attention deficit, echolalia and tantrums.</td>\n",
       "      <td>inguinal hernia, cryptorchidism, proximally pl...</td>\n",
       "      <td>perineal and scalp abscesses, recurrent chest ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>IVa</td>\n",
       "      <td>Mat</td>\n",
       "      <td>F</td>\n",
       "      <td>32</td>\n",
       "      <td>SGA</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>mod</td>\n",
       "      <td>empathetic personality.</td>\n",
       "      <td>scoliosis</td>\n",
       "      <td>glaucoma, asthma, and eczema</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>IVb</td>\n",
       "      <td>U</td>\n",
       "      <td>F</td>\n",
       "      <td>68</td>\n",
       "      <td>U</td>\n",
       "      <td>U</td>\n",
       "      <td>U</td>\n",
       "      <td>U</td>\n",
       "      <td>U</td>\n",
       "      <td>mild</td>\n",
       "      <td>U</td>\n",
       "      <td>horseshoe kidney with multiple cysts</td>\n",
       "      <td>hiatus hernia</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Case ID Inheritance Gender Age (Years)  \\\n",
       "1       I          DN      M           4   \n",
       "2      II          DN      M           7   \n",
       "3     III          DN      M           7   \n",
       "4     IVa         Mat      F          32   \n",
       "5     IVb           U      F          68   \n",
       "\n",
       "                       Prenatal and Neonatal History PN Growth Retardation  \\\n",
       "1                       SGA and feeding difficulties                     Y   \n",
       "2                       SGA and feeding difficulties                     Y   \n",
       "3  SGA, polycythaemia, jaundice and hypoglycaemia...                     Y   \n",
       "4                                                SGA                     Y   \n",
       "5                                                  U                     U   \n",
       "\n",
       "  Microcephaly Motor Delay Speech Delay DD/ ID  \\\n",
       "1            N           Y            Y    mod   \n",
       "2            N           Y            Y   mild   \n",
       "3            Y           N            Y   Mild   \n",
       "4            N           Y            Y    mod   \n",
       "5            U           U            U   mild   \n",
       "\n",
       "  Behavioral, Psychiatric, and Neurological Features  \\\n",
       "1                possible absence and focal seizures   \n",
       "2  sociable, empathetic, hand flapping tendency a...   \n",
       "3         attention deficit, echolalia and tantrums.   \n",
       "4                            empathetic personality.   \n",
       "5                                                  U   \n",
       "\n",
       "                Malformations and Physical Anomalies  \\\n",
       "1  VSD with tortuous aortic arch, horseshoe kidne...   \n",
       "2  Rt pelvic kidney, Rt inguinal hernia and scoli...   \n",
       "3  inguinal hernia, cryptorchidism, proximally pl...   \n",
       "4                                          scoliosis   \n",
       "5               horseshoe kidney with multiple cysts   \n",
       "\n",
       "                                 Additional Comments  \n",
       "1  early-onset hypothyroidism, limitation of join...  \n",
       "2                         GOR, asthma and allergies.  \n",
       "3  perineal and scalp abscesses, recurrent chest ...  \n",
       "4                       glaucoma, asthma, and eczema  \n",
       "5                                      hiatus hernia  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_table.to_csv(r'data/TableExtraction.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find pdfs of file (supplementary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "suppl_flags = ['Supporting Information', 'Supplementary Information', 'Supplemental Files', 'Supplementary', 'Supplementals', 'Supporting Files']\n",
    "\n",
    "raw_table.to_csv(r'data/pdfs/TableExtraction.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing Table Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Case ID</th>\n",
       "      <th>Inheritance</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age (Years)</th>\n",
       "      <th>Prenatal and Neonatal History</th>\n",
       "      <th>PN Growth Retardation</th>\n",
       "      <th>Microcephaly</th>\n",
       "      <th>Motor Delay</th>\n",
       "      <th>Speech Delay</th>\n",
       "      <th>DD/ ID</th>\n",
       "      <th>Behavioral, Psychiatric, and Neurological Features</th>\n",
       "      <th>Malformations and Physical Anomalies</th>\n",
       "      <th>Additional Comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I</td>\n",
       "      <td>DN</td>\n",
       "      <td>M</td>\n",
       "      <td>4</td>\n",
       "      <td>SGA and feeding difficulties</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>mod</td>\n",
       "      <td>possible absence and focal seizures</td>\n",
       "      <td>VSD with tortuous aortic arch, horseshoe kidne...</td>\n",
       "      <td>early-onset hypothyroidism, limitation of join...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>II</td>\n",
       "      <td>DN</td>\n",
       "      <td>M</td>\n",
       "      <td>7</td>\n",
       "      <td>SGA and feeding difficulties</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>mild</td>\n",
       "      <td>sociable, empathetic, hand flapping tendency a...</td>\n",
       "      <td>Rt pelvic kidney, Rt inguinal hernia and scoli...</td>\n",
       "      <td>GOR, asthma and allergies.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>III</td>\n",
       "      <td>DN</td>\n",
       "      <td>M</td>\n",
       "      <td>7</td>\n",
       "      <td>SGA, polycythaemia, jaundice and hypoglycaemia...</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Mild</td>\n",
       "      <td>attention deficit, echolalia and tantrums.</td>\n",
       "      <td>inguinal hernia, cryptorchidism, proximally pl...</td>\n",
       "      <td>perineal and scalp abscesses, recurrent chest ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>IVa</td>\n",
       "      <td>Mat</td>\n",
       "      <td>F</td>\n",
       "      <td>32</td>\n",
       "      <td>SGA</td>\n",
       "      <td>Y</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>mod</td>\n",
       "      <td>empathetic personality.</td>\n",
       "      <td>scoliosis</td>\n",
       "      <td>glaucoma, asthma, and eczema</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>IVb</td>\n",
       "      <td>U</td>\n",
       "      <td>F</td>\n",
       "      <td>68</td>\n",
       "      <td>U</td>\n",
       "      <td>U</td>\n",
       "      <td>U</td>\n",
       "      <td>U</td>\n",
       "      <td>U</td>\n",
       "      <td>mild</td>\n",
       "      <td>U</td>\n",
       "      <td>horseshoe kidney with multiple cysts</td>\n",
       "      <td>hiatus hernia</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Case ID Inheritance Gender Age (Years)  \\\n",
       "0       I          DN      M           4   \n",
       "1      II          DN      M           7   \n",
       "2     III          DN      M           7   \n",
       "3     IVa         Mat      F          32   \n",
       "4     IVb           U      F          68   \n",
       "\n",
       "                       Prenatal and Neonatal History PN Growth Retardation  \\\n",
       "0                       SGA and feeding difficulties                     Y   \n",
       "1                       SGA and feeding difficulties                     Y   \n",
       "2  SGA, polycythaemia, jaundice and hypoglycaemia...                     Y   \n",
       "3                                                SGA                     Y   \n",
       "4                                                  U                     U   \n",
       "\n",
       "  Microcephaly Motor Delay Speech Delay DD/ ID  \\\n",
       "0            N           Y            Y    mod   \n",
       "1            N           Y            Y   mild   \n",
       "2            Y           N            Y   Mild   \n",
       "3            N           Y            Y    mod   \n",
       "4            U           U            U   mild   \n",
       "\n",
       "  Behavioral, Psychiatric, and Neurological Features  \\\n",
       "0                possible absence and focal seizures   \n",
       "1  sociable, empathetic, hand flapping tendency a...   \n",
       "2         attention deficit, echolalia and tantrums.   \n",
       "3                            empathetic personality.   \n",
       "4                                                  U   \n",
       "\n",
       "                Malformations and Physical Anomalies  \\\n",
       "0  VSD with tortuous aortic arch, horseshoe kidne...   \n",
       "1  Rt pelvic kidney, Rt inguinal hernia and scoli...   \n",
       "2  inguinal hernia, cryptorchidism, proximally pl...   \n",
       "3                                          scoliosis   \n",
       "4               horseshoe kidney with multiple cysts   \n",
       "\n",
       "                                 Additional Comments  \n",
       "0  early-onset hypothyroidism, limitation of join...  \n",
       "1                         GOR, asthma and allergies.  \n",
       "2  perineal and scalp abscesses, recurrent chest ...  \n",
       "3                       glaucoma, asthma, and eczema  \n",
       "4                                      hiatus hernia  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_table = pd.read_csv(r'data/TableExtraction.csv')\n",
    "raw_table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "soup = BeautifulSoup(data)\n",
    "raw = soup.get_text()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Main text spanning from: 13350 to 57531\n"
     ]
    }
   ],
   "source": [
    "print('Main text spanning from:', raw.find('Main Text'), 'to', raw.find('Acknowledgments'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = nltk.word_tokenize(raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'English' object has no attribute 'tag_map'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-ad977ac65b58>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mspacy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mnlp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mspacy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"en_core_sci_sm\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mnlp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtag_map\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m: 'English' object has no attribute 'tag_map'"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_sci_sm\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "__call__() takes exactly 2 positional arguments (1 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-9e5f948f4411>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mnlp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtagger\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mpipes.pyx\u001b[0m in \u001b[0;36mspacy.pipeline.pipes.Tagger.__call__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: __call__() takes exactly 2 positional arguments (1 given)"
     ]
    }
   ],
   "source": [
    "nlp.tagger()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy import displacy\n",
    "import scispacy\n",
    "from scispacy.linking import EntityLinker\n",
    "from scispacy.abbreviation import AbbreviationDetector\n",
    "\n",
    "nlp = spacy.load(\"en_core_sci_lg\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\levrex\\Anaconda3\\envs\\my_env\\lib\\site-packages\\sklearn\\base.py:334: UserWarning: Trying to unpickle estimator TfidfTransformer from version 0.20.3 when using version 0.23.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "C:\\Users\\levrex\\Anaconda3\\envs\\my_env\\lib\\site-packages\\sklearn\\base.py:334: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 0.20.3 when using version 0.23.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#hpo = scispacy.linking.EntityLinker(resolve_abbreviations=True, name='umls') # , name='hpo'\n",
    "# Add the abbreviation pipe to the spacy pipeline.\n",
    "#abbreviation_pipe = AbbreviationDetector(nlp)\n",
    "#nlp.add_pipe(abbreviation_pipe)\n",
    "#nlp.add_pipe(hpo)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'graph' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-34-9e8b44c8ae46>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mgraph\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'HP'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'graph' is not defined"
     ]
    }
   ],
   "source": [
    "graph['HP']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['bleeding', 'cause', 'she', 'need', 'he', 'e.', \"o'clock\", 'who', 'had', 'v.']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\levrex\\Anaconda3\\envs\\my_env\\lib\\site-packages\\ipykernel_launcher.py:3: UserWarning: [W008] Evaluating Lexeme.similarity based on empty vectors.\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "def get_related(word): \n",
    "    filtered_words = [w for w in word.vocab if w.is_lower == word.is_lower and w.prob >= -15] \n",
    "    similarity = sorted(filtered_words, key=lambda w: word.similarity(w), reverse=True) \n",
    "    return similarity[:10] \n",
    "print([w.lower_ for w in get_related(nlp.vocab[u'bleeding'])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Intracranial JJ adjective\n",
      "bleeding VBG verb, gerund or present participle\n",
      "High JJ adjective\n",
      "frequency NN noun, singular or mass\n",
      "hearing NN noun, singular or mass\n",
      "loss NN noun, singular or mass\n",
      "Short JJ adjective\n",
      "stature NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "wide JJ adjective\n",
      "nasal JJ adjective\n",
      "bridge NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "  _SP None\n",
      "microcephaly NNS noun, plural\n",
      "  _SP None\n",
      "hearing NN noun, singular or mass\n",
      "loss NN noun, singular or mass\n",
      "   _SP None\n",
      "ID NN noun, singular or mass\n",
      "Developmental JJ adjective\n",
      "delay NN noun, singular or mass\n",
      "( -LRB- left round bracket\n",
      "Global JJ adjective\n",
      ") -RRB- right round bracket\n",
      "  _SP None\n",
      "Motor NNP noun, proper singular\n",
      "Delay NNP noun, proper singular\n",
      "Speech NNP noun, proper singular\n",
      "delay NN noun, singular or mass\n",
      "  _SP None\n",
      "seizures NNS noun, plural\n",
      "  _SP None\n",
      "trigonocephaly NN noun, singular or mass\n",
      "  _SP None\n",
      "coloboma NN noun, singular or mass\n",
      "  _SP None\n",
      "hypertelorism JJ adjective\n",
      "  _SP None\n",
      "high JJ adjective\n",
      "arched JJ adjective\n",
      "eyebrows NNS noun, plural\n",
      "Hypertrichosis NN noun, singular or mass\n",
      "  _SP None\n",
      "congenital JJ adjective\n",
      "ptosis NN noun, singular or mass\n",
      "Epicanthus NNP noun, proper singular\n",
      "Telecanthus NNP noun, proper singular\n",
      "  _SP None\n",
      "pachygyria NN noun, singular or mass\n",
      "Other JJ adjective\n",
      "brain NN noun, singular or mass\n",
      "MRI NN noun, singular or mass\n",
      "abnormalities NNS noun, plural\n",
      "( -LRB- left round bracket\n",
      "i.e. FW foreign word\n",
      "brain NN noun, singular or mass\n",
      "atrophy NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "corpus NN noun, singular or mass\n",
      "callosum NN noun, singular or mass\n",
      "agenesis NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "hydrocephalus NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "white JJ adjective\n",
      "matter NN noun, singular or mass\n",
      "leasions NNS noun, plural\n",
      ", , punctuation mark, comma\n",
      "periventricular JJ adjective\n",
      "nodular JJ adjective\n",
      "heterotopias NNS noun, plural\n",
      ") -RRB- right round bracket\n",
      ". . punctuation mark, sentence closer\n",
      "Long JJ adjective\n",
      "eyelashes NNS noun, plural\n",
      "Deeply NNP noun, proper singular\n",
      "set VBD verb, past tense\n",
      "eyes NNS noun, plural\n",
      "Broad/wide JJ adjective\n",
      "nasal NN noun, singular or mass\n",
      "bridge/ NN noun, singular or mass\n",
      "tip NN noun, singular or mass\n",
      "Bulbous JJ adjective\n",
      "nose NN noun, singular or mass\n",
      "Failure NN noun, singular or mass\n",
      "to TO infinitival \"to\"\n",
      "thrive VB verb, base form\n",
      "Feeding VBG verb, gerund or present participle\n",
      "Difficulties NNPS noun, proper plural\n",
      "Prominent JJ adjective\n",
      "eyes/proptosis NN noun, singular or mass\n",
      "Down NN noun, singular or mass\n",
      "slanting VBG verb, gerund or present participle\n",
      "eyebrows NNS noun, plural\n",
      "Mid NNP noun, proper singular\n",
      "face VBP verb, non-3rd person singular present\n",
      "hypoplasia NN noun, singular or mass\n",
      "Low JJ adjective\n",
      "set NN noun, singular or mass\n",
      "ears NNS noun, plural\n",
      "Dysplastic JJ adjective\n",
      "ears NNS noun, plural\n",
      "Small NNP noun, proper singular\n",
      "ears NNS noun, plural\n",
      "with IN conjunction, subordinating or preposition\n",
      "overfolded JJ adjective\n",
      "helices/prominent JJ adjective\n",
      "helix NN noun, singular or mass\n",
      "Long JJ adjective\n",
      "philtrum NN noun, singular or mass\n",
      "Smooth JJ adjective\n",
      "philtrum NN noun, singular or mass\n",
      "Thin JJ adjective\n",
      "lips NNS noun, plural\n",
      "Pointed/small NN noun, singular or mass\n",
      "chin JJ adjective\n",
      "Pterygium NN noun, singular or mass\n",
      "colli/webbed JJ adjective\n",
      "neck NN noun, singular or mass\n",
      "Retrognatia/micrognathia NN noun, singular or mass\n",
      "Kypho/scoliotic JJ adjective\n",
      "posture NN noun, singular or mass\n",
      "Joint JJ adjective\n",
      "laxity NN noun, singular or mass\n",
      "Clinodactyly NNP noun, proper singular\n",
      "Tapering NNP noun, proper singular\n",
      "digits NNS noun, plural\n",
      "  _SP None\n",
      "Hernia VBP verb, non-3rd person singular present\n",
      "High JJ adjective\n",
      "palate NN noun, singular or mass\n",
      "Cleft JJ adjective\n",
      "lip NN noun, singular or mass\n",
      "Cleft NNP noun, proper singular\n",
      "palate NN noun, singular or mass\n",
      "Transverse NNP noun, proper singular\n",
      "right JJ adjective\n",
      "palmar NN noun, singular or mass\n",
      "crease/deep NN noun, singular or mass\n",
      "oakmar NN noun, singular or mass\n",
      "creases NNS noun, plural\n",
      "Pectus NNP noun, proper singular\n",
      "excavatum/pectus NN noun, singular or mass\n",
      "deformity NN noun, singular or mass\n",
      "Radial JJ adjective\n",
      "abnormalities NNS noun, plural\n",
      "( -LRB- left round bracket\n",
      "fingers/toes NNS noun, plural\n",
      ") -RRB- right round bracket\n",
      "Hypotonia NN noun, singular or mass\n",
      "Dystonia NN noun, singular or mass\n",
      "Frontal JJ adjective\n",
      "bossing NN noun, singular or mass\n",
      "Scaphocephaly NNP noun, proper singular\n",
      "Bitemporal JJ adjective\n",
      "narrowing VBG verb, gerund or present participle\n",
      "Prominence NN noun, singular or mass\n",
      "of IN conjunction, subordinating or preposition\n",
      "metopic JJ adjective\n",
      "suture/ridge NN noun, singular or mass\n",
      "Large JJ adjective\n",
      "anterior JJ adjective\n",
      "fontanelle NN noun, singular or mass\n",
      "High NNP noun, proper singular\n",
      "anterior JJ adjective\n",
      "hairline NN noun, singular or mass\n",
      "Low JJ adjective\n",
      "anterior JJ adjective\n",
      "hairline NN noun, singular or mass\n",
      "Low JJ adjective\n",
      "posterior JJ adjective\n",
      "hairline NN noun, singular or mass\n",
      "Sparse NNP noun, proper singular\n",
      "eyebrows NNS noun, plural\n",
      "Flaired VBN verb, past participle\n",
      "and CC conjunction, coordinating\n",
      "straight JJ adjective\n",
      "eyebrows NNS noun, plural\n",
      "Synophris NNP noun, proper singular\n",
      "Upslanting/long IN conjunction, subordinating or preposition\n",
      "palpebral JJ adjective\n",
      "fissures NNS noun, plural\n",
      "Heart NN noun, singular or mass\n",
      "defects NNS noun, plural\n",
      "( -LRB- left round bracket\n",
      "Other JJ adjective\n",
      "then RB adverb\n",
      "ASD NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "VSD NN noun, singular or mass\n",
      "or CC conjunction, coordinating\n",
      "PDA NN noun, singular or mass\n",
      ") -RRB- right round bracket\n",
      "ASD NN noun, singular or mass\n",
      "VSD NN noun, singular or mass\n",
      "Cardiomegaly NN noun, singular or mass\n",
      "Persistent NNP noun, proper singular\n",
      "Embryonic NNP noun, proper singular\n",
      "Sructures NNPS noun, proper plural\n",
      "Large NNP noun, proper singular\n",
      "squared VBD verb, past tense\n",
      "nose NN noun, singular or mass\n",
      "tip NN noun, singular or mass\n",
      "upturned JJ adjective\n",
      "nose NN noun, singular or mass\n",
      "Prominent JJ adjective\n",
      "nasal JJ adjective\n",
      "root NN noun, singular or mass\n",
      "on IN conjunction, subordinating or preposition\n",
      "profile NN noun, singular or mass\n",
      "Thin JJ adjective\n",
      "upper JJ adjective\n",
      "lip NN noun, singular or mass\n",
      "Large/wide NN noun, singular or mass\n",
      "mouth NN noun, singular or mass\n",
      "tented VBD verb, past tense\n",
      "upper JJ adjective\n",
      "lip NN noun, singular or mass\n",
      "with IN conjunction, subordinating or preposition\n",
      "downturned JJ adjective\n",
      "corners NNS noun, plural\n",
      "Thick/prominent/everted NN noun, singular or mass\n",
      "lower JJR adjective, comparative\n",
      "lip NN noun, singular or mass\n",
      "Prominent JJ adjective\n",
      "cheeks NNS noun, plural\n",
      "Autism NNP noun, proper singular\n",
      "Spectrum NN noun, singular or mass\n",
      "disorder NN noun, singular or mass\n",
      "ADHD/ADD NN noun, singular or mass\n",
      "Other JJ adjective\n",
      "Psychiatric JJ adjective\n",
      "diagnosis NN noun, singular or mass\n",
      "( -LRB- left round bracket\n",
      "shizofrenia NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "depression NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "other JJ adjective\n",
      ", , punctuation mark, comma\n",
      "not RB adverb\n",
      "autism NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "or CC conjunction, coordinating\n",
      "ADHD/ADD NN noun, singular or mass\n",
      ") -RRB- right round bracket\n",
      "Behavioral JJ adjective\n",
      "abnormalities NNS noun, plural\n",
      "( -LRB- left round bracket\n",
      "tantrums NNS noun, plural\n",
      ", , punctuation mark, comma\n",
      "auto NN noun, singular or mass\n",
      "mutulation NN noun, singular or mass\n",
      ") -RRB- right round bracket\n",
      "Horseshoe NNP noun, proper singular\n",
      "kidney NN noun, singular or mass\n",
      "Renal JJ adjective\n",
      "Cysts NNS noun, plural\n",
      "Other JJ adjective\n",
      "Renal JJ adjective\n",
      "abnormality NN noun, singular or mass\n",
      "Cryptorchidism NN noun, singular or mass\n",
      "Recurrent JJ adjective\n",
      "infections NNS noun, plural\n",
      "Photosensitivity NN noun, singular or mass\n",
      "Polyarthralgia NNP noun, proper singular\n",
      "Hepatomegaly NNP noun, proper singular\n",
      "Hypothyroidism NNP noun, proper singular\n",
      "Leukocytose NNP noun, proper singular\n",
      "Trombocytopenie NNP noun, proper singular\n",
      "Asthma NNP noun, proper singular\n",
      "Strabismus NN noun, singular or mass\n",
      "Growth NN noun, singular or mass\n",
      "hormone NN noun, singular or mass\n",
      "deficiency NN noun, singular or mass\n"
     ]
    }
   ],
   "source": [
    "for word in doc:\n",
    "    print(word.text, word.tag_, spacy.explain(word.tag_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Intracranial bleeding High frequency hearing loss Short stature, wide nasal bridge,  microcephaly  hearing loss   ID Developmental delay (Global)  Motor Delay Speech delay  seizures  trigonocephaly  coloboma  hypertelorism  high arched eyebrows Hypertrichosis  congenital ptosis Epicanthus Telecanthus  pachygyria Other brain MRI abnormalities (i.e. brain atrophy, corpus callosum agenesis, hydrocephalus, white matter leasions, periventricular nodular heterotopias). Long eyelashes Deeply set eyes Broad/wide nasal bridge/ tip Bulbous nose Failure to thrive Feeding Difficulties Prominent eyes/proptosis Down slanting eyebrows Mid face hypoplasia Low set ears Dysplastic ears Small ears with overfolded helices/prominent helix Long philtrum Smooth philtrum Thin lips Pointed/small chin Pterygium colli/webbed neck Retrognatia/micrognathia Kypho/scoliotic posture Joint laxity Clinodactyly Tapering digits \\xa0Hernia High palate Cleft lip Cleft palate Transverse right palmar crease/deep oakmar creases Pectus excavatum/pectus deformity Radial abnormalities (fingers/toes) Hypotonia Dystonia Frontal bossing Scaphocephaly Bitemporal narrowing Prominence of metopic suture/ridge Large anterior fontanelle High anterior hairline Low anterior hairline Low posterior hairline Sparse eyebrows Flaired and straight eyebrows Synophris Upslanting/long palpebral fissures Heart defects (Other then ASD, VSD or PDA) ASD VSD Cardiomegaly Persistent Embryonic Structures Large squared nose tip upturned nose Prominent nasal root on profile Thin upper lip Large/wide mouth tented upper lip with downturned corners Thick/prominent/everted lower lip Prominent cheeks Autism Spectrum disorder ADHD/ADD Other Psychiatric diagnosis (shizofrenia, depression, other, not autism, or ADHD/ADD) Behavioral abnormalities (tantrums, auto mutulation) Horseshoe kidney Renal Cysts Other Renal abnormality Cryptorchidism Recurrent infections Photosensitivity Polyarthralgia Hepatomegaly Hypothyroidism Leukocytose Trombocytopenie Asthma Strabismus Growth hormone deficiency\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    intracranial bleeding\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    high frequency hearing loss\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    short stature\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", wide \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    nasal bridge\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ",  \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    microcephaly\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "  hearing loss   id \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    developmental delay\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " (\n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    global\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ")  motor \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    delay\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    speech\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " delay  seizures  trigonocephaly  \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    coloboma\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "  hypertelorism  high arched eyebrows hypertrichosis  congenital ptosis epicanthus telecanthus  pachygyria other \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    brain mri abnormalities\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " (i.e. \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    brain atrophy\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    corpus callosum agenesis\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hydrocephalus\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    white matter\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    leasions\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    periventricular nodular heterotopias\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "). \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    long eyelashes\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " deeply set \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    eyes\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    broad/wide\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " nasal bridge/ \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    tip bulbous nose failure\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " to \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    thrive\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " feeding difficulties prominent \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    eyes/proptosis down\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " slanting eyebrows mid face hypoplasia low set \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    ears\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " dysplastic ears \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    small ears\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " with \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    overfolded\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    helices/prominent helix long philtrum smooth philtrum thin lips\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    pointed/small chin pterygium\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    colli/webbed neck retrognatia/micrognathia\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " kypho/scoliotic posture joint laxity \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    clinodactyly tapering digits\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "  \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hernia\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    high palate cleft lip cleft palate transverse\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " right palmar \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    crease/deep oakmar creases pectus\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " excavatum/pectus \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    deformity radial abnormalities\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " (fingers/toes) hypotonia dystonia \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    frontal bossing scaphocephaly\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " bitemporal narrowing prominence of \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    metopic\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " suture/ridge large anterior fontanelle high anterior hairline low anterior hairline low posterior hairline sparse eyebrows flaired and \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    straight eyebrows synophris\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    upslanting/long palpebral fissures heart defects\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " (other then \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    asd\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    vsd\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " or \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    pda) asd vsd\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    cardiomegaly\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    persistent\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    embryonic structures\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " large \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    squared nose tip upturned nose\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " prominent \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    nasal root\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " on profile \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    thin upper lip\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    large/wide mouth tented upper lip\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " with \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    downturned corners\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " thick/prominent/everted \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    lower\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " lip \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    prominent\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    cheeks autism spectrum disorder\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    adhd/add\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " other \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    psychiatric diagnosis\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " (\n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    shizofrenia\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    depression\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", other, not \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    autism\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", or \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    adhd/add\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ") \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    behavioral abnormalities\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " (\n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    tantrums\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    auto mutulation\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ") \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    horseshoe kidney renal cysts\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " other \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    renal abnormality cryptorchidism\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    recurrent\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    infections\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    photosensitivity polyarthralgia\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " hepatomegaly \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hypothyroidism\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    leukocytose\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    trombocytopenie asthma\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " strabismus \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    growth hormone deficiency\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "text = 'Intracranial bleeding High frequency hearing loss Short stature, wide nasal bridge,  microcephaly  hearing loss   ID Developmental delay (Global)  Motor Delay Speech delay  seizures  trigonocephaly  coloboma  hypertelorism  high arched eyebrows Hypertrichosis  congenital ptosis Epicanthus Telecanthus  pachygyria Other brain MRI abnormalities (i.e. brain atrophy, corpus callosum agenesis, hydrocephalus, white matter leasions, periventricular nodular heterotopias). Long eyelashes Deeply set eyes Broad/wide nasal bridge/ tip Bulbous nose Failure to thrive Feeding Difficulties Prominent eyes/proptosis Down slanting eyebrows Mid face hypoplasia Low set ears Dysplastic ears Small ears with overfolded helices/prominent helix Long philtrum Smooth philtrum Thin lips Pointed/small chin Pterygium colli/webbed neck Retrognatia/micrognathia Kypho/scoliotic posture Joint laxity Clinodactyly Tapering digits \\xa0Hernia High palate Cleft lip Cleft palate Transverse right palmar crease/deep oakmar creases Pectus excavatum/pectus deformity Radial abnormalities (fingers/toes) Hypotonia Dystonia Frontal bossing Scaphocephaly Bitemporal narrowing Prominence of metopic suture/ridge Large anterior fontanelle High anterior hairline Low anterior hairline Low posterior hairline Sparse eyebrows Flaired and straight eyebrows Synophris Upslanting/long palpebral fissures Heart defects (Other then ASD, VSD or PDA) ASD VSD Cardiomegaly Persistent Embryonic Structures Large squared nose tip upturned nose Prominent nasal root on profile Thin upper lip Large/wide mouth tented upper lip with downturned corners Thick/prominent/everted lower lip Prominent cheeks Autism Spectrum disorder ADHD/ADD Other Psychiatric diagnosis (shizofrenia, depression, other, not autism, or ADHD/ADD) Behavioral abnormalities (tantrums, auto mutulation) Horseshoe kidney Renal Cysts Other Renal abnormality Cryptorchidism Recurrent infections Photosensitivity Polyarthralgia Hepatomegaly Hypothyroidism Leukocytose Trombocytopenie Asthma Strabismus Growth hormone deficiency'\n",
    "#text = 'Increased serum lactate,'\n",
    "#text = 'Intracranial bleeding'\n",
    "#text = text.replace(\"/\", \" / \")\n",
    "doc = nlp(text.lower())\n",
    "\n",
    "\n",
    "\n",
    "colors = {\"ENTITY\": \"linear-gradient(90deg, #aa9cfc, #fc9ce7)\"}\n",
    "options = {\"ents\": [\"ENTITY\"], \"colors\": colors}\n",
    "#displacy.render(doc, style=\"ent\", options=options, manual=True)\n",
    "\n",
    "displacy.render(doc, style=\"ent\", options=options)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "intracranial JJ adjective\n",
      "bleeding VBG verb, gerund or present participle\n",
      "high JJ adjective\n",
      "frequency NN noun, singular or mass\n",
      "hearing NN noun, singular or mass\n",
      "loss NN noun, singular or mass\n",
      "short JJ adjective\n",
      "stature NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "wide JJ adjective\n",
      "nasal JJ adjective\n",
      "bridge NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "  _SP None\n",
      "microcephaly NNS noun, plural\n",
      "  _SP None\n",
      "hearing NN noun, singular or mass\n",
      "loss NN noun, singular or mass\n",
      "   _SP None\n",
      "i PRP pronoun, personal\n",
      "d NN noun, singular or mass\n",
      "developmental JJ adjective\n",
      "delay NN noun, singular or mass\n",
      "( -LRB- left round bracket\n",
      "global JJ adjective\n",
      ") -RRB- right round bracket\n",
      "  _SP None\n",
      "motor NN noun, singular or mass\n",
      "delay NN noun, singular or mass\n",
      "speech NN noun, singular or mass\n",
      "delay NN noun, singular or mass\n",
      "  _SP None\n",
      "seizures NNS noun, plural\n",
      "  _SP None\n",
      "trigonocephaly NN noun, singular or mass\n",
      "  _SP None\n",
      "coloboma NN noun, singular or mass\n",
      "  _SP None\n",
      "hypertelorism JJ adjective\n",
      "  _SP None\n",
      "high JJ adjective\n",
      "arched JJ adjective\n",
      "eyebrows NNS noun, plural\n",
      "hypertrichosis JJ adjective\n",
      "  _SP None\n",
      "congenital JJ adjective\n",
      "ptosis NN noun, singular or mass\n",
      "epicanthus NN noun, singular or mass\n",
      "telecanthus JJ adjective\n",
      "  _SP None\n",
      "pachygyria NN noun, singular or mass\n",
      "other JJ adjective\n",
      "brain NN noun, singular or mass\n",
      "mri NN noun, singular or mass\n",
      "abnormalities NNS noun, plural\n",
      "( -LRB- left round bracket\n",
      "i.e. FW foreign word\n",
      "brain NN noun, singular or mass\n",
      "atrophy NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "corpus NN noun, singular or mass\n",
      "callosum NN noun, singular or mass\n",
      "agenesis NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "hydrocephalus NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "white JJ adjective\n",
      "matter NN noun, singular or mass\n",
      "leasions NNS noun, plural\n",
      ", , punctuation mark, comma\n",
      "periventricular JJ adjective\n",
      "nodular JJ adjective\n",
      "heterotopias NNS noun, plural\n",
      ") -RRB- right round bracket\n",
      ". . punctuation mark, sentence closer\n",
      "long JJ adjective\n",
      "eyelashes NNS noun, plural\n",
      "deeply RB adverb\n",
      "set VBN verb, past participle\n",
      "eyes NNS noun, plural\n",
      "broad RB adverb\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "wide JJ adjective\n",
      "nasal JJ adjective\n",
      "bridge NN noun, singular or mass\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "  _SP None\n",
      "tip NN noun, singular or mass\n",
      "bulbous JJ adjective\n",
      "nose NN noun, singular or mass\n",
      "failure NN noun, singular or mass\n",
      "to TO infinitival \"to\"\n",
      "thrive VB verb, base form\n",
      "feeding VBG verb, gerund or present participle\n",
      "difficulties NNS noun, plural\n",
      "prominent JJ adjective\n",
      "eyes NNS noun, plural\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "proptosis NN noun, singular or mass\n",
      "down RP adverb, particle\n",
      "slanting VBG verb, gerund or present participle\n",
      "eyebrows NNS noun, plural\n",
      "mid JJ adjective\n",
      "face NN noun, singular or mass\n",
      "hypoplasia NN noun, singular or mass\n",
      "low JJ adjective\n",
      "set NN noun, singular or mass\n",
      "ears NNS noun, plural\n",
      "dysplastic JJ adjective\n",
      "ears NNS noun, plural\n",
      "small JJ adjective\n",
      "ears NNS noun, plural\n",
      "with IN conjunction, subordinating or preposition\n",
      "overfolded JJ adjective\n",
      "helices NNS noun, plural\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "prominent JJ adjective\n",
      "helix NN noun, singular or mass\n",
      "long JJ adjective\n",
      "philtrum JJ adjective\n",
      "smooth JJ adjective\n",
      "philtrum NN noun, singular or mass\n",
      "thin JJ adjective\n",
      "lips NNS noun, plural\n",
      "pointed VBD verb, past tense\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "small JJ adjective\n",
      "chin JJ adjective\n",
      "pterygium NN noun, singular or mass\n",
      "colli NN noun, singular or mass\n",
      "/ , punctuation mark, comma\n",
      "webbed VBN verb, past participle\n",
      "neck NN noun, singular or mass\n",
      "retrognatia NN noun, singular or mass\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "micrognathia JJ adjective\n",
      "kypho NN noun, singular or mass\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "scoliotic JJ adjective\n",
      "posture NN noun, singular or mass\n",
      "joint JJ adjective\n",
      "laxity NN noun, singular or mass\n",
      "clinodactyly NNS noun, plural\n",
      "tapering VBG verb, gerund or present participle\n",
      "digits NNS noun, plural\n",
      "  _SP None\n",
      "hernia VBP verb, non-3rd person singular present\n",
      "high JJ adjective\n",
      "palate NN noun, singular or mass\n",
      "cleft NN noun, singular or mass\n",
      "lip NN noun, singular or mass\n",
      "cleft NN noun, singular or mass\n",
      "palate NN noun, singular or mass\n",
      "transverse NN noun, singular or mass\n",
      "right JJ adjective\n",
      "palmar JJ adjective\n",
      "crease NN noun, singular or mass\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "deep JJ adjective\n",
      "oakmar NN noun, singular or mass\n",
      "creases NNS noun, plural\n",
      "pectus NN noun, singular or mass\n",
      "excavatum NN noun, singular or mass\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "pectus NN noun, singular or mass\n",
      "deformity NN noun, singular or mass\n",
      "radial JJ adjective\n",
      "abnormalities NNS noun, plural\n",
      "( -LRB- left round bracket\n",
      "fingers NNS noun, plural\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "toes NNS noun, plural\n",
      ") -RRB- right round bracket\n",
      "hypotonia JJ adjective\n",
      "dystonia NN noun, singular or mass\n",
      "frontal JJ adjective\n",
      "bossing NN noun, singular or mass\n",
      "scaphocephaly NN noun, singular or mass\n",
      "bitemporal JJ adjective\n",
      "narrowing NN noun, singular or mass\n",
      "prominence NN noun, singular or mass\n",
      "of IN conjunction, subordinating or preposition\n",
      "metopic JJ adjective\n",
      "suture NN noun, singular or mass\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "ridge NN noun, singular or mass\n",
      "large JJ adjective\n",
      "anterior JJ adjective\n",
      "fontanelle NN noun, singular or mass\n",
      "high JJ adjective\n",
      "anterior JJ adjective\n",
      "hairline NN noun, singular or mass\n",
      "low JJ adjective\n",
      "anterior JJ adjective\n",
      "hairline NN noun, singular or mass\n",
      "low JJ adjective\n",
      "posterior JJ adjective\n",
      "hairline NN noun, singular or mass\n",
      "sparse JJ adjective\n",
      "eyebrows NNS noun, plural\n",
      "flaired VBD verb, past tense\n",
      "and CC conjunction, coordinating\n",
      "straight JJ adjective\n",
      "eyebrows NNS noun, plural\n",
      "synophris NN noun, singular or mass\n",
      "upslanting VBG verb, gerund or present participle\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "long JJ adjective\n",
      "palpebral JJ adjective\n",
      "fissures NNS noun, plural\n",
      "heart NN noun, singular or mass\n",
      "defects NNS noun, plural\n",
      "( -LRB- left round bracket\n",
      "other JJ adjective\n",
      "then RB adverb\n",
      "asd NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "vsd NN noun, singular or mass\n",
      "or CC conjunction, coordinating\n",
      "pda NN noun, singular or mass\n",
      ") -RRB- right round bracket\n",
      "asd NN noun, singular or mass\n",
      "vsd NN noun, singular or mass\n",
      "cardiomegaly NN noun, singular or mass\n",
      "persistent JJ adjective\n",
      "embryonic JJ adjective\n",
      "sructures NNS noun, plural\n",
      "large JJ adjective\n",
      "squared VBN verb, past participle\n",
      "nose NN noun, singular or mass\n",
      "tip NN noun, singular or mass\n",
      "upturned JJ adjective\n",
      "nose NN noun, singular or mass\n",
      "prominent JJ adjective\n",
      "nasal NN noun, singular or mass\n",
      "root NN noun, singular or mass\n",
      "on IN conjunction, subordinating or preposition\n",
      "profile NN noun, singular or mass\n",
      "thin JJ adjective\n",
      "upper JJ adjective\n",
      "lip NN noun, singular or mass\n",
      "large JJ adjective\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "wide JJ adjective\n",
      "mouth NN noun, singular or mass\n",
      "tented VBD verb, past tense\n",
      "upper JJ adjective\n",
      "lip NN noun, singular or mass\n",
      "with IN conjunction, subordinating or preposition\n",
      "downturned JJ adjective\n",
      "corners NNS noun, plural\n",
      "thick JJ adjective\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "prominent JJ adjective\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "everted JJ adjective\n",
      "lower JJR adjective, comparative\n",
      "lip NN noun, singular or mass\n",
      "prominent JJ adjective\n",
      "cheeks JJ adjective\n",
      "autism NN noun, singular or mass\n",
      "spectrum NN noun, singular or mass\n",
      "disorder NN noun, singular or mass\n",
      "adhd JJ adjective\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "add VB verb, base form\n",
      "other JJ adjective\n",
      "psychiatric JJ adjective\n",
      "diagnosis NN noun, singular or mass\n",
      "( -LRB- left round bracket\n",
      "shizofrenia NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "depression NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "other JJ adjective\n",
      ", , punctuation mark, comma\n",
      "not RB adverb\n",
      "autism NN noun, singular or mass\n",
      ", , punctuation mark, comma\n",
      "or CC conjunction, coordinating\n",
      "adhd JJ adjective\n",
      "/ : punctuation mark, colon or ellipsis\n",
      "add VB verb, base form\n",
      ") -RRB- right round bracket\n",
      "behavioral JJ adjective\n",
      "abnormalities NNS noun, plural\n",
      "( -LRB- left round bracket\n",
      "tantrums NNS noun, plural\n",
      ", , punctuation mark, comma\n",
      "auto NN noun, singular or mass\n",
      "mutulation NN noun, singular or mass\n",
      ") -RRB- right round bracket\n",
      "horseshoe JJ adjective\n",
      "kidney NN noun, singular or mass\n",
      "renal JJ adjective\n",
      "cysts NNS noun, plural\n",
      "other JJ adjective\n",
      "renal JJ adjective\n",
      "abnormality NN noun, singular or mass\n",
      "cryptorchidism NN noun, singular or mass\n",
      "recurrent JJ adjective\n",
      "infections NNS noun, plural\n",
      "photosensitivity NN noun, singular or mass\n",
      "polyarthralgia NN noun, singular or mass\n",
      "hepatomegaly NN noun, singular or mass\n",
      "hypothyroidism NN noun, singular or mass\n",
      "leukocytose NN noun, singular or mass\n",
      "trombocytopenie NN noun, singular or mass\n",
      "asthma NN noun, singular or mass\n",
      "strabismus NN noun, singular or mass\n",
      "growth NN noun, singular or mass\n",
      "hormone NN noun, singular or mass\n",
      "deficiency NN noun, singular or mass\n"
     ]
    }
   ],
   "source": [
    "for word in doc:\n",
    "    print(word.text, word.tag_, spacy.explain(word.tag_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<spacy.vocab.Vocab at 0x1eab6e4ed90>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wide nasal bridge,  microcephaly  \n"
     ]
    }
   ],
   "source": [
    "d1 = doc[9:16]#.val#.copy()\n",
    "print(d1)\n",
    "#[word for word in d1] : #[True, False]\n",
    "#    print(word.tag_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'ent' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-32-f129bb0d91d9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m \u001b[0mdoc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ment\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0ment\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'ent' is not defined"
     ]
    }
   ],
   "source": [
    "print( doc[ent.start:ent.end])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Elongate entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nasal bridge  vs  wide nasal bridge\n",
      "brain mri abnormalities  vs  other brain mri abnormalities\n",
      "eyes  vs  deeply set eyes\n",
      "tip bulbous nose failure  vs  tip bulbous nose failure to thrive\n",
      "eyes/proptosis down  vs  prominent eyes/proptosis down\n",
      "asd  vs  other then asd\n",
      "squared nose tip upturned nose  vs  large squared nose tip upturned nose\n",
      "nasal root  vs  prominent nasal root\n",
      "lower  vs  thick/prominent/everted lower\n",
      "psychiatric diagnosis  vs  other psychiatric diagnosis\n",
      "autism  vs  not autism\n",
      "renal abnormality cryptorchidism  vs  other renal abnormality cryptorchidism\n"
     ]
    }
   ],
   "source": [
    "def check_left(doc, ent, prev_ent_end, window_size=3):\n",
    "    \"\"\" \n",
    "    doc = document\n",
    "    ent = entity\n",
    "    \"\"\"\n",
    "    d1 = doc[max(0,prev_ent_end,ent.start-window_size):ent.start]\n",
    "    ELONGATE = ['/', 'JJ', 'RB', 'RP', 'VBN'] # , 'IN', 'TO', 'VB'\n",
    "    l = [word.tag_ for word in d1]\n",
    "    l_stop = [True if i in ELONGATE else False for i in l]\n",
    "    #if 'JJ' in l:\n",
    "    #    print(d1, l)\n",
    "    if False in l_stop:\n",
    "        ix_elong = (len(l_stop)-1) - l_stop[::-1].index(False)\n",
    "        #print(ix_elong, l_stop)\n",
    "    else :\n",
    "        ix_elong = 0\n",
    "    l_fin = [d1[ix_elong:][j].text if l_stop[ix_elong:][j] else '' for j in range(len(d1[ix_elong:]))]\n",
    "    l_fin.append(doc[ent.start:ent.end].text)\n",
    "    if ent.text != ' '.join(l_fin).lstrip():\n",
    "        print(ent.text, ' vs ', ' '.join(l_fin).lstrip()) \n",
    "    return\n",
    "\n",
    "\n",
    "def check_right(doc, ent, next_ent_start, window_size=3):\n",
    "    # On profile, in, within blabla\n",
    "    d1 = doc[ent.end:min(len(doc),ent.end+window_size)] # next_ent_start,\n",
    "    ELONGATE = ['/', 'IN', 'TO', 'VB'] # , 'IN', 'TO', 'VB'\n",
    "    l = [word.tag_ for word in d1]\n",
    "    if l != []:\n",
    "        if l[0] == 'IN':\n",
    "            ELONGATE.extend(['JJ', 'NNS', 'NN'])\n",
    "        l_stop = [True if i in ELONGATE else False for i in l]\n",
    "        #print(l, l_stop)\n",
    "        if False in l_stop:\n",
    "            ix_elong = l_stop.index(False)\n",
    "        else :\n",
    "            ix_elong = 0\n",
    "        l_fin = [doc[ent.start:ent.end].text]\n",
    "        l_fin.extend([d1[:ix_elong][j].text if l_stop[:ix_elong][j] else '' for j in range(len(d1[:ix_elong]))])\n",
    "        if ent.text != ' '.join(l_fin).lstrip():\n",
    "            print(ent.text, ' vs ', ' '.join(l_fin).lstrip()) \n",
    "\n",
    "    \"\"\"\n",
    "    #if 'JJ' in l:\n",
    "    #    print(d1, l)\n",
    "    if False in l_stop:\n",
    "        ix_elong = (len(l_stop)-1) - l_stop[::-1].index(False)\n",
    "        #print(ix_elong, l_stop)\n",
    "    else :\n",
    "        ix_elong = 0\n",
    "    l_fin = [d1[ix_elong:][j].text if l_stop[ix_elong:][j] else '' for j in range(len(d1[ix_elong:]))]\n",
    "    l_fin.append(doc[ent.start:ent.end].text)\n",
    "    if ent.text != ' '.join(l_fin).lstrip():\n",
    "        print(ent.text, ' vs ', ' '.join(l_fin).lstrip()) \n",
    "    \"\"\"\n",
    "    return\n",
    "\n",
    "def extend_entities(doc):\n",
    "    OR_SIGN = ['/']\n",
    "    cnt = 0\n",
    "    window_size = 3\n",
    "     #  'NN'\n",
    "    prev_ent_end = 0 # Keep track of the latest entity\n",
    "    for ent in doc.ents:\n",
    "        #print(ent.text, ent.start_char, ent.end_char)\n",
    "        #len(doc)\n",
    "        #d1 = doc[max(0,prev_ent_end,ent.start-window_size):ent.start] # prev_ent_end,\n",
    "        if len(doc.ents) > cnt + 1:\n",
    "            next_ent_start = doc.ents[cnt+1].start\n",
    "        else :\n",
    "            next_ent_start = len(doc.ents)\n",
    "        check_left(doc, ent, prev_ent_end, window_size)\n",
    "        check_right(doc, ent, next_ent_start, window_size)\n",
    "        #if 'JJ' in l:\n",
    "        #    print(d1)\n",
    "        #    print(l)\n",
    "        #l.append(doc[ent.start:ent.end])\n",
    "        #if lstop[-1] == True:\n",
    "        \n",
    "        \n",
    "        \n",
    "        #for word in doc[max(0,prev_ent_end,ent.start-window_size):ent.start]:\n",
    "        #    \n",
    "        #    \n",
    "        #    #print(word, word.tag_, spacy.explain(word.tag_))\n",
    "        #    # add the adjective, if comma is near add till a comma is found?\n",
    "        #    # calculate probability\n",
    "        prev_ent_end = ent.end\n",
    "        cnt += 1\n",
    "        #if cnt > 5:\n",
    "        #    break\n",
    "            \n",
    "        #prev_ent = ent\n",
    "        # prev_ent\n",
    "        # look at tokens before & after\n",
    "        #for word in doc:\n",
    "        #    print(word.text, word.tag_, spacy.explain(word.tag_))\n",
    "extend_entities(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Intracranial bleeding 0 21 ENTITY\n",
      "High frequency hearing loss 22 49 ENTITY\n",
      "Short stature 50 63 ENTITY\n",
      "nasal bridge 70 82 ENTITY\n",
      "microcephaly 85 97 ENTITY\n",
      "Developmental delay 117 136 ENTITY\n",
      "Global 138 144 ENTITY\n",
      "Delay 153 158 ENTITY\n",
      "Speech 159 165 ENTITY\n",
      "coloboma 199 207 ENTITY\n",
      "Hypertrichosis 245 259 ENTITY\n",
      "Telecanthus 290 301 ENTITY\n",
      "brain 320 325 ENTITY\n",
      "abnormalities 330 343 ENTITY\n",
      "brain atrophy 350 363 ENTITY\n",
      "corpus callosum agenesis 365 389 ENTITY\n",
      "hydrocephalus 391 404 ENTITY\n",
      "white matter 406 418 ENTITY\n",
      "leasions 419 427 ENTITY\n",
      "periventricular nodular heterotopias 429 465 ENTITY\n",
      "Long eyelashes 468 482 ENTITY\n",
      "Deeply set 483 493 ENTITY\n",
      "eyes 494 498 ENTITY\n",
      "Broad/wide 499 509 ENTITY\n",
      "nasal 510 515 ENTITY\n",
      "Bulbous nose 528 540 ENTITY\n",
      "Failure 541 548 ENTITY\n",
      "thrive 552 558 ENTITY\n",
      "Feeding Difficulties 559 579 ENTITY\n",
      "eyes/proptosis Down slanting eyebrows 590 627 ENTITY\n",
      "ears 656 660 ENTITY\n",
      "Dysplastic 661 671 ENTITY\n",
      "Small ears 677 687 ENTITY\n",
      "overfolded 693 703 ENTITY\n",
      "helices/prominent helix 704 727 ENTITY\n",
      "Long philtrum Smooth philtrum Thin lips 728 767 ENTITY\n",
      "chin Pterygium 782 796 ENTITY\n",
      "Retrognatia/micrognathia 815 839 ENTITY\n",
      "Joint laxity 864 876 ENTITY\n",
      "Clinodactyly Tapering digits 877 905 ENTITY\n",
      "Hernia 907 913 ENTITY\n",
      "High palate 914 925 ENTITY\n",
      "Cleft lip Cleft palate Transverse right palmar 926 972 ENTITY\n",
      "crease/deep oakmar creases Pectus 973 1006 ENTITY\n",
      "deformity 1024 1033 ENTITY\n",
      "Radial abnormalities 1034 1054 ENTITY\n",
      "Hypotonia Dystonia 1070 1088 ENTITY\n",
      "Frontal bossing Scaphocephaly 1089 1118 ENTITY\n",
      "metopic 1154 1161 ENTITY\n",
      "Large anterior fontanelle 1175 1200 ENTITY\n",
      "posterior 1250 1259 ENTITY\n",
      "Flaired 1285 1292 ENTITY\n",
      "straight eyebrows 1297 1314 ENTITY\n",
      "Synophris 1315 1324 ENTITY\n",
      "Upslanting/long 1325 1340 ENTITY\n",
      "palpebral fissures 1341 1359 ENTITY\n",
      "Heart defects 1360 1373 ENTITY\n",
      "ASD 1386 1389 ENTITY\n",
      "VSD 1391 1394 ENTITY\n",
      "PDA 1398 1401 ENTITY\n",
      "ASD 1403 1406 ENTITY\n",
      "VSD 1407 1410 ENTITY\n",
      "Cardiomegaly 1411 1423 ENTITY\n",
      "Persistent 1424 1434 ENTITY\n",
      "Embryonic Sructures 1435 1454 ENTITY\n",
      "nasal root 1502 1512 ENTITY\n",
      "Thin upper lip 1524 1538 ENTITY\n",
      "Large/wide mouth 1539 1555 ENTITY\n",
      "downturned corners 1578 1596 ENTITY\n",
      "lower 1621 1626 ENTITY\n",
      "Prominent 1631 1640 ENTITY\n",
      "Psychiatric diagnosis 1688 1709 ENTITY\n",
      "shizofrenia 1711 1722 ENTITY\n",
      "depression 1724 1734 ENTITY\n",
      "autism 1747 1753 ENTITY\n",
      "ADHD/ADD) 1758 1767 ENTITY\n",
      "Behavioral abnormalities 1768 1792 ENTITY\n",
      "tantrums 1794 1802 ENTITY\n",
      "auto mutulation 1804 1819 ENTITY\n",
      "Horseshoe kidney 1821 1837 ENTITY\n",
      "Renal abnormality 1856 1873 ENTITY\n",
      "Cryptorchidism 1874 1888 ENTITY\n",
      "Recurrent 1889 1898 ENTITY\n",
      "infections 1899 1909 ENTITY\n",
      "Photosensitivity 1910 1926 ENTITY\n",
      "Hypothyroidism 1955 1969 ENTITY\n",
      "Leukocytose 1970 1981 ENTITY\n",
      "Trombocytopenie Asthma Strabismus 1982 2015 ENTITY\n",
      "Growth hormone deficiency 2016 2041 ENTITY\n"
     ]
    }
   ],
   "source": [
    "for ent in doc.ents:\n",
    "    print(ent.text, ent.start_char, ent.end_char, ent.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add extra entities to based on HPO gold standard corpus from Bio-Lark\n",
    "REF: http://www.bio-lark.org/hpo_res.html\n",
    "\n",
    " 228 manually annotated abstracts cited by the Online Mendelian Inheritance in Man (OMIM) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A syndrome of brachydactyly (absence of some middle or distal phalanges), aplastic or hypoplastic nails, symphalangism (ankylois of proximal interphalangeal joints), synostosis of some carpal and tarsal bones, craniosynostosis, and dysplastic hip joints is reported in five members of an Italian family. It may represent a previously undescribed autosomal dominant trait.'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('48', '66', 'ENTITY')"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TRAIN_DATA[23][1]['entities'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "228\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import ast\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "path_annot = 'corpus/stand-off/'\n",
    "path_text = 'corpus/text/'\n",
    "\n",
    "annot_files = [f for f in listdir(path_annot) if isfile(join(path_annot, f))]\n",
    "text_files = [f for f in listdir(path_text) if isfile(join(path_text, f))]\n",
    "#print(annot_files)\n",
    "\n",
    "\n",
    "TRAIN_DATA = []\n",
    "#i = 0\n",
    "\n",
    "for i in range(len(annot_files)):\n",
    "    with open(path_annot + annot_files[i], 'r') as f:\n",
    "        content = f.readlines()\n",
    "    content = [x.strip() for x in content] \n",
    "\n",
    "    with open(path_text + text_files[i], 'r') as f:\n",
    "        text = f.read().strip()\n",
    "        text = re.sub(r\"[^\\x00-\\x7F\\']+\",' ', text)\n",
    "    \n",
    "    list_ent = []\n",
    "    for match in content:\n",
    "        m = re.findall('\\[(\\d+)::(\\d+)\\].*\\|\\s(.*)', match)[0]\n",
    "        if list_ent == []:\n",
    "            list_ent.append((int(m[0]), int(m[1]), \"ENTITY\"))\n",
    "        elif int(m[0]) > list_ent[-1][1]:\n",
    "            list_ent.append((int(m[0]), int(m[1]), \"ENTITY\"))\n",
    "    pattern = {\"entities\": list_ent}\n",
    "    #print((text, pattern))\n",
    "    TRAIN_DATA.append((text, pattern))\n",
    "    #print(i)\n",
    "    #break\n",
    "print(len(TRAIN_DATA))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(TRAIN_DATA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['485', '500', 'ENTITY'], dtype='<U11')"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(list_ent)[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['17', '48', '67', '90', '139', '161', '222', '239', '265', '298',\n",
       "       '315', '383', '428', '485'], dtype='<U11')"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(list_ent)[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting the pipeline component\n",
    "ner=nlp.get_pipe(\"ner\")\n",
    "\n",
    "for _, annotations in TRAIN_DATA:\n",
    "    for ent in annotations.get(\"entities\"):\n",
    "        ner.add_label(ent[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tagger', <spacy.pipeline.pipes.Tagger at 0x1f856518e10>),\n",
       " ('parser', <spacy.pipeline.pipes.DependencyParser at 0x1f8bb0e50a8>),\n",
       " ('ner', <spacy.pipeline.pipes.EntityRecognizer at 0x1f8bb0e5108>),\n",
       " ('AbbreviationDetector',\n",
       "  <scispacy.abbreviation.AbbreviationDetector at 0x1f90d6610f0>),\n",
       " ('EntityLinker', <scispacy.linking.EntityLinker at 0x1f856518d30>)]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Disable pipeline components you dont need to change\n",
    "pipe_exceptions = [\"ner\"]\n",
    "unaffected_pipes = [pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ITERATION: 0:\n",
      "Losses {'ner': 35.036204637285664}\n",
      "Losses {'ner': 44.02100002202862}\n",
      "Losses {'ner': 59.77458034993108}\n",
      "Losses {'ner': 94.32432258020609}\n",
      "Losses {'ner': 110.56919562018712}\n",
      "Losses {'ner': 138.41004795517998}\n",
      "Losses {'ner': 166.0835644640703}\n",
      "Losses {'ner': 200.93594663503805}\n",
      "Losses {'ner': 217.85332694679724}\n",
      "Losses {'ner': 241.1772115074074}\n",
      "Losses {'ner': 275.4585195047058}\n",
      "Losses {'ner': 297.1602318736933}\n",
      "Losses {'ner': 328.19201047871326}\n",
      "Losses {'ner': 352.6637017983813}\n",
      "Losses {'ner': 375.54548514413915}\n",
      "Losses {'ner': 410.83691697585806}\n",
      "Losses {'ner': 428.85813329165745}\n",
      "Losses {'ner': 452.60591103740643}\n",
      "Losses {'ner': 480.48671401401424}\n",
      "Losses {'ner': 494.939742868436}\n",
      "Losses {'ner': 532.6810089579276}\n",
      "Losses {'ner': 570.1728318442231}\n",
      "Losses {'ner': 586.5167868276486}\n",
      "Losses {'ner': 619.6466234408155}\n",
      "Losses {'ner': 648.90989517486}\n",
      "Losses {'ner': 665.2680721053738}\n",
      "Losses {'ner': 675.2681605637889}\n",
      "Losses {'ner': 699.8523403380649}\n",
      "Losses {'ner': 717.821057793445}\n",
      "Losses {'ner': 751.6370819968805}\n",
      "Losses {'ner': 778.9749726751987}\n",
      "Losses {'ner': 806.1832922809489}\n",
      "Losses {'ner': 825.2846959313364}\n",
      "Losses {'ner': 852.4039976125104}\n",
      "Losses {'ner': 868.808018544606}\n",
      "Losses {'ner': 882.911480165295}\n",
      "Losses {'ner': 918.1847357340203}\n",
      "Losses {'ner': 945.0264584664883}\n",
      "Losses {'ner': 960.5677802122709}\n",
      "Losses {'ner': 987.0692867652433}\n",
      "Losses {'ner': 1005.4832838567473}\n",
      "Losses {'ner': 1018.1921746021636}\n",
      "Losses {'ner': 1048.8755726069958}\n",
      "Losses {'ner': 1064.2492449132426}\n",
      "Losses {'ner': 1068.6826947463792}\n",
      "Losses {'ner': 1087.2939503344746}\n",
      "Losses {'ner': 1124.492185678786}\n",
      "Losses {'ner': 1148.3316365638786}\n",
      "Losses {'ner': 1170.0447146939186}\n",
      "Losses {'ner': 1175.1239699629728}\n",
      "Losses {'ner': 1200.909284936129}\n",
      "Losses {'ner': 1243.1027974662534}\n",
      "Losses {'ner': 1254.2437817636014}\n",
      "Losses {'ner': 1266.1073088296225}\n",
      "Losses {'ner': 1282.7686862107184}\n",
      "Losses {'ner': 1299.4665167435135}\n",
      "Losses {'ner': 1325.3588246826087}\n",
      "ITERATION: 1:\n",
      "Losses {'ner': 20.485256021707755}\n",
      "Losses {'ner': 29.580181936152712}\n",
      "Losses {'ner': 44.26596134603648}\n",
      "Losses {'ner': 66.95296050031362}\n",
      "Losses {'ner': 88.05976187471634}\n",
      "Losses {'ner': 138.20104739732443}\n",
      "Losses {'ner': 164.18601608191005}\n",
      "Losses {'ner': 171.74966258830418}\n",
      "Losses {'ner': 199.71052222368334}\n",
      "Losses {'ner': 227.017372198473}\n",
      "Losses {'ner': 257.39644886469}\n",
      "Losses {'ner': 297.6369092903053}\n",
      "Losses {'ner': 313.2929828119258}\n",
      "Losses {'ner': 351.2814101849259}\n",
      "Losses {'ner': 380.41394962207835}\n",
      "Losses {'ner': 394.8237315369066}\n",
      "Losses {'ner': 412.47286596774813}\n",
      "Losses {'ner': 444.5638945609377}\n",
      "Losses {'ner': 460.5998613710893}\n",
      "Losses {'ner': 464.4314368574213}\n",
      "Losses {'ner': 476.1179151433022}\n",
      "Losses {'ner': 498.6125281631827}\n",
      "Losses {'ner': 526.19352064206}\n",
      "Losses {'ner': 550.9089827164345}\n",
      "Losses {'ner': 566.1332016458342}\n",
      "Losses {'ner': 606.5743289429244}\n",
      "Losses {'ner': 645.6807391105575}\n",
      "Losses {'ner': 655.5499087520099}\n",
      "Losses {'ner': 689.3034896709487}\n",
      "Losses {'ner': 718.1550591456235}\n",
      "Losses {'ner': 735.5436599100703}\n",
      "Losses {'ner': 749.1848181519719}\n",
      "Losses {'ner': 776.0465586029869}\n",
      "Losses {'ner': 804.0548388346818}\n",
      "Losses {'ner': 829.923185281012}\n",
      "Losses {'ner': 851.6073850527149}\n",
      "Losses {'ner': 867.1774280520979}\n",
      "Losses {'ner': 877.1853150160161}\n",
      "Losses {'ner': 896.5613239429379}\n",
      "Losses {'ner': 927.3467390582574}\n",
      "Losses {'ner': 942.1432761133797}\n",
      "Losses {'ner': 973.3991933655593}\n",
      "Losses {'ner': 988.1958211130036}\n",
      "Losses {'ner': 1013.6582539031765}\n",
      "Losses {'ner': 1033.268704269563}\n",
      "Losses {'ner': 1040.517791999987}\n",
      "Losses {'ner': 1070.566940599751}\n",
      "Losses {'ner': 1100.7647573034028}\n",
      "Losses {'ner': 1140.829796764072}\n",
      "Losses {'ner': 1154.89125782048}\n",
      "Losses {'ner': 1164.0088701733032}\n",
      "Losses {'ner': 1168.8686632330046}\n",
      "Losses {'ner': 1215.576914485718}\n",
      "Losses {'ner': 1246.0636559708503}\n",
      "Losses {'ner': 1280.9046818129111}\n",
      "Losses {'ner': 1302.4845638151482}\n",
      "Losses {'ner': 1344.4208856151083}\n",
      "ITERATION: 2:\n",
      "Losses {'ner': 21.29599665886201}\n",
      "Losses {'ner': 37.827411527057095}\n",
      "Losses {'ner': 73.73503522226672}\n",
      "Losses {'ner': 87.43705497962507}\n",
      "Losses {'ner': 95.45644350416173}\n",
      "Losses {'ner': 115.87960244245973}\n",
      "Losses {'ner': 131.52445890867654}\n",
      "Losses {'ner': 163.82637510265027}\n",
      "Losses {'ner': 188.6268683504896}\n",
      "Losses {'ner': 208.03789376809442}\n",
      "Losses {'ner': 237.19217717587895}\n",
      "Losses {'ner': 257.5403104529319}\n",
      "Losses {'ner': 316.31315975671856}\n",
      "Losses {'ner': 333.0068837807117}\n",
      "Losses {'ner': 339.2575255541479}\n",
      "Losses {'ner': 359.10260531782393}\n",
      "Losses {'ner': 371.85696257102524}\n",
      "Losses {'ner': 403.71960946926674}\n",
      "Losses {'ner': 426.3854002521159}\n",
      "Losses {'ner': 445.1649041777248}\n",
      "Losses {'ner': 480.6535931756491}\n",
      "Losses {'ner': 513.067090290966}\n",
      "Losses {'ner': 538.6323947072196}\n",
      "Losses {'ner': 550.9955917369169}\n",
      "Losses {'ner': 574.6541504503347}\n",
      "Losses {'ner': 599.4420216887423}\n",
      "Losses {'ner': 634.0744111548481}\n",
      "Losses {'ner': 677.755144160184}\n",
      "Losses {'ner': 696.9374948544587}\n",
      "Losses {'ner': 746.9214625357474}\n",
      "Losses {'ner': 757.5077085736723}\n",
      "Losses {'ner': 769.6030058725382}\n",
      "Losses {'ner': 807.0902550585824}\n",
      "Losses {'ner': 877.0906899317424}\n",
      "Losses {'ner': 933.6216359396722}\n",
      "Losses {'ner': 981.4789615964}\n",
      "Losses {'ner': 1011.337301718846}\n",
      "Losses {'ner': 1058.41472061702}\n",
      "Losses {'ner': 1100.2650885952353}\n",
      "Losses {'ner': 1129.4625202260395}\n",
      "Losses {'ner': 1164.176269321837}\n",
      "Losses {'ner': 1193.76192214617}\n",
      "Losses {'ner': 1215.4815470290464}\n",
      "Losses {'ner': 1256.0142289171322}\n",
      "Losses {'ner': 1282.8235170684325}\n",
      "Losses {'ner': 1321.2913327469992}\n",
      "Losses {'ner': 1337.6698280404662}\n",
      "Losses {'ner': 1363.950339367394}\n",
      "Losses {'ner': 1397.9504955826947}\n",
      "Losses {'ner': 1420.952163412204}\n",
      "Losses {'ner': 1436.55915622637}\n",
      "Losses {'ner': 1475.6695967918615}\n",
      "Losses {'ner': 1499.1444997659194}\n",
      "Losses {'ner': 1544.3596359698342}\n",
      "Losses {'ner': 1559.4000007219172}\n",
      "Losses {'ner': 1596.3803561303507}\n",
      "Losses {'ner': 1612.1375901294764}\n",
      "ITERATION: 3:\n",
      "Losses {'ner': 21.019649989596708}\n",
      "Losses {'ner': 39.64805664115695}\n",
      "Losses {'ner': 51.21625032630515}\n",
      "Losses {'ner': 111.3245092379542}\n",
      "Losses {'ner': 132.1283261227103}\n",
      "Losses {'ner': 157.30223625958888}\n",
      "Losses {'ner': 199.5817755709445}\n",
      "Losses {'ner': 243.74945201215223}\n",
      "Losses {'ner': 264.5657952856013}\n",
      "Losses {'ner': 278.55658734005146}\n",
      "Losses {'ner': 301.6418354480152}\n",
      "Losses {'ner': 329.361399314374}\n",
      "Losses {'ner': 346.2395012176207}\n",
      "Losses {'ner': 373.59933703009074}\n",
      "Losses {'ner': 402.37869785377643}\n",
      "Losses {'ner': 428.6716106171598}\n",
      "Losses {'ner': 443.94168653336396}\n",
      "Losses {'ner': 464.4197760241724}\n",
      "Losses {'ner': 474.975965124867}\n",
      "Losses {'ner': 497.95737785534914}\n",
      "Losses {'ner': 519.0158672879076}\n",
      "Losses {'ner': 542.8837532082431}\n",
      "Losses {'ner': 606.9159933778316}\n",
      "Losses {'ner': 656.2948787142847}\n",
      "Losses {'ner': 674.3726817814746}\n",
      "Losses {'ner': 700.4368364865073}\n",
      "Losses {'ner': 710.394821851965}\n",
      "Losses {'ner': 748.1902063413432}\n",
      "Losses {'ner': 785.9855342446532}\n",
      "Losses {'ner': 800.405757789718}\n",
      "Losses {'ner': 822.898024451385}\n",
      "Losses {'ner': 852.747724876973}\n",
      "Losses {'ner': 865.4920826370318}\n",
      "Losses {'ner': 896.4240462757126}\n",
      "Losses {'ner': 938.0093633821936}\n",
      "Losses {'ner': 956.1905074751022}\n",
      "Losses {'ner': 970.8662742032715}\n",
      "Losses {'ner': 993.9519490370106}\n",
      "Losses {'ner': 1014.4627570153361}\n",
      "Losses {'ner': 1040.6333736026545}\n",
      "Losses {'ner': 1061.54755644031}\n",
      "Losses {'ner': 1087.4270105665423}\n",
      "Losses {'ner': 1114.696191577787}\n",
      "Losses {'ner': 1130.506419901095}\n",
      "Losses {'ner': 1139.4980947595745}\n",
      "Losses {'ner': 1164.2703874592394}\n",
      "Losses {'ner': 1184.2564052814064}\n",
      "Losses {'ner': 1201.4044766391441}\n",
      "Losses {'ner': 1222.3139777192625}\n",
      "Losses {'ner': 1235.244275497327}\n",
      "Losses {'ner': 1254.3752383353126}\n",
      "Losses {'ner': 1272.4896626595096}\n",
      "Losses {'ner': 1296.4854681758102}\n",
      "Losses {'ner': 1323.7186616774147}\n",
      "Losses {'ner': 1339.597077565008}\n",
      "Losses {'ner': 1391.1092706299119}\n",
      "Losses {'ner': 1415.7913045331375}\n",
      "ITERATION: 4:\n",
      "Losses {'ner': 16.263211715185967}\n",
      "Losses {'ner': 25.9828947738014}\n",
      "Losses {'ner': 33.87417773590272}\n",
      "Losses {'ner': 60.25210581217871}\n",
      "Losses {'ner': 89.33945527812139}\n",
      "Losses {'ner': 103.55589164972153}\n",
      "Losses {'ner': 117.31356923354124}\n",
      "Losses {'ner': 134.4046321968175}\n",
      "Losses {'ner': 155.8969112472728}\n",
      "Losses {'ner': 199.34098841001142}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Losses {'ner': 237.4895221417104}\n",
      "Losses {'ner': 257.6626754090464}\n",
      "Losses {'ner': 271.3900318662336}\n",
      "Losses {'ner': 289.136709933372}\n",
      "Losses {'ner': 305.88185304533454}\n",
      "Losses {'ner': 325.60243634296967}\n",
      "Losses {'ner': 359.08682298845633}\n",
      "Losses {'ner': 404.53517239585017}\n",
      "Losses {'ner': 437.2320951661586}\n",
      "Losses {'ner': 455.0725806162045}\n",
      "Losses {'ner': 488.192143811244}\n",
      "Losses {'ner': 512.7375799977272}\n",
      "Losses {'ner': 540.396782079606}\n",
      "Losses {'ner': 553.8180622289822}\n",
      "Losses {'ner': 562.3736414897887}\n",
      "Losses {'ner': 573.3535580944902}\n",
      "Losses {'ner': 580.8720345176864}\n",
      "Losses {'ner': 585.4042777516422}\n",
      "Losses {'ner': 603.1812977632709}\n",
      "Losses {'ner': 630.7897320618196}\n",
      "Losses {'ner': 655.6510019667883}\n",
      "Losses {'ner': 677.7811351912067}\n",
      "Losses {'ner': 715.2195803028367}\n",
      "Losses {'ner': 742.5464370109908}\n",
      "Losses {'ner': 775.3123496763287}\n",
      "Losses {'ner': 792.1909906967832}\n",
      "Losses {'ner': 814.1480727722284}\n",
      "Losses {'ner': 830.6373353226193}\n",
      "Losses {'ner': 846.9196784262538}\n",
      "Losses {'ner': 884.4877865570131}\n",
      "Losses {'ner': 907.1804226105396}\n",
      "Losses {'ner': 921.1039313704878}\n",
      "Losses {'ner': 940.4914050611136}\n",
      "Losses {'ner': 970.7665879758009}\n",
      "Losses {'ner': 999.4623850293741}\n",
      "Losses {'ner': 1020.5442073377874}\n",
      "Losses {'ner': 1040.1488654319394}\n",
      "Losses {'ner': 1068.206138956017}\n",
      "Losses {'ner': 1106.525249408793}\n",
      "Losses {'ner': 1141.826749949951}\n",
      "Losses {'ner': 1169.5810958898596}\n",
      "Losses {'ner': 1193.853866640458}\n",
      "Losses {'ner': 1212.3905022141932}\n",
      "Losses {'ner': 1245.5131404713286}\n",
      "Losses {'ner': 1285.0271421561285}\n",
      "Losses {'ner': 1302.7286489532364}\n",
      "Losses {'ner': 1345.930840458774}\n",
      "ITERATION: 5:\n",
      "Losses {'ner': 37.91861903730867}\n",
      "Losses {'ner': 68.94633363087749}\n",
      "Losses {'ner': 92.68519018264166}\n",
      "Losses {'ner': 114.31153401194513}\n",
      "Losses {'ner': 157.60031510095484}\n",
      "Losses {'ner': 175.90930351751138}\n",
      "Losses {'ner': 194.42187946693718}\n",
      "Losses {'ner': 207.90968569519953}\n",
      "Losses {'ner': 234.75449282082752}\n",
      "Losses {'ner': 272.89984829503135}\n",
      "Losses {'ner': 290.7815424304724}\n",
      "Losses {'ner': 323.44013218082614}\n",
      "Losses {'ner': 352.1808404350145}\n",
      "Losses {'ner': 368.9942393379846}\n",
      "Losses {'ner': 383.48822539777973}\n",
      "Losses {'ner': 413.3967722469823}\n",
      "Losses {'ner': 448.2829885036799}\n",
      "Losses {'ner': 464.56369164265027}\n",
      "Losses {'ner': 473.8813486534874}\n",
      "Losses {'ner': 506.80940140042287}\n",
      "Losses {'ner': 520.7130646841755}\n",
      "Losses {'ner': 556.7712048115968}\n",
      "Losses {'ner': 566.401382990128}\n",
      "Losses {'ner': 600.122197901117}\n",
      "Losses {'ner': 624.4100956195725}\n",
      "Losses {'ner': 634.1704836198805}\n",
      "Losses {'ner': 671.362755673912}\n",
      "Losses {'ner': 697.4346313463244}\n",
      "Losses {'ner': 714.3173702853015}\n",
      "Losses {'ner': 728.8710590787872}\n",
      "Losses {'ner': 763.8885157991375}\n",
      "Losses {'ner': 801.9489879849584}\n",
      "Losses {'ner': 834.8390053747212}\n",
      "Losses {'ner': 854.0358278701028}\n",
      "Losses {'ner': 864.4883001342765}\n",
      "Losses {'ner': 878.6314181697031}\n",
      "Losses {'ner': 899.6417162011436}\n",
      "Losses {'ner': 924.0371353884053}\n",
      "Losses {'ner': 933.0384954797948}\n",
      "Losses {'ner': 982.8873353052797}\n",
      "Losses {'ner': 1020.641268568844}\n",
      "Losses {'ner': 1038.032146472516}\n",
      "Losses {'ner': 1087.6649238677085}\n",
      "Losses {'ner': 1108.5590413638079}\n",
      "Losses {'ner': 1123.6679112622246}\n",
      "Losses {'ner': 1133.5397539798118}\n",
      "Losses {'ner': 1167.9817368326817}\n",
      "Losses {'ner': 1205.9074467273163}\n",
      "Losses {'ner': 1215.1616719532271}\n",
      "Losses {'ner': 1247.723767757367}\n",
      "Losses {'ner': 1272.6502424013056}\n",
      "Losses {'ner': 1284.1787804972341}\n",
      "Losses {'ner': 1292.308237911695}\n",
      "Losses {'ner': 1318.0525448206113}\n",
      "Losses {'ner': 1340.191856689984}\n",
      "Losses {'ner': 1346.2686263490984}\n",
      "Losses {'ner': 1361.650599975627}\n",
      "ITERATION: 6:\n",
      "Losses {'ner': 13.558461031683004}\n",
      "Losses {'ner': 22.37142528047746}\n",
      "Losses {'ner': 36.28564893196988}\n",
      "Losses {'ner': 56.432151447106015}\n",
      "Losses {'ner': 77.23192248790448}\n",
      "Losses {'ner': 92.56531863899792}\n",
      "Losses {'ner': 102.4503435987516}\n",
      "Losses {'ner': 134.2527663019954}\n",
      "Losses {'ner': 148.9996875370663}\n",
      "Losses {'ner': 198.35309611002882}\n",
      "Losses {'ner': 210.34691526534598}\n",
      "Losses {'ner': 219.56634159721298}\n",
      "Losses {'ner': 225.76341714964576}\n",
      "Losses {'ner': 246.83342441686466}\n",
      "Losses {'ner': 275.3897802523926}\n",
      "Losses {'ner': 299.1673645668153}\n",
      "Losses {'ner': 319.94442847304254}\n",
      "Losses {'ner': 343.8198689003775}\n",
      "Losses {'ner': 358.03533973992046}\n",
      "Losses {'ner': 394.55848745542767}\n",
      "Losses {'ner': 401.11829127696393}\n",
      "Losses {'ner': 410.56210209361876}\n",
      "Losses {'ner': 430.03124656410375}\n",
      "Losses {'ner': 453.06818634129473}\n",
      "Losses {'ner': 474.2038732138169}\n",
      "Losses {'ner': 487.94620590110776}\n",
      "Losses {'ner': 492.7797043223542}\n",
      "Losses {'ner': 501.2577718497265}\n",
      "Losses {'ner': 534.3931414372126}\n",
      "Losses {'ner': 562.2090547171791}\n",
      "Losses {'ner': 574.8407757513406}\n",
      "Losses {'ner': 609.994072533575}\n",
      "Losses {'ner': 619.3156078014422}\n",
      "Losses {'ner': 670.2192520303856}\n",
      "Losses {'ner': 705.2613879282253}\n",
      "Losses {'ner': 724.1156780073829}\n",
      "Losses {'ner': 770.832700916671}\n",
      "Losses {'ner': 827.7928803122536}\n",
      "Losses {'ner': 845.6214161859293}\n",
      "Losses {'ner': 857.7834086039998}\n",
      "Losses {'ner': 933.732631629789}\n",
      "Losses {'ner': 964.0952462102191}\n",
      "Losses {'ner': 999.0528814482727}\n",
      "Losses {'ner': 1003.9858391480536}\n",
      "Losses {'ner': 1052.3752111143056}\n",
      "Losses {'ner': 1066.3875339285094}\n",
      "Losses {'ner': 1095.20624775388}\n",
      "Losses {'ner': 1136.3555224043319}\n",
      "Losses {'ner': 1147.5304022539874}\n",
      "Losses {'ner': 1168.4883541154702}\n",
      "Losses {'ner': 1185.4563034181044}\n",
      "Losses {'ner': 1211.8082263611818}\n",
      "Losses {'ner': 1238.0647868163178}\n",
      "Losses {'ner': 1283.1260966605732}\n",
      "Losses {'ner': 1316.2496536915414}\n",
      "Losses {'ner': 1343.3055417415703}\n",
      "Losses {'ner': 1358.4272467775804}\n",
      "ITERATION: 7:\n",
      "Losses {'ner': 21.43018421312445}\n",
      "Losses {'ner': 33.56640170303285}\n",
      "Losses {'ner': 37.50322753715787}\n",
      "Losses {'ner': 75.45127594539932}\n",
      "Losses {'ner': 114.49894732679698}\n",
      "Losses {'ner': 149.8490133034293}\n",
      "Losses {'ner': 157.36350065912123}\n",
      "Losses {'ner': 167.53422942063952}\n",
      "Losses {'ner': 210.44110063514853}\n",
      "Losses {'ner': 234.04826451917336}\n",
      "Losses {'ner': 244.04390124882696}\n",
      "Losses {'ner': 254.66334301190648}\n",
      "Losses {'ner': 273.68151728515124}\n",
      "Losses {'ner': 293.98116052743546}\n",
      "Losses {'ner': 310.065717356993}\n",
      "Losses {'ner': 318.6960380736668}\n",
      "Losses {'ner': 353.17590529046186}\n",
      "Losses {'ner': 366.1514308543684}\n",
      "Losses {'ner': 408.53995249810225}\n",
      "Losses {'ner': 414.1304156247838}\n",
      "Losses {'ner': 428.1488760743324}\n",
      "Losses {'ner': 449.8095362781475}\n",
      "Losses {'ner': 456.11672708605226}\n",
      "Losses {'ner': 484.87254072409166}\n",
      "Losses {'ner': 499.3474081914992}\n",
      "Losses {'ner': 510.5392257760482}\n",
      "Losses {'ner': 526.9200384305908}\n",
      "Losses {'ner': 541.9839075511005}\n",
      "Losses {'ner': 562.5041836036721}\n",
      "Losses {'ner': 581.0655546302328}\n",
      "Losses {'ner': 595.2970668374483}\n",
      "Losses {'ner': 598.8937274816237}\n",
      "Losses {'ner': 603.3473248032835}\n",
      "Losses {'ner': 635.2764054135107}\n",
      "Losses {'ner': 647.9306841881726}\n",
      "Losses {'ner': 662.3905238975174}\n",
      "Losses {'ner': 677.3086709561757}\n",
      "Losses {'ner': 709.2319534458219}\n",
      "Losses {'ner': 735.479486874618}\n",
      "Losses {'ner': 754.5155191947468}\n",
      "Losses {'ner': 778.5305466222106}\n",
      "Losses {'ner': 803.4281804401202}\n",
      "Losses {'ner': 844.141492683419}\n",
      "Losses {'ner': 855.6474418046021}\n",
      "Losses {'ner': 880.1918463943259}\n",
      "Losses {'ner': 890.5941721319361}\n",
      "Losses {'ner': 911.0411715430348}\n",
      "Losses {'ner': 943.8637740759543}\n",
      "Losses {'ner': 965.2921562216864}\n",
      "Losses {'ner': 985.2885714049826}\n",
      "Losses {'ner': 1028.7405385242396}\n",
      "Losses {'ner': 1078.6978960714855}\n",
      "Losses {'ner': 1090.0463258215693}\n",
      "Losses {'ner': 1113.321993207757}\n",
      "Losses {'ner': 1125.9990164773558}\n",
      "Losses {'ner': 1143.7461639617763}\n",
      "Losses {'ner': 1154.1579015759712}\n",
      "ITERATION: 8:\n",
      "Losses {'ner': 3.1083376529070215}\n",
      "Losses {'ner': 25.444492135337878}\n",
      "Losses {'ner': 56.331129591629164}\n",
      "Losses {'ner': 75.58133248208398}\n",
      "Losses {'ner': 94.71355798882107}\n",
      "Losses {'ner': 114.65537221887755}\n",
      "Losses {'ner': 134.4563364913733}\n",
      "Losses {'ner': 144.58370321782553}\n",
      "Losses {'ner': 161.87433475226973}\n",
      "Losses {'ner': 172.8310028496646}\n",
      "Losses {'ner': 190.24141783172783}\n",
      "Losses {'ner': 211.86830485900393}\n",
      "Losses {'ner': 239.1104033803466}\n",
      "Losses {'ner': 265.1000092872481}\n",
      "Losses {'ner': 287.83429271300963}\n",
      "Losses {'ner': 322.42944482716615}\n",
      "Losses {'ner': 336.20399922781274}\n",
      "Losses {'ner': 351.3896840521027}\n",
      "Losses {'ner': 363.5987700554668}\n",
      "Losses {'ner': 384.3023256769522}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Losses {'ner': 414.23954378272555}\n",
      "Losses {'ner': 443.75489547611886}\n",
      "Losses {'ner': 449.45142920555753}\n",
      "Losses {'ner': 469.358690887955}\n",
      "Losses {'ner': 512.7802467492036}\n",
      "Losses {'ner': 534.5030224675172}\n",
      "Losses {'ner': 549.3580444723591}\n",
      "Losses {'ner': 571.7482146190703}\n",
      "Losses {'ner': 592.6080040781042}\n",
      "Losses {'ner': 619.0064013621547}\n",
      "Losses {'ner': 653.6184858951137}\n",
      "Losses {'ner': 683.1511581426403}\n",
      "Losses {'ner': 723.672038715029}\n",
      "Losses {'ner': 757.0936225919413}\n",
      "Losses {'ner': 780.0094728657996}\n",
      "Losses {'ner': 796.3416391817873}\n",
      "Losses {'ner': 812.068992257452}\n",
      "Losses {'ner': 821.6073346720569}\n",
      "Losses {'ner': 833.1439944560037}\n",
      "Losses {'ner': 854.3666698148934}\n",
      "Losses {'ner': 882.420153209737}\n",
      "Losses {'ner': 917.560257730803}\n",
      "Losses {'ner': 918.5415382968648}\n",
      "Losses {'ner': 939.118166246072}\n",
      "Losses {'ner': 955.8529177925218}\n",
      "Losses {'ner': 982.7828985223892}\n",
      "Losses {'ner': 1011.0531235044504}\n",
      "Losses {'ner': 1040.4413589787177}\n",
      "Losses {'ner': 1055.5490011893503}\n",
      "Losses {'ner': 1072.147753268067}\n",
      "Losses {'ner': 1090.9188474365988}\n",
      "Losses {'ner': 1111.0422076647872}\n",
      "Losses {'ner': 1131.4683513253494}\n",
      "Losses {'ner': 1155.1341258650446}\n",
      "Losses {'ner': 1166.3199110479136}\n",
      "Losses {'ner': 1200.0998208438732}\n",
      "Losses {'ner': 1211.910580715217}\n",
      "ITERATION: 9:\n",
      "Losses {'ner': 11.207400688371706}\n",
      "Losses {'ner': 48.180197131743114}\n",
      "Losses {'ner': 64.7511727689443}\n",
      "Losses {'ner': 86.25141279751092}\n",
      "Losses {'ner': 126.79839025137603}\n",
      "Losses {'ner': 153.6236093917641}\n",
      "Losses {'ner': 177.9247986051401}\n",
      "Losses {'ner': 183.36516627680118}\n",
      "Losses {'ner': 204.04180138020087}\n",
      "Losses {'ner': 209.8931686641917}\n",
      "Losses {'ner': 221.58205575952383}\n",
      "Losses {'ner': 231.62756007030424}\n",
      "Losses {'ner': 248.67408957756874}\n",
      "Losses {'ner': 266.2998857752191}\n",
      "Losses {'ner': 283.2434976233217}\n",
      "Losses {'ner': 293.0876614443114}\n",
      "Losses {'ner': 328.9293174698105}\n",
      "Losses {'ner': 344.39074341171914}\n",
      "Losses {'ner': 362.6128151748796}\n",
      "Losses {'ner': 386.96864999367847}\n",
      "Losses {'ner': 406.2047499084258}\n",
      "Losses {'ner': 420.9435719469093}\n",
      "Losses {'ner': 428.36536790805684}\n",
      "Losses {'ner': 447.741525199626}\n",
      "Losses {'ner': 487.125987089739}\n",
      "Losses {'ner': 511.0144868524407}\n",
      "Losses {'ner': 523.2331882272902}\n",
      "Losses {'ner': 539.6148146469104}\n",
      "Losses {'ner': 552.0188422488212}\n",
      "Losses {'ner': 574.3446815950224}\n",
      "Losses {'ner': 600.291648042301}\n",
      "Losses {'ner': 636.4390202474252}\n",
      "Losses {'ner': 650.1120702539589}\n",
      "Losses {'ner': 672.5550024783698}\n",
      "Losses {'ner': 686.611287405187}\n",
      "Losses {'ner': 702.6812744828251}\n",
      "Losses {'ner': 731.9919221940938}\n",
      "Losses {'ner': 746.5301264406805}\n",
      "Losses {'ner': 758.2916571813265}\n",
      "Losses {'ner': 781.1580346359445}\n",
      "Losses {'ner': 791.1257283579349}\n",
      "Losses {'ner': 813.3170794531297}\n",
      "Losses {'ner': 840.2318245988981}\n",
      "Losses {'ner': 856.4151069425684}\n",
      "Losses {'ner': 864.9387670026933}\n",
      "Losses {'ner': 892.4170719136325}\n",
      "Losses {'ner': 913.1697857046711}\n",
      "Losses {'ner': 926.9371254499347}\n",
      "Losses {'ner': 941.9701346704564}\n",
      "Losses {'ner': 950.60168744562}\n",
      "Losses {'ner': 965.9750465316702}\n",
      "Losses {'ner': 983.2476763989864}\n",
      "Losses {'ner': 1008.3861677194889}\n",
      "Losses {'ner': 1024.2385532124815}\n",
      "Losses {'ner': 1035.1016466725766}\n",
      "Losses {'ner': 1048.6241072335592}\n",
      "Losses {'ner': 1069.781075501926}\n"
     ]
    }
   ],
   "source": [
    "# Import requirements\n",
    "import random\n",
    "from spacy.util import minibatch, compounding\n",
    "from pathlib import Path\n",
    "\n",
    "# TRAINING THE MODEL\n",
    "with nlp.disable_pipes(*unaffected_pipes):\n",
    "\n",
    "  # Training for 30 iterations\n",
    "  for iteration in range(10):\n",
    "    print('ITERATION: ' + str(iteration) + ':')\n",
    "    # shuufling examples  before every iteration\n",
    "    random.shuffle(TRAIN_DATA)\n",
    "    losses = {}\n",
    "    # batch up the examples using spaCy's minibatch\n",
    "    batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))\n",
    "    for batch in batches:\n",
    "        texts, annotations = zip(*batch)\n",
    "        nlp.update(\n",
    "                    texts,  # batch of texts\n",
    "                    annotations,  # batch of annotations\n",
    "                    drop=0.3,  # dropout - make it harder to memorise data\n",
    "                    losses=losses,\n",
    "                )\n",
    "        print(\"Losses\", losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp.to_disk('/tmp/my_model') # , disable=['tokenizer']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\levrex\\Anaconda3\\envs\\my_env\\lib\\site-packages\\scispacy\\candidate_generation.py:283: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  extended_neighbors[empty_vectors_boolean_flags] = numpy.array(neighbors)[:-1]\n",
      "C:\\Users\\levrex\\Anaconda3\\envs\\my_env\\lib\\site-packages\\scispacy\\candidate_generation.py:284: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  extended_distances[empty_vectors_boolean_flags] = numpy.array(distances)[:-1]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    intracranial bleeding high frequency hearing loss\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    short stature\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    wide nasal bridge\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ",  microcephaly  \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hearing loss\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "   id \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    developmental delay\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " (\n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    global)  motor delay speech delay\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "  \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    seizures  trigonocephaly\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "  \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    coloboma\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "  \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hypertelorism  high arched eyebrows\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hypertrichosis  congenital ptosis\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " epicanthus \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    telecanthus  pachygyria\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " other brain mri abnormalities (i.e. \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    brain atrophy\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    corpus callosum agenesis\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hydrocephalus\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    white matter leasions\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    periventricular nodular heterotopias\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "). long \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    eyelashes deeply set eyes broad/wide nasal bridge/ tip bulbous nose failure\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " to thrive feeding difficulties prominent \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    eyes/proptosis down slanting eyebrows\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    mid face hypoplasia\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    low set ears dysplastic ears small ears\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " with overfolded \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    helices/prominent helix long philtrum smooth philtrum thin lips pointed/small chin pterygium\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " colli/webbed \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    neck retrognatia/micrognathia kypho/scoliotic posture joint laxity clinodactyly\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    tapering digits\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "  hernia \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    high palate cleft lip\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    cleft palate transverse right palmar crease/deep oakmar creases pectus\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    excavatum/pectus deformity\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " radial abnormalities (fingers/toes) \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hypotonia\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    dystonia\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    frontal bossing scaphocephaly\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " bitemporal narrowing prominence of metopic suture/ridge \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    large anterior fontanelle high anterior hairline low anterior hairline low posterior hairline sparse eyebrows\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " flaired and \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    straight eyebrows synophris\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " upslanting/long \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    palpebral fissures\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    heart defects\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " (other then asd, vsd or pda) asd vsd cardiomegaly persistent embryonic sructures large squared nose tip \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    upturned nose\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    prominent nasal root\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " on profile thin upper lip large/wide \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    mouth tented upper lip\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " with \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    downturned corners\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " thick/prominent/everted \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    lower lip prominent cheeks autism\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " spectrum disorder adhd/add other psychiatric diagnosis (shizofrenia, depression, other, not \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    autism\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", or adhd/add) \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    behavioral abnormalities\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " (tantrums, auto mutulation) horseshoe kidney \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    renal cysts\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " other \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    renal abnormality cryptorchidism\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " recurrent \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    infections photosensitivity\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " polyarthralgia \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    hepatomegaly hypothyroidism\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " leukocytose trombocytopenie asthma \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    strabismus\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: linear-gradient(90deg, #aa9cfc, #fc9ce7); padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    growth hormone deficiency\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       "</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "text = 'Intracranial bleeding High frequency hearing loss Short stature, wide nasal bridge,  microcephaly  hearing loss   ID Developmental delay (Global)  Motor Delay Speech delay  seizures  trigonocephaly  coloboma  hypertelorism  high arched eyebrows Hypertrichosis  congenital ptosis Epicanthus Telecanthus  pachygyria Other brain MRI abnormalities (i.e. brain atrophy, corpus callosum agenesis, hydrocephalus, white matter leasions, periventricular nodular heterotopias). Long eyelashes Deeply set eyes Broad/wide nasal bridge/ tip Bulbous nose Failure to thrive Feeding Difficulties Prominent eyes/proptosis Down slanting eyebrows Mid face hypoplasia Low set ears Dysplastic ears Small ears with overfolded helices/prominent helix Long philtrum Smooth philtrum Thin lips Pointed/small chin Pterygium colli/webbed neck Retrognatia/micrognathia Kypho/scoliotic posture Joint laxity Clinodactyly Tapering digits \\xa0Hernia High palate Cleft lip Cleft palate Transverse right palmar crease/deep oakmar creases Pectus excavatum/pectus deformity Radial abnormalities (fingers/toes) Hypotonia Dystonia Frontal bossing Scaphocephaly Bitemporal narrowing Prominence of metopic suture/ridge Large anterior fontanelle High anterior hairline Low anterior hairline Low posterior hairline Sparse eyebrows Flaired and straight eyebrows Synophris Upslanting/long palpebral fissures Heart defects (Other then ASD, VSD or PDA) ASD VSD Cardiomegaly Persistent Embryonic Sructures Large squared nose tip upturned nose Prominent nasal root on profile Thin upper lip Large/wide mouth tented upper lip with downturned corners Thick/prominent/everted lower lip Prominent cheeks Autism Spectrum disorder ADHD/ADD Other Psychiatric diagnosis (shizofrenia, depression, other, not autism, or ADHD/ADD) Behavioral abnormalities (tantrums, auto mutulation) Horseshoe kidney Renal Cysts Other Renal abnormality Cryptorchidism Recurrent infections Photosensitivity Polyarthralgia Hepatomegaly Hypothyroidism Leukocytose Trombocytopenie Asthma Strabismus Growth hormone deficiency'\n",
    "doc = nlp(text.lower())\n",
    "\n",
    "\n",
    "\n",
    "colors = {\"ENTITY\": \"linear-gradient(90deg, #aa9cfc, #fc9ce7)\"}\n",
    "options = {\"ents\": [\"ENTITY\"], \"colors\": colors}\n",
    "\n",
    "displacy.render(doc, style=\"ent\", options=options)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#from spacy import displacy\n",
    "#displacy.render(next(doc.sents), style='dep', jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C1836542\n",
      "HP:0005280\n",
      "C1849367\n",
      "not found\n",
      "C1854113\n",
      "HP:0000426\n",
      "C4551564\n",
      "not found\n",
      "C4021805\n",
      "HP:0000422\n"
     ]
    }
   ],
   "source": [
    "entity = doc.ents[2]\n",
    "\n",
    "for umls_ent in entity._.kb_ents:\n",
    "    print(hpo.kb.cui_to_entity[umls_ent[0]].concept_id)\n",
    "    try :\n",
    "        print(umls_to_hpo[hpo.kb.cui_to_entity[umls_ent[0]].concept_id])\n",
    "    except :\n",
    "        print('not found')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tagger', <spacy.pipeline.pipes.Tagger at 0x107949244a8>),\n",
       " ('parser', <spacy.pipeline.pipes.DependencyParser at 0x1079308a828>),\n",
       " ('ner', <spacy.pipeline.pipes.EntityRecognizer at 0x10793227e88>),\n",
       " ('AbbreviationDetector',\n",
       "  <scispacy.abbreviation.AbbreviationDetector at 0x108348390f0>),\n",
       " ('EntityLinker', <scispacy.linking.EntityLinker at 0x10782c229b0>)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#nlp.remove_pipe('EntityLinker')\n",
    "nlp.pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "\n",
    "w2v_model = Word2Vec.load(\"word2vec/w2v_HPO.bin\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = 'intracranial bleeding'\n",
    "entities = ['intracranial haemorrhage']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### synonym function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<p>Do not forget to buy <mark>milk</mark> today.</p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display, HTML\n",
    "\n",
    "context = '<p>Do not forget to buy <mark>milk</mark> today.</p>'\n",
    "display(HTML(context)) # ,colors = 'r', default_color=\"black\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#text='scoliotic posture'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = 'intracranial bleeding'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "calcification bleeding\n",
      "choroid bleeding\n",
      "artery bleeding\n",
      "intracranial hemorrhage\n",
      "intracranial haemorrhage\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{0: ['intracranial haemorrhage', 'intracranial bleeding', 0.3129790425300598]}"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = text.split(' ')\n",
    "\n",
    "def getSynonym():\n",
    "    \n",
    "    top_n = 3\n",
    "    cand_id = 0\n",
    "    d_cand = {}\n",
    "    for i in range(len(tokens)):\n",
    "        new_tokens = tokens.copy()\n",
    "        word = tokens[i]\n",
    "        old_string = ' '.join(new_tokens) \n",
    "        for j in range(top_n):\n",
    "            if word in w2v_model.wv.vocab.keys():\n",
    "                new_tokens[i], proba_ = w2v_model.wv.most_similar(positive=[word])[j]\n",
    "                string = ' '.join(new_tokens) \n",
    "                #print(string)\n",
    "                if string in entities: \n",
    "                    d_cand[cand_id] = [string, old_string, proba_]\n",
    "                    break #save time\n",
    "d_cand"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Dataframe for training word2vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>All</td>\n",
       "      <td>All,</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Abnormality of body height</td>\n",
       "      <td>Abnormality of body height, HP:0008522, HP:000...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Multicystic kidney dysplasia</td>\n",
       "      <td>Multicystic kidney dysplasia, HP:0008522, HP:0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mode of inheritance</td>\n",
       "      <td>Mode of inheritance, HP:0008522, HP:0008584, H...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Autosomal dominant inheritance</td>\n",
       "      <td>Autosomal dominant inheritance, HP:0008522, HP...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             name  \\\n",
       "0                             All   \n",
       "1      Abnormality of body height   \n",
       "2    Multicystic kidney dysplasia   \n",
       "3             Mode of inheritance   \n",
       "4  Autosomal dominant inheritance   \n",
       "\n",
       "                                                text  \n",
       "0                                              All,   \n",
       "1  Abnormality of body height, HP:0008522, HP:000...  \n",
       "2  Multicystic kidney dysplasia, HP:0008522, HP:0...  \n",
       "3  Mode of inheritance, HP:0008522, HP:0008584, H...  \n",
       "4  Autosomal dominant inheritance, HP:0008522, HP...  "
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df = pd.DataFrame(columns=['name','text'])\n",
    "cnt = 0\n",
    "for i in graph.nodes: # graph.nodes\n",
    "    #synonym_str = ''\n",
    "    name = graph.nodes[i]['name']\n",
    "    text = name + ', '\n",
    "    try:\n",
    "        for i in graph.nodes[i]['synonym']:\n",
    "            text +=', '.join(graph.nodes['HP:0005101']['alt_id'])\n",
    "    except:\n",
    "        synonym_str = ''\n",
    "    \n",
    "    try : \n",
    "        for i in graph.nodes[i]['synonym']:\n",
    "            text += i.split('\"')[1] + ', '\n",
    "    except :\n",
    "        synonym_str = ''\n",
    "    try : \n",
    "        text += graph.nodes[i]['def'].split('[')[0] + ', '\n",
    "    except :\n",
    "        definition = ''\n",
    "    # Add\n",
    "    df.loc[cnt] = [name, text]\n",
    "    cnt +=1\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(r'data/Word2VecModel_2.csv', index=False) #(r'data/TableExtraction.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<generator at 0x21a8a5ef678>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#doc.sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'C0795905'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-100-bc6df7a8b9d0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mhpo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcui_to_entity\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'C0795905'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m: 'C0795905'"
     ]
    }
   ],
   "source": [
    "hpo.kb.cui_to_entity['C0795905']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('C1836542', 0.9082361459732056),\n",
       " ('C1849367', 0.8585161566734314),\n",
       " ('C1854113', 0.8437902927398682),\n",
       " ('C4551564', 0.8261430263519287),\n",
       " ('C4021805', 0.8239642381668091)]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entity._.kb_ents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity = next(doc.sents)\n",
    "for umls_ent in entity._.kb_ents:\n",
    "    print(linker.kb.cui_to_entity[umls_ent[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'EntityLinker' object has no attribute '_'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-54-a600aa4104bd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mlinker\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkb_ents\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m: 'EntityLinker' object has no attribute '_'"
     ]
    }
   ],
   "source": [
    "from spacy.pipeline import EntityLinker\n",
    "entity_linker = EntityLinker(nlp.vocab)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<generator at 0x295bc1a58b8>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(x.orth_,x.pos_, x.lemma_) for x in [y for y in nlp(str(sentences[20])) if not y.is_stop and y.pos_ != 'PUNCT']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Argument 'string' has incorrect type (expected str, got spacy.tokens.span.Span)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-499dc937bd28>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m#import spacy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mdisplacy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnlp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msents\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mjupyter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstyle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'ent'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\my_env\\lib\\site-packages\\spacy\\language.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, text, disable, component_cfg)\u001b[0m\n\u001b[0;32m    439\u001b[0m                 \u001b[0mErrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mE088\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlength\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_length\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_length\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    440\u001b[0m             )\n\u001b[1;32m--> 441\u001b[1;33m         \u001b[0mdoc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_doc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    442\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcomponent_cfg\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    443\u001b[0m             \u001b[0mcomponent_cfg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\my_env\\lib\\site-packages\\spacy\\language.py\u001b[0m in \u001b[0;36mmake_doc\u001b[1;34m(self, text)\u001b[0m\n\u001b[0;32m    465\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    466\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mmake_doc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 467\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    468\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    469\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_format_docs_and_golds\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdocs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgolds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: Argument 'string' has incorrect type (expected str, got spacy.tokens.span.Span)"
     ]
    }
   ],
   "source": [
    "#import scispacy\n",
    "#import spacy\n",
    "\n",
    "displacy.render(nlp(next(doc.sents)), jupyter=True, style='ent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xml:lang=\"en\" id=\"8f44f499b82a4123943bfb993f7fca68-0\" class=\"displacy\" width=\"2675\" height=\"574.5\" direction=\"ltr\" style=\"max-width: none; height: 574.5px; color: #000000; background: #ffffff; font-family: Arial; direction: ltr\">\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"50\">Influenza,</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"50\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"225\">commonly</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"225\">ADV</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"400\">known</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"400\">VERB</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"575\">as &quot;</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"575\">ADP</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"750\">the</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"750\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"925\">flu&quot;,</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"925\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"1100\">is</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"1100\">AUX</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"1275\">an</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"1275\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"1450\">infectious</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"1450\">ADJ</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"1625\">disease</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"1625\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"1800\">caused</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"1800\">VERB</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"1975\">by</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"1975\">ADP</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"2150\">an</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"2150\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"2325\">influenza</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"2325\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"484.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"2500\">virus.</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"2500\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-0\" stroke-width=\"2px\" d=\"M70,439.5 C70,2.0 1625.0,2.0 1625.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-0\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">nsubj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M70,441.5 L62,429.5 78,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-1\" stroke-width=\"2px\" d=\"M245,439.5 C245,352.0 380.0,352.0 380.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-1\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">advmod</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M245,441.5 L237,429.5 253,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-2\" stroke-width=\"2px\" d=\"M70,439.5 C70,264.5 385.0,264.5 385.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-2\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">acl</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M385.0,441.5 L393.0,429.5 377.0,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-3\" stroke-width=\"2px\" d=\"M420,439.5 C420,352.0 555.0,352.0 555.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-3\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">prep</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M555.0,441.5 L563.0,429.5 547.0,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-4\" stroke-width=\"2px\" d=\"M770,439.5 C770,352.0 905.0,352.0 905.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-4\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M770,441.5 L762,429.5 778,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-5\" stroke-width=\"2px\" d=\"M595,439.5 C595,264.5 910.0,264.5 910.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-5\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">pobj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M910.0,441.5 L918.0,429.5 902.0,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-6\" stroke-width=\"2px\" d=\"M1120,439.5 C1120,177.0 1615.0,177.0 1615.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-6\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">cop</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M1120,441.5 L1112,429.5 1128,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-7\" stroke-width=\"2px\" d=\"M1295,439.5 C1295,264.5 1610.0,264.5 1610.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-7\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M1295,441.5 L1287,429.5 1303,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-8\" stroke-width=\"2px\" d=\"M1470,439.5 C1470,352.0 1605.0,352.0 1605.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-8\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">amod</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M1470,441.5 L1462,429.5 1478,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-9\" stroke-width=\"2px\" d=\"M1645,439.5 C1645,352.0 1780.0,352.0 1780.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-9\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">acl</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M1780.0,441.5 L1788.0,429.5 1772.0,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-10\" stroke-width=\"2px\" d=\"M1995,439.5 C1995,177.0 2490.0,177.0 2490.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-10\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">case</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M1995,441.5 L1987,429.5 2003,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-11\" stroke-width=\"2px\" d=\"M2170,439.5 C2170,264.5 2485.0,264.5 2485.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-11\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M2170,441.5 L2162,429.5 2178,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-12\" stroke-width=\"2px\" d=\"M2345,439.5 C2345,352.0 2480.0,352.0 2480.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-12\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">compound</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M2345,441.5 L2337,429.5 2353,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-8f44f499b82a4123943bfb993f7fca68-0-13\" stroke-width=\"2px\" d=\"M1820,439.5 C1820,89.5 2495.0,89.5 2495.0,439.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-8f44f499b82a4123943bfb993f7fca68-0-13\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">nmod</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M2495.0,441.5 L2503.0,429.5 2487.0,429.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "</svg></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "displacy.render(next(doc.sents), style='dep', jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "The most common symptoms include: a high fever, runny nose, sore throat, muscle pains, headache, coughing, and feeling tired ..."
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(doc.sents)[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The most common \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    symptoms\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " include: a \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    high fever\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    runny nose\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    sore throat\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    muscle pains\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    headache\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    coughing\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       ", and \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    feeling\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    tired\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ENTITY</span>\n",
       "</mark>\n",
       " ... </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "displacy.render(list(doc.sents)[2], style='ent', jupyter=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "displacy.render(nlp(str(sentences[20])), jupyter=True, style='ent')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = nltk.Text(word.lower() for word in nltk.corpus.brown.words())\n",
    "text.similar('woman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'wienerwurst', 'firedog', 'hot_dog', 'wiener', 'frump', 'detent', 'hotdog', 'tag', 'andiron', 'frank', 'Canis_familiaris', 'domestic_dog', 'click', 'pawl', 'blackguard', 'weenie', 'bounder', 'hound', 'cad', 'track', 'chase_after', 'dog', 'frankfurter', 'give_chase', 'go_after', 'tail', 'trail', 'dog-iron', 'heel', 'chase'}\n"
     ]
    }
   ],
   "source": [
    "import nltk \n",
    "from nltk.corpus import wordnet \n",
    "synonyms = [] \n",
    "\n",
    "  \n",
    "for syn in wordnet.synsets('dog'): \n",
    "    for l in syn.lemmas(): \n",
    "        synonyms.append(l.name()) \n",
    "        if l.antonyms(): \n",
    "            antonyms.append(l.antonyms()[0].name())\n",
    "print(set(synonyms))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "my_env",
   "language": "python",
   "name": "my_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}