Keyword Extractor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('clusters.json', 'r') as f:\n",
    "    dic = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_list = []\n",
    "for key in dic:\n",
    "    values = []\n",
    "    for val in dic[key]:\n",
    "        values+=val\n",
    "    new_list.append(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_list_1 = []\n",
    "for a in new_list:\n",
    "    new_list_1.append(\" \".join(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "cv=CountVectorizer(max_df=0.85,max_features=10000)\n",
    "word_count_vector=cv.fit_transform(new_list_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    " \n",
    "tfidf_transformer=TfidfTransformer(smooth_idf=True,use_idf=True)\n",
    "tfidf_transformer.fit(word_count_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sort_coo(coo_matrix):\n",
    "    tuples = zip(coo_matrix.col, coo_matrix.data)\n",
    "    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)\n",
    " \n",
    "def extract_topn_from_vector(feature_names, sorted_items, topn=10):\n",
    "    \"\"\"get the feature names and tf-idf score of top n items\"\"\"\n",
    "    \n",
    "    #use only topn items from vector\n",
    "    sorted_items = sorted_items[:topn]\n",
    " \n",
    "    score_vals = []\n",
    "    feature_vals = []\n",
    "    \n",
    "    # word index and corresponding tf-idf score\n",
    "    for idx, score in sorted_items:\n",
    "        \n",
    "        #keep track of feature name and its corresponding score\n",
    "        score_vals.append(round(score, 3))\n",
    "        feature_vals.append(feature_names[idx])\n",
    " \n",
    "    #create a tuples of feature,score\n",
    "    #results = zip(feature_vals,score_vals)\n",
    "    results= {}\n",
    "    for idx in range(len(feature_vals)):\n",
    "        results[feature_vals[idx]]=score_vals[idx]\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Doc  0  top keywords:\n",
      "hills 0.607\n",
      "1230 0.487\n",
      "report 0.432\n",
      "editions 0.238\n",
      "click 0.207\n",
      "morning 0.204\n",
      "welcome 0.134\n",
      "view 0.106\n",
      "presented 0.083\n",
      "happy 0.05\n",
      "Doc  1  top keywords:\n",
      "ukraine 0.26\n",
      "washington 0.234\n",
      "post 0.159\n",
      "us 0.157\n",
      "administration 0.142\n",
      "gop 0.119\n",
      "would 0.107\n",
      "whistleblower 0.102\n",
      "republican 0.101\n",
      "republicans 0.099\n",
      "Doc  2  top keywords:\n",
      "klobuchar 0.731\n",
      "amy 0.508\n",
      "sen 0.195\n",
      "dminn 0.16\n",
      "would 0.095\n",
      "minnesota 0.082\n",
      "iowa 0.068\n",
      "experience 0.066\n",
      "stage 0.06\n",
      "quarter 0.055\n",
      "Doc  3  top keywords:\n",
      "us 0.231\n",
      "party 0.126\n",
      "trumps 0.106\n",
      "minister 0.103\n",
      "washington 0.102\n",
      "government 0.101\n",
      "climate 0.097\n",
      "china 0.095\n",
      "kong 0.093\n",
      "hong 0.093\n",
      "Doc  4  top keywords:\n",
      "york 0.653\n",
      "times 0.309\n",
      "deadspin 0.13\n",
      "post 0.122\n",
      "police 0.112\n",
      "washington 0.108\n",
      "editorials 0.107\n",
      "us 0.102\n",
      "city 0.102\n",
      "mayor 0.085\n",
      "Doc  5  top keywords:\n",
      "interviews 0.394\n",
      "broadcast 0.394\n",
      "exclusive 0.32\n",
      "abc 0.32\n",
      "breaking 0.293\n",
      "online 0.291\n",
      "coverage 0.291\n",
      "world 0.267\n",
      "get 0.265\n",
      "find 0.265\n",
      "Doc  6  top keywords:\n",
      "inquiry 0.72\n",
      "ukraine 0.226\n",
      "hearings 0.172\n",
      "trumps 0.171\n",
      "public 0.142\n",
      "us 0.134\n",
      "testify 0.125\n",
      "pelosi 0.117\n",
      "testimony 0.112\n",
      "republicans 0.112\n",
      "Doc  7  top keywords:\n",
      "elizabeth 0.304\n",
      "joe 0.288\n",
      "sen 0.19\n",
      "primary 0.187\n",
      "sanders 0.169\n",
      "iowa 0.146\n",
      "us 0.135\n",
      "care 0.129\n",
      "buttigieg 0.125\n",
      "bernie 0.117\n",
      "Doc  8  top keywords:\n",
      "brexit 0.724\n",
      "boris 0.272\n",
      "uk 0.22\n",
      "johnson 0.213\n",
      "british 0.203\n",
      "minister 0.202\n",
      "prime 0.176\n",
      "european 0.143\n",
      "parliament 0.133\n",
      "johnsons 0.114\n",
      "Doc  9  top keywords:\n",
      "trumps 0.616\n",
      "ukraine 0.353\n",
      "us 0.212\n",
      "joe 0.174\n",
      "vice 0.132\n",
      "ukrainian 0.114\n",
      "republican 0.093\n",
      "whistleblower 0.085\n",
      "washington 0.084\n",
      "volodymyr 0.078\n"
     ]
    }
   ],
   "source": [
    "feature_names=cv.get_feature_names()\n",
    "for i in range(len(new_list_1)):\n",
    "    doc=new_list_1[i]\n",
    "    tf_idf_vector=tfidf_transformer.transform(cv.transform([doc]))\n",
    "    sorted_items=sort_coo(tf_idf_vector.tocoo())\n",
    "    keywords=extract_topn_from_vector(feature_names,sorted_items,10)\n",
    "    print(\"Doc \",i,\" top keywords:\")\n",
    "    for k in keywords:\n",
    "        print(k,keywords[k])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}