parser.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import copy\n",
    "import collections\n",
    "from operator import itemgetter\n",
    "\n",
    "DEBUG = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "authors_count = 100\n",
    "min_edges_for_article = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_articles():\n",
    "    articles = []\n",
    "    with open('DBLP-citation-Jan8.txt') as csv_file:\n",
    "        csv_reader = csv.reader(csv_file, delimiter='#')\n",
    "        for row in csv_reader:\n",
    "            if len(row) >= 2:\n",
    "                data = row[1]\n",
    "                if len(data)<2:\n",
    "                    continue\n",
    "                elif data[:1] == '*':\n",
    "                    article = {}\n",
    "                    article['quoted'] = []\n",
    "                    article['title'] = data[1:]\n",
    "                elif data[:1] == '@':\n",
    "                    article['authors'] = data[1:]\n",
    "                elif data[:1] == 't':\n",
    "                    article['date'] = data[1:]\n",
    "                elif data[:5] == 'index':\n",
    "                    article['index'] = data[5:]\n",
    "                    articles.append(article)\n",
    "                elif data[:1] == '%':\n",
    "                    if 'quoted' in article:\n",
    "                        article['quoted'].append(data[1:])\n",
    "                    else:\n",
    "                        article['quoted'].append(data[1:])\n",
    "                else:\n",
    "                    continue\n",
    "    return articles\n",
    "\n",
    "def filter_articles_without_authors(articles):\n",
    "    return [article for article in articles if 'authors' in article]\n",
    "\n",
    "def find_most_frequent_authors(articles, authors_count):\n",
    "    top=authors_count\n",
    "    authors_frequency = collections.defaultdict(int)\n",
    "    for article in articles:\n",
    "        authors = article['authors'].split(\",\")\n",
    "        for author in authors:\n",
    "            authors_frequency[author]+=1\n",
    "    print('All authors ' + str(len(authors_frequency)))\n",
    "    top_authors = list(sorted(authors_frequency.items(), key=itemgetter(1),reverse=True))\n",
    "    top_authors = top_authors[:top]\n",
    "    top_authors_without_occurences = [el[0] for el in top_authors]\n",
    "    return top_authors_without_occurences\n",
    "\n",
    "def filter_articles_not_containing_top_authors(articles, top_authors):\n",
    "    def all_authors_in_top(article, top_authors):\n",
    "        for author in article['authors'].split(\",\"):\n",
    "            if author not in top_authors:\n",
    "                return False\n",
    "        return True\n",
    "    return [article for article in articles if all_authors_in_top(article, top_authors)]\n",
    "\n",
    "def remove_articles_with_few_edges(articles, min_edges_for_article):\n",
    "    min_edges_cap = min_edges_for_article\n",
    "    print('All articles ' + str(len(articles)))\n",
    "    sufficient_edges_articles = [article for article in articles if len(article['quoted']) >= min_edges_cap]\n",
    "    print('Articles with sufficient edges ' + str(len(sufficient_edges_articles)))\n",
    "    return sufficient_edges_articles\n",
    "\n",
    "def update_quoted_by(articles):\n",
    "    number_of_edges = 0\n",
    "    all_indexes = set()\n",
    "    previous = 0\n",
    "    new = 0\n",
    "    for article in articles:\n",
    "        all_indexes.add(article['index'])\n",
    "    articles_copy = copy.deepcopy(articles)\n",
    "    for article in articles_copy:\n",
    "        previous += len(article['quoted'])\n",
    "        updated_quoted = [quoted for quoted in article['quoted'] if quoted in all_indexes]\n",
    "        article['quoted'] = updated_quoted\n",
    "        new += len(article['quoted'])\n",
    "        number_of_edges += len(article['quoted'])\n",
    "\n",
    "    return articles_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles = read_articles()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All articles 1511035\n",
      "Articles with sufficient edges 71585\n"
     ]
    }
   ],
   "source": [
    "articles = remove_articles_with_few_edges(articles, min_edges_for_article)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles = filter_articles_without_authors(articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All authors 80056\n"
     ]
    }
   ],
   "source": [
    "most_frequent_authors = find_most_frequent_authors(articles, authors_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles = filter_articles_not_containing_top_authors(articles, most_frequent_authors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Quotations of processed articles 4150\n",
      "Quotations of remaining articles 110\n",
      "Qotations of removed articles 4040\n"
     ]
    }
   ],
   "source": [
    "articles = update_quoted_by(articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "220"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(articles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "NODES_FILE = 'dblp_nodes.csv'\n",
    "EDGES_FILE = 'dblp_edges.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(NODES_FILE, 'w') as f:\n",
    "    idx = 0\n",
    "    for a in articles:\n",
    "        f.write(a['index'] + '|' + str(idx) + '|' + a['title'] + '\\n')\n",
    "        idx_dict[a['index']] = idx\n",
    "        idx += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(EDGES_FILE, 'w') as f:\n",
    "    for a in articles:\n",
    "        for q in a['quoted']:\n",
    "            f.write(str(idx_dict[q]) + ' ' + str(idx_dict[a['index']]) + ' ' + a['date'] + '\\n')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}