preprocess_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d3e7b0e5",
   "metadata": {},
   "source": [
    "# Wiki-Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "140b801b-cd25-4648-aaa1-510eb12cf423",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default\n",
      "Reusing dataset wiki_split (/home/bhadresh/.cache/huggingface/datasets/wiki_split/default/0.1.0/83151e5f3f5a622601303c2f562ffda5502643e38f3f1fee48999d0030f82b07)\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "KeyboardInterrupt: ",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_380005/3069813551.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mdatasets\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mdataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"wiki_split\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/load.py\u001b[0m in \u001b[0;36mload_dataset\u001b[0;34m(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, keep_in_memory, save_infos, script_version, use_auth_token, task, streaming, **config_kwargs)\u001b[0m\n\u001b[1;32m    866\u001b[0m         \u001b[0mkeep_in_memory\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mkeep_in_memory\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mis_small_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuilder_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    867\u001b[0m     )\n\u001b[0;32m--> 868\u001b[0;31m     \u001b[0mds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuilder_instance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_verifications\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mignore_verifications\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0min_memory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkeep_in_memory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    869\u001b[0m     \u001b[0;31m# Rename and cast features to match task schema\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    870\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtask\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/builder.py\u001b[0m in \u001b[0;36mas_dataset\u001b[0;34m(self, split, run_post_process, ignore_verifications, in_memory)\u001b[0m\n\u001b[1;32m    744\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    745\u001b[0m         \u001b[0;31m# Create a dataset for each of the given splits\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 746\u001b[0;31m         datasets = utils.map_nested(\n\u001b[0m\u001b[1;32m    747\u001b[0m             partial(\n\u001b[1;32m    748\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_build_single_dataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/utils/py_utils.py\u001b[0m in \u001b[0;36mmap_nested\u001b[0;34m(function, data_struct, dict_only, map_list, map_tuple, map_numpy, num_proc, types)\u001b[0m\n\u001b[1;32m    201\u001b[0m         \u001b[0mnum_proc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mnum_proc\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterable\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mnum_proc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 203\u001b[0;31m         mapped = [\n\u001b[0m\u001b[1;32m    204\u001b[0m             \u001b[0m_single_map_nested\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtypes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdisable_tqdm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m         ]\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/utils/py_utils.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mnum_proc\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterable\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0mnum_proc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m         mapped = [\n\u001b[0;32m--> 204\u001b[0;31m             \u001b[0m_single_map_nested\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtypes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdisable_tqdm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m         ]\n\u001b[1;32m    206\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/utils/py_utils.py\u001b[0m in \u001b[0;36m_single_map_nested\u001b[0;34m(args)\u001b[0m\n\u001b[1;32m    140\u001b[0m     \u001b[0;31m# Singleton first to spare some computation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_struct\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_struct\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtypes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 142\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_struct\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    143\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m     \u001b[0;31m# Reduce logging to keep things readable in multiprocessing with tqdm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/builder.py\u001b[0m in \u001b[0;36m_build_single_dataset\u001b[0;34m(self, split, run_post_process, ignore_verifications, in_memory)\u001b[0m\n\u001b[1;32m    774\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    775\u001b[0m         \u001b[0;31m# Build base dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 776\u001b[0;31m         ds = self._as_dataset(\n\u001b[0m\u001b[1;32m    777\u001b[0m             \u001b[0msplit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    778\u001b[0m             \u001b[0min_memory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0min_memory\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/builder.py\u001b[0m in \u001b[0;36m_as_dataset\u001b[0;34m(self, split, in_memory)\u001b[0m\n\u001b[1;32m    855\u001b[0m             \u001b[0min_memory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0min_memory\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    856\u001b[0m         )\n\u001b[0;32m--> 857\u001b[0;31m         \u001b[0mfingerprint\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_dataset_fingerprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    858\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mDataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfingerprint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfingerprint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mdataset_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    859\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/builder.py\u001b[0m in \u001b[0;36m_get_dataset_fingerprint\u001b[0;34m(self, split)\u001b[0m\n\u001b[1;32m    861\u001b[0m         \u001b[0;34m\"\"\"The dataset fingerprint is the hash of the relative directory dataset_name/config_name/version/hash, as well as the split specs.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    862\u001b[0m         \u001b[0mhasher\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mHasher\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 863\u001b[0;31m         \u001b[0mhasher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_relative_data_dir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    864\u001b[0m         \u001b[0mhasher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# for example: train, train+test, train[:10%], test[:33%](pct1_dropremainder)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    865\u001b[0m         \u001b[0mfingerprint\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhasher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhexdigest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/fingerprint.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, value)\u001b[0m\n\u001b[1;32m    193\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    194\u001b[0m         \u001b[0mheader_for_update\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf\"=={type(value)}==\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 195\u001b[0;31m         \u001b[0mvalue_for_update\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhash\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    196\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mheader_for_update\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"utf8\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue_for_update\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"utf-8\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/fingerprint.py\u001b[0m in \u001b[0;36mhash\u001b[0;34m(cls, value)\u001b[0m\n\u001b[1;32m    189\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 191\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhash_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/fingerprint.py\u001b[0m in \u001b[0;36mhash_default\u001b[0;34m(cls, value)\u001b[0m\n\u001b[1;32m    182\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    183\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mhash_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 184\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhash_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    185\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/utils/py_utils.py\u001b[0m in \u001b[0;36mdumps\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m    337\u001b[0m     \u001b[0;34m\"\"\"pickle an object to a string\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    338\u001b[0m     \u001b[0mfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mStringIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 339\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0m_no_cache_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    340\u001b[0m         \u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetvalue\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.8/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    111\u001b[0m         \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    114\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/datasets/utils/py_utils.py\u001b[0m in \u001b[0;36m_no_cache_fields\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_no_cache_fields\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m         \u001b[0;32mimport\u001b[0m \u001b[0mtransformers\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m         if (\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/transformers/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[0;31m# Check the dependencies satisfy the minimal versions required.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdependency_versions_check\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     44\u001b[0m from .file_utils import (\n\u001b[1;32m     45\u001b[0m     \u001b[0m_BaseLazyModule\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/transformers/dependency_versions_check.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mpkg\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"tokenizers\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0;31m# must be loaded here, or else tqdm check may fail\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m             \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mfile_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mis_tokenizers_available\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_tokenizers_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/transformers/file_utils.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mfilelock\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mFileLock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mhuggingface_hub\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mHfApi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mHfFolder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRepository\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtransformers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mversions\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mimportlib_metadata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/huggingface_hub/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mfile_download\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcached_download\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhf_hub_download\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhf_hub_url\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mhf_api\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mHfApi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mHfFolder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mhub_mixin\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mModelHubMixin\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mrepository\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRepository\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0msnapshot_download\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msnapshot_download\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/huggingface_hub/hub_mixin.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_torch_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/preprocessor/lib/python3.8/site-packages/torch/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    195\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mUSE_GLOBAL_DEPS\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0m_load_global_deps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 197\u001b[0;31m     \u001b[0;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m  \u001b[0;31m# noqa: F403\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    199\u001b[0m \u001b[0;31m# Appease the type checker; ordinarily this binding is inserted by the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: KeyboardInterrupt: "
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"wiki_split\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71f9d223-c0be-4192-bdcd-c5c99904acda",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f2cc3ab-c5d9-4040-af22-97bec4d89c46",
   "metadata": {},
   "source": [
    "## Train data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2da41889-6511-4b56-b65c-9f25c04bbbc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = dataset['train']\n",
    "train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9287389c-4773-4a75-945f-dd4e3b8458aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_dataset = train_dataset.map(lambda x: {\"complex_sentence\": \"sentence split: \"+x[\"complex_sentence\"]})\n",
    "# train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bdfbe08-68a2-4976-bb98-442ba5d49743",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = train_dataset.map(\n",
    "    lambda x: {\"target_sentences\": x[\"simple_sentence_1\"]+ \" <sep> \" +x[\"simple_sentence_2\"]}, \n",
    "    remove_columns=['simple_sentence_1', 'simple_sentence_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c051601-d3b8-46b0-8a76-32cb6ea1f9ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca541fbf-1476-4124-9a0a-008c3827993f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset.to_csv('train_file.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1208c0b-d428-4ea0-b963-cda0641c49ae",
   "metadata": {},
   "source": [
    "## Validation data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f4f8719-041a-4449-a7de-f7eef9e1972a",
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_dataset = dataset['validation']\n",
    "validation_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67a6660a-c47a-4b81-a177-746816ec8da8",
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_dataset = validation_dataset.map(\n",
    "    lambda x: {\"target_sentences\": x[\"simple_sentence_1\"]+ \" <sep> \" +x[\"simple_sentence_2\"]}, \n",
    "    remove_columns=['simple_sentence_1', 'simple_sentence_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5183ddb1-9dd5-4476-a68f-cc6ccce01ea6",
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_dataset.to_csv('validation_file.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63191e75-ae5f-4a63-a14a-2ba65f4c0c34",
   "metadata": {},
   "source": [
    "## Predict Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22e7a16c-af11-4458-8ec8-6c21f105ad58",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = dataset['test']\n",
    "test_dataset = test_dataset.map(\n",
    "    lambda x: {\"target_sentences\": x[\"simple_sentence_1\"]+ \" <sep> \" +x[\"simple_sentence_2\"]}, \n",
    "    remove_columns=['simple_sentence_1', 'simple_sentence_2'])\n",
    "test_dataset.to_csv('test_file.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efefcfc0-d2ba-4745-b09f-2fc04d56d764",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "365ea11e",
   "metadata": {},
   "source": [
    "# Web Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d974c156",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2df066d",
   "metadata": {},
   "source": [
    "## test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "01bba7c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "complex_sentence    The American , Duncan Rouleau created the char...\n",
       "simple_sentence     Baymax was created by Duncan Rouleau . Duncan ...\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_simple = pd.read_csv('web_split_seq2seq/test_simple.txt', sep='tab', header=None)\n",
    "test_complex = pd.read_csv('web_split_seq2seq/test_complex.txt', sep='tab', header=None)\n",
    "test_df = pd.concat([test_complex, test_simple], axis=1)\n",
    "test_df.columns = ['complex_sentence', 'simple_sentence']\n",
    "test_df.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "8b6c235b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "929 929\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>complex_sentence</th>\n",
       "      <th>simple_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The American , Duncan Rouleau created the char...</td>\n",
       "      <td>[Baymax was created by Duncan Rouleau . Duncan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The musician Andrew White 's genre is alternat...</td>\n",
       "      <td>[Baymax was created by Duncan Rouleau . Duncan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Above the Veil is from Australia and was prece...</td>\n",
       "      <td>[Baymax was created by Duncan Rouleau . Duncan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10 Hygiea has an apoapsis of 523951582.33968 k...</td>\n",
       "      <td>[Baymax was created by Duncan Rouleau . Duncan...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Addiction journal is about addiction and is pu...</td>\n",
       "      <td>[Baymax was created by Duncan Rouleau . Duncan...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    complex_sentence  \\\n",
       "0  The American , Duncan Rouleau created the char...   \n",
       "1  The musician Andrew White 's genre is alternat...   \n",
       "2  Above the Veil is from Australia and was prece...   \n",
       "3  10 Hygiea has an apoapsis of 523951582.33968 k...   \n",
       "4  Addiction journal is about addiction and is pu...   \n",
       "\n",
       "                                     simple_sentence  \n",
       "0  [Baymax was created by Duncan Rouleau . Duncan...  \n",
       "1  [Baymax was created by Duncan Rouleau . Duncan...  \n",
       "2  [Baymax was created by Duncan Rouleau . Duncan...  \n",
       "3  [Baymax was created by Duncan Rouleau . Duncan...  \n",
       "4  [Baymax was created by Duncan Rouleau . Duncan...  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def modify_eval_data(df):\n",
    "    simp_sen = []\n",
    "    comp_sens = []\n",
    "    simp_sens = []\n",
    "    pre_comp_sen = df.iloc[0]['complex_sentence']\n",
    "    for i in range(len(df)):\n",
    "        if df.iloc[i]['complex_sentence']==pre_comp_sen:\n",
    "            simp_sen.append(df.iloc[i]['simple_sentence'])\n",
    "        else:\n",
    "            simp_sens.append(simp_sen)\n",
    "            comp_sens.append(pre_comp_sen)\n",
    "        pre_comp_sen = df.iloc[i]['complex_sentence']\n",
    "    print(len(comp_sens), len(comp_sens))\n",
    "    new_df = pd.DataFrame()\n",
    "    new_df['complex_sentence'] = comp_sens\n",
    "    new_df['simple_sentence'] = simp_sens\n",
    "    return new_df\n",
    "test_dataset = modify_eval_data(test_df)\n",
    "test_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "07bb5a7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 929 entries, 0 to 928\n",
      "Data columns (total 2 columns):\n",
      " #   Column            Non-Null Count  Dtype \n",
      "---  ------            --------------  ----- \n",
      " 0   complex_sentence  929 non-null    object\n",
      " 1   simple_sentence   929 non-null    object\n",
      "dtypes: object(2)\n",
      "memory usage: 14.6+ KB\n"
     ]
    }
   ],
   "source": [
    "test_dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "68a8f9f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 4294967296 bytes == 0x2d33ec000 @  0x7f062eb22680 0x7f062eb42bdd 0x7f05790793e3 0x7f057907a648 0x7f057907a368 0x7f057907a285 0x7f057907a285 0x7f057907a7f1 0x7f0579077eaa 0x5f2cc9 0x5f30ff 0x5705f6 0x5f5956 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x50b868 0x56bc9b 0x568d9a 0x68cdc7 0x5ff5d4 0x5c3cb0 0x56aadf 0x501148 0x56c422 0x501148 0x56c422 0x501148 0x504d56\n",
      "tcmalloc: large alloc 8589934592 bytes == 0x413bec000 @  0x7f062eb22680 0x7f062eb42bdd 0x7f05790793e3 0x7f057907a648 0x7f057907a368 0x7f057907a285 0x7f057907a285 0x7f057907a7f1 0x7f0579077eaa 0x5f2cc9 0x5f30ff 0x5705f6 0x5f5956 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x50b868 0x56bc9b 0x568d9a 0x68cdc7 0x5ff5d4 0x5c3cb0 0x56aadf 0x501148 0x56c422 0x501148 0x56c422 0x501148 0x504d56\n",
      "tcmalloc: large alloc 7566508032 bytes == 0x1340a000 @  0x7f062eb22680 0x7f062eb43824 0x4f0f6a 0x7f0579077fa8 0x5f2cc9 0x5f30ff 0x5705f6 0x5f5956 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x50b868 0x56bc9b 0x568d9a 0x68cdc7 0x5ff5d4 0x5c3cb0 0x56aadf 0x501148 0x56c422 0x501148 0x56c422 0x501148 0x504d56 0x56acb6 0x5f5956 0x56aadf 0x5f5956 0x56acb6\n",
      "tcmalloc: large alloc 7566508032 bytes == 0x2133ec000 @  0x7f062eb22680 0x7f062eb43824 0x5f7b11 0x61e99e 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x50b868 0x56bc9b 0x568d9a 0x68cdc7 0x5ff5d4 0x5c3cb0 0x56aadf 0x501148 0x56c422 0x501148 0x56c422 0x501148 0x504d56 0x56acb6 0x5f5956 0x56aadf 0x5f5956 0x56acb6 0x568d9a 0x5f5b33 0x50b7f8\n"
     ]
    }
   ],
   "source": [
    "test_dataset.to_json('web_split_seq2seq/test_file.json', orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "51377bc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset.to_csv('web_split_seq2seq/test_file.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f114f2c0",
   "metadata": {},
   "source": [
    "## Validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "5295e2c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bhadresh/preprocessor/lib/python3.8/site-packages/pandas/util/_decorators.py:311: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  return func(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "complex_sentence    Alcatraz Versus the Scrivener 's Bones , writt...\n",
       "simple_sentence     The book Alcatraz Versus the Evil Librarians i...\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_simple = pd.read_csv('web_split_seq2seq/validation_simple.txt', sep='tab', header=None)\n",
    "validation_complex = pd.read_csv('web_split_seq2seq/validation_complex.txt', sep='tab', header=None)\n",
    "validation_df = pd.concat([validation_complex, validation_simple], axis=1)\n",
    "validation_df.columns = ['complex_sentence', 'simple_sentence']\n",
    "validation_df.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ed4a6d4a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "953 953\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>complex_sentence</th>\n",
       "      <th>simple_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alcatraz Versus the Scrivener 's Bones , writt...</td>\n",
       "      <td>[The book Alcatraz Versus the Evil Librarians ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alderney Airport serves the city of Alderney ,...</td>\n",
       "      <td>[The book Alcatraz Versus the Evil Librarians ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Icebreaker Aleksey Chrikov was built at th...</td>\n",
       "      <td>[The book Alcatraz Versus the Evil Librarians ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The Alderney Airport serves the island of Alde...</td>\n",
       "      <td>[The book Alcatraz Versus the Evil Librarians ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Adisham Hall is located in Sri Lanka and it 's...</td>\n",
       "      <td>[The book Alcatraz Versus the Evil Librarians ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    complex_sentence  \\\n",
       "0  Alcatraz Versus the Scrivener 's Bones , writt...   \n",
       "1  Alderney Airport serves the city of Alderney ,...   \n",
       "2  The Icebreaker Aleksey Chrikov was built at th...   \n",
       "3  The Alderney Airport serves the island of Alde...   \n",
       "4  Adisham Hall is located in Sri Lanka and it 's...   \n",
       "\n",
       "                                     simple_sentence  \n",
       "0  [The book Alcatraz Versus the Evil Librarians ...  \n",
       "1  [The book Alcatraz Versus the Evil Librarians ...  \n",
       "2  [The book Alcatraz Versus the Evil Librarians ...  \n",
       "3  [The book Alcatraz Versus the Evil Librarians ...  \n",
       "4  [The book Alcatraz Versus the Evil Librarians ...  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_dataset = modify_eval_data(validation_df)\n",
    "validation_dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "064e9dac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tcmalloc: large alloc 4294967296 bytes == 0x2d33ec000 @  0x7f062eb22680 0x7f062eb42bdd 0x7f05790793e3 0x7f057907a648 0x7f057907a368 0x7f057907a285 0x7f057907a285 0x7f057907a7f1 0x7f0579077eaa 0x5f2cc9 0x5f30ff 0x5705f6 0x5f5956 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x50b868 0x56bc9b 0x568d9a 0x68cdc7 0x5ff5d4 0x5c3cb0 0x56aadf 0x501148 0x56c422 0x501148 0x56c422 0x501148 0x504d56\n",
      "tcmalloc: large alloc 8589934592 bytes == 0x413bec000 @  0x7f062eb22680 0x7f062eb42bdd 0x7f05790793e3 0x7f057907a648 0x7f057907a368 0x7f057907a285 0x7f057907a285 0x7f057907a7f1 0x7f0579077eaa 0x5f2cc9 0x5f30ff 0x5705f6 0x5f5956 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x50b868 0x56bc9b 0x568d9a 0x68cdc7 0x5ff5d4 0x5c3cb0 0x56aadf 0x501148 0x56c422 0x501148 0x56c422 0x501148 0x504d56\n",
      "tcmalloc: large alloc 6968238080 bytes == 0x1340a000 @  0x7f062eb22680 0x7f062eb43824 0x4f0f6a 0x7f0579077fa8 0x5f2cc9 0x5f30ff 0x5705f6 0x5f5956 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x50b868 0x56bc9b 0x568d9a 0x68cdc7 0x5ff5d4 0x5c3cb0 0x56aadf 0x501148 0x56c422 0x501148 0x56c422 0x501148 0x504d56 0x56acb6 0x5f5956 0x56aadf 0x5f5956 0x56acb6\n",
      "tcmalloc: large alloc 6968238080 bytes == 0x2133ec000 @  0x7f062eb22680 0x7f062eb43824 0x5f7b11 0x61e99e 0x504d56 0x56acb6 0x568d9a 0x5f5b33 0x56bc9b 0x568d9a 0x50b868 0x56bc9b 0x568d9a 0x68cdc7 0x5ff5d4 0x5c3cb0 0x56aadf 0x501148 0x56c422 0x501148 0x56c422 0x501148 0x504d56 0x56acb6 0x5f5956 0x56aadf 0x5f5956 0x56acb6 0x568d9a 0x5f5b33 0x50b7f8\n"
     ]
    }
   ],
   "source": [
    "validation_dataset.to_json('web_split_seq2seq/validation_file.json', orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "09176fb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_dataset.to_csv('web_split_seq2seq/validation_file.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f1d79be",
   "metadata": {},
   "source": [
    "## Trainset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5a0cecfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask import dataframe as dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0097b4c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_simple = dd.read_csv('web_split_seq2seq/train_simple.txt', sep='\\n', header=None)\n",
    "train_complex = dd.read_csv('web_split_seq2seq/train_complex.txt', sep='\\n', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4982909e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The kidney bean is an ingredient in Bandeja pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kidney beans are an ingredient in Bandeja pais...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kidney beans are an ingredient in Bandeja pais...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The kidney bean is an ingredient in Bandeja pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The kidney bean is an ingredient in Bandeja pa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   0\n",
       "0  The kidney bean is an ingredient in Bandeja pa...\n",
       "1  Kidney beans are an ingredient in Bandeja pais...\n",
       "2  Kidney beans are an ingredient in Bandeja pais...\n",
       "3  The kidney bean is an ingredient in Bandeja pa...\n",
       "4  The kidney bean is an ingredient in Bandeja pa..."
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_simple.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5bafb326",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   0\n",
       "0  Bandeja pasta -LRB- Colombia -RRB- includes th...\n",
       "1  Bandeja pasta -LRB- Colombia -RRB- includes th...\n",
       "2  Bandeja pasta -LRB- Colombia -RRB- includes th...\n",
       "3  Bandeja pasta -LRB- Colombia -RRB- includes th...\n",
       "4  Bandeja pasta -LRB- Colombia -RRB- includes th..."
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_complex.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9487eb38",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>complex_sentence</th>\n",
       "      <th>simple_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "      <td>The kidney bean is an ingredient in Bandeja pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "      <td>Kidney beans are an ingredient in Bandeja pais...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "      <td>Kidney beans are an ingredient in Bandeja pais...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "      <td>The kidney bean is an ingredient in Bandeja pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bandeja pasta -LRB- Colombia -RRB- includes th...</td>\n",
       "      <td>The kidney bean is an ingredient in Bandeja pa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    complex_sentence  \\\n",
       "0  Bandeja pasta -LRB- Colombia -RRB- includes th...   \n",
       "1  Bandeja pasta -LRB- Colombia -RRB- includes th...   \n",
       "2  Bandeja pasta -LRB- Colombia -RRB- includes th...   \n",
       "3  Bandeja pasta -LRB- Colombia -RRB- includes th...   \n",
       "4  Bandeja pasta -LRB- Colombia -RRB- includes th...   \n",
       "\n",
       "                                     simple_sentence  \n",
       "0  The kidney bean is an ingredient in Bandeja pa...  \n",
       "1  Kidney beans are an ingredient in Bandeja pais...  \n",
       "2  Kidney beans are an ingredient in Bandeja pais...  \n",
       "3  The kidney bean is an ingredient in Bandeja pa...  \n",
       "4  The kidney bean is an ingredient in Bandeja pa...  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.DataFrame()\n",
    "train_df['complex_sentence'] = list(train_complex[0])\n",
    "train_df['simple_sentence'] = list(train_simple[0])\n",
    "train_df.to_csv('web_split_seq2seq/train_file.csv', index=False)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af1f7025",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}