1_download_raw_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-02-10T21:30:51.308250Z",
     "iopub.status.busy": "2023-02-10T21:30:51.307534Z",
     "iopub.status.idle": "2023-02-10T21:30:51.341023Z",
     "shell.execute_reply": "2023-02-10T21:30:51.340163Z",
     "shell.execute_reply.started": "2023-02-10T21:30:51.308096Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "import os\n",
    "import shutil\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-10T21:30:51.343610Z",
     "iopub.status.busy": "2023-02-10T21:30:51.343017Z",
     "iopub.status.idle": "2023-02-10T21:30:51.349019Z",
     "shell.execute_reply": "2023-02-10T21:30:51.348072Z",
     "shell.execute_reply.started": "2023-02-10T21:30:51.343545Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_path = './dataset'\n",
    "temp_dir = './temp'\n",
    "\n",
    "os.makedirs(dataset_path, exist_ok=True)\n",
    "os.makedirs(temp_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "meteonet_url_form = 'https://meteonet.umr-cnrm.fr/dataset/data/NW/weather_models/2D_parameters/NW_weather_models_2D_parameters_{year}.tar.gz'\n",
    "\n",
    "output_file_name_form = os.path.join(temp_dir, 'meteonet_{year}.tar.gz')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-10T21:30:51.367460Z",
     "iopub.status.busy": "2023-02-10T21:30:51.367187Z",
     "iopub.status.idle": "2023-02-10T21:30:51.376989Z",
     "shell.execute_reply": "2023-02-10T21:30:51.376268Z",
     "shell.execute_reply.started": "2023-02-10T21:30:51.367428Z"
    }
   },
   "outputs": [],
   "source": [
    "def process_n_save(source, temp_dir, save_dir, year):\n",
    "    \n",
    "    year = str(year)\n",
    "    \n",
    "    shutil.unpack_archive(source, os.path.join(temp_dir, f'data_{year}'))\n",
    "    \n",
    "    files = os.listdir(os.path.join(temp_dir, f'data_{year}', f\"/NW_weather_models_2D_parameters_{year}\"))\n",
    "    \n",
    "    for file in files:\n",
    "        os.mkdir(os.path.join(temp_dir, 'Extracted', file))\n",
    "        shutil.unpack_archive(os.path.join(temp_dir, f'data_{year}', f\"/NW_weather_models_2D_parameters_{year}\", file),os.path.join(temp_dir, 'Extracted', file))\n",
    "    \n",
    "    # Save Wind\n",
    "    text_files = glob.glob(str(os.path.join(temp_dir, 'Extracted')) + \"/**/arome_10m_NW_*.grib\", recursive = True)\n",
    "    for file_ in text_files:\n",
    "        shutil.copy(file_, os.path.join(save_dir, 'Wind'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Download from Meteonet Data"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 2016 Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://meteonet.umr-cnrm.fr/dataset/data/NW/weather_models/2D_parameters/NW_weather_models_2D_parameters_2016.tar.gz\n",
      "./temp/meteonet_2016.tar.gz\n"
     ]
    }
   ],
   "source": [
    "year = 2016\n",
    "\n",
    "print(meteonet_url_form.format(year=year))\n",
    "print(output_file_name_form.format(year=year))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = meteonet_url_form.format(year=year)\n",
    "file_down_path = output_file_name_form.format(year=year)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2023-02-10 22:53:06--  http://url/\n",
      "Resolving url (url)... failed: Name or service not known.\n",
      "wget: unable to resolve host address ‘url’\n",
      "--2023-02-10 22:53:07--  https://meteonet.umr-cnrm.fr/dataset/data/NW/weather_models/2D_parameters/NW_weather_models_2D_parameters_2016.tar.gz\n",
      "Resolving meteonet.umr-cnrm.fr (meteonet.umr-cnrm.fr)... 193.49.97.131\n",
      "Connecting to meteonet.umr-cnrm.fr (meteonet.umr-cnrm.fr)|193.49.97.131|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 9153906827 (8.5G) [application/x-gzip]\n",
      "Saving to: ‘./temp/meteonet_2016.tar.gz’\n",
      "\n",
      "./temp/meteonet_201 100%[===================>]   8.52G  3.63MB/s    in 40m 11s \n",
      "\n",
      "2023-02-10 23:33:19 (3.62 MB/s) - ‘./temp/meteonet_2016.tar.gz’ saved [9153906827/9153906827]\n",
      "\n",
      "FINISHED --2023-02-10 23:33:19--\n",
      "Total wall clock time: 40m 14s\n",
      "Downloaded: 1 files, 8.5G in 40m 11s (3.62 MB/s)\n"
     ]
    }
   ],
   "source": [
    "!wget url $url -O $file_down_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "EOFError",
     "evalue": "Compressed file ended before the end-of-stream marker was reached",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[0;31mEOFError\u001b[0m                                  Traceback (most recent call last)\n",
      "\u001b[0;32m/tmp/ipykernel_15957/2973289080.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprocess_n_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_down_path_2016\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemp_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2016\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_15957/3212762275.py\u001b[0m in \u001b[0;36mprocess_n_save\u001b[0;34m(source, temp_dir, save_dir, year)\u001b[0m\n",
      "\u001b[1;32m      3\u001b[0m     \u001b[0myear\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myear\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mshutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack_archive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'data_{year}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m      7\u001b[0m     \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'data_{year}'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"/NW_weather_models_2D_parameters_{year}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/shutil.py\u001b[0m in \u001b[0;36munpack_archive\u001b[0;34m(filename, extract_dir, format)\u001b[0m\n",
      "\u001b[1;32m   1269\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_UNPACK_FORMATS\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   1270\u001b[0m         \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_UNPACK_FORMATS\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 1271\u001b[0;31m         \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextract_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   1272\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   1273\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/shutil.py\u001b[0m in \u001b[0;36m_unpack_tarfile\u001b[0;34m(filename, extract_dir)\u001b[0m\n",
      "\u001b[1;32m   1202\u001b[0m             \"%s is not a compressed or uncompressed tar file\" % filename)\n",
      "\u001b[1;32m   1203\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 1204\u001b[0;31m         \u001b[0mtarobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextractall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextract_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   1205\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   1206\u001b[0m         \u001b[0mtarobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mextractall\u001b[0;34m(self, path, members, numeric_owner)\u001b[0m\n",
      "\u001b[1;32m   2043\u001b[0m                 \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0o700\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2044\u001b[0m             \u001b[0;31m# Do not set_attrs directories, as we will do that further down\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2045\u001b[0;31m             self.extract(tarinfo, path, set_attrs=not tarinfo.isdir(),\n",
      "\u001b[0m\u001b[1;32m   2046\u001b[0m                          numeric_owner=numeric_owner)\n",
      "\u001b[1;32m   2047\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mextract\u001b[0;34m(self, member, path, set_attrs, numeric_owner)\u001b[0m\n",
      "\u001b[1;32m   2084\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2085\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2086\u001b[0;31m             self._extract_member(tarinfo, os.path.join(path, tarinfo.name),\n",
      "\u001b[0m\u001b[1;32m   2087\u001b[0m                                  \u001b[0mset_attrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mset_attrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2088\u001b[0m                                  numeric_owner=numeric_owner)\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36m_extract_member\u001b[0;34m(self, tarinfo, targetpath, set_attrs, numeric_owner)\u001b[0m\n",
      "\u001b[1;32m   2157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2158\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misreg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2159\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakefile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargetpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   2160\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2161\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargetpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mmakefile\u001b[0;34m(self, tarinfo, targetpath)\u001b[0m\n",
      "\u001b[1;32m   2206\u001b[0m                 \u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtruncate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2207\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2208\u001b[0;31m                 \u001b[0mcopyfileobj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mReadError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   2209\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2210\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmakeunknown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargetpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mcopyfileobj\u001b[0;34m(src, dst, length, exception, bufsize)\u001b[0m\n",
      "\u001b[1;32m    245\u001b[0m     \u001b[0mblocks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mremainder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdivmod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlength\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    246\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblocks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m--> 247\u001b[0;31m         \u001b[0mbuf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m    248\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mbufsize\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    249\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"unexpected end of data\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/gzip.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, size)\u001b[0m\n",
      "\u001b[1;32m    298\u001b[0m             \u001b[0;32mimport\u001b[0m \u001b[0merrno\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    299\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merrno\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEBADF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"read() on write-only GzipFile object\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m--> 300\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m    301\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    302\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/_compression.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n",
      "\u001b[1;32m     66\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreadinto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m     67\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mmemoryview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mview\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mview\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"B\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mbyte_view\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m---> 68\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbyte_view\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m     69\u001b[0m             \u001b[0mbyte_view\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m     70\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/gzip.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, size)\u001b[0m\n",
      "\u001b[1;32m    504\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbuf\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34mb\"\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m--> 506\u001b[0;31m                 raise EOFError(\"Compressed file ended before the \"\n",
      "\u001b[0m\u001b[1;32m    507\u001b[0m                                \"end-of-stream marker was reached\")\n",
      "\u001b[1;32m    508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;31mEOFError\u001b[0m: Compressed file ended before the end-of-stream marker was reached"
     ]
    }
   ],
   "source": [
    "\n",
    "process_n_save(file_down_path, temp_dir, dataset_path, year)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 2017 Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://meteonet.umr-cnrm.fr/dataset/data/NW/weather_models/2D_parameters/NW_weather_models_2D_parameters_2017.tar.gz\n",
      "./temp/meteonet_2017.tar.gz\n"
     ]
    }
   ],
   "source": [
    "year = 2017\n",
    "\n",
    "print(meteonet_url_form.format(year=year))\n",
    "print(output_file_name_form.format(year=year))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2023-02-10 23:42:25--  http://url/\n",
      "Resolving url (url)... failed: Name or service not known.\n",
      "wget: unable to resolve host address ‘url’\n",
      "--2023-02-10 23:42:26--  https://meteonet.umr-cnrm.fr/dataset/data/NW/weather_models/2D_parameters/NW_weather_models_2D_parameters_2017.tar.gz\n",
      "Resolving meteonet.umr-cnrm.fr (meteonet.umr-cnrm.fr)... 193.49.97.131\n",
      "Connecting to meteonet.umr-cnrm.fr (meteonet.umr-cnrm.fr)|193.49.97.131|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 8789487640 (8.2G) [application/x-gzip]\n",
      "Saving to: ‘./temp/meteonet_2017.tar.gz’\n",
      "\n",
      "./temp/meteonet_201  69%[============>       ]   5.68G  1013KB/s    in 27m 38s \n",
      "\n",
      "2023-02-11 00:10:05 (3.51 MB/s) - Connection closed at byte 6097333885. Retrying.\n",
      "\n",
      "--2023-02-11 00:10:06--  (try: 2)  https://meteonet.umr-cnrm.fr/dataset/data/NW/weather_models/2D_parameters/NW_weather_models_2D_parameters_2017.tar.gz\n",
      "Connecting to meteonet.umr-cnrm.fr (meteonet.umr-cnrm.fr)|193.49.97.131|:443... connected.\n",
      "HTTP request sent, awaiting response... 206 Partial Content\n",
      "Length: 8789487640 (8.2G), 2692153755 (2.5G) remaining [application/x-gzip]\n",
      "Saving to: ‘./temp/meteonet_2017.tar.gz’\n",
      "\n",
      "./temp/meteonet_201 100%[+++++++++++++======>]   8.19G  2.37MB/s    in 21m 29s \n",
      "\n",
      "2023-02-11 00:31:37 (1.99 MB/s) - ‘./temp/meteonet_2017.tar.gz’ saved [8789487640/8789487640]\n",
      "\n",
      "FINISHED --2023-02-11 00:31:37--\n",
      "Total wall clock time: 49m 12s\n",
      "Downloaded: 1 files, 2.5G in 49m 7s (892 KB/s)\n"
     ]
    }
   ],
   "source": [
    "url = meteonet_url_form.format(year=year)\n",
    "file_down_path = output_file_name_form.format(year=year)\n",
    "!wget url $url -O $file_down_path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "EOFError",
     "evalue": "Compressed file ended before the end-of-stream marker was reached",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[0;31mEOFError\u001b[0m                                  Traceback (most recent call last)\n",
      "\u001b[0;32m/tmp/ipykernel_15957/2973289080.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprocess_n_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_down_path_2016\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemp_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2016\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_15957/3212762275.py\u001b[0m in \u001b[0;36mprocess_n_save\u001b[0;34m(source, temp_dir, save_dir, year)\u001b[0m\n",
      "\u001b[1;32m      3\u001b[0m     \u001b[0myear\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myear\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mshutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack_archive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'data_{year}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m      7\u001b[0m     \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'data_{year}'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"/NW_weather_models_2D_parameters_{year}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/shutil.py\u001b[0m in \u001b[0;36munpack_archive\u001b[0;34m(filename, extract_dir, format)\u001b[0m\n",
      "\u001b[1;32m   1269\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_UNPACK_FORMATS\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   1270\u001b[0m         \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_UNPACK_FORMATS\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 1271\u001b[0;31m         \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextract_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   1272\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   1273\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/shutil.py\u001b[0m in \u001b[0;36m_unpack_tarfile\u001b[0;34m(filename, extract_dir)\u001b[0m\n",
      "\u001b[1;32m   1202\u001b[0m             \"%s is not a compressed or uncompressed tar file\" % filename)\n",
      "\u001b[1;32m   1203\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 1204\u001b[0;31m         \u001b[0mtarobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextractall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextract_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   1205\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   1206\u001b[0m         \u001b[0mtarobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mextractall\u001b[0;34m(self, path, members, numeric_owner)\u001b[0m\n",
      "\u001b[1;32m   2043\u001b[0m                 \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0o700\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2044\u001b[0m             \u001b[0;31m# Do not set_attrs directories, as we will do that further down\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2045\u001b[0;31m             self.extract(tarinfo, path, set_attrs=not tarinfo.isdir(),\n",
      "\u001b[0m\u001b[1;32m   2046\u001b[0m                          numeric_owner=numeric_owner)\n",
      "\u001b[1;32m   2047\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mextract\u001b[0;34m(self, member, path, set_attrs, numeric_owner)\u001b[0m\n",
      "\u001b[1;32m   2084\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2085\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2086\u001b[0;31m             self._extract_member(tarinfo, os.path.join(path, tarinfo.name),\n",
      "\u001b[0m\u001b[1;32m   2087\u001b[0m                                  \u001b[0mset_attrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mset_attrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2088\u001b[0m                                  numeric_owner=numeric_owner)\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36m_extract_member\u001b[0;34m(self, tarinfo, targetpath, set_attrs, numeric_owner)\u001b[0m\n",
      "\u001b[1;32m   2157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2158\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misreg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2159\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakefile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargetpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   2160\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2161\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargetpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mmakefile\u001b[0;34m(self, tarinfo, targetpath)\u001b[0m\n",
      "\u001b[1;32m   2206\u001b[0m                 \u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtruncate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2207\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2208\u001b[0;31m                 \u001b[0mcopyfileobj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mReadError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   2209\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2210\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmakeunknown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargetpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mcopyfileobj\u001b[0;34m(src, dst, length, exception, bufsize)\u001b[0m\n",
      "\u001b[1;32m    245\u001b[0m     \u001b[0mblocks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mremainder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdivmod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlength\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    246\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblocks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m--> 247\u001b[0;31m         \u001b[0mbuf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m    248\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mbufsize\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    249\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"unexpected end of data\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/gzip.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, size)\u001b[0m\n",
      "\u001b[1;32m    298\u001b[0m             \u001b[0;32mimport\u001b[0m \u001b[0merrno\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    299\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merrno\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEBADF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"read() on write-only GzipFile object\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m--> 300\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m    301\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    302\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/_compression.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n",
      "\u001b[1;32m     66\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreadinto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m     67\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mmemoryview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mview\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mview\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"B\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mbyte_view\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m---> 68\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbyte_view\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m     69\u001b[0m             \u001b[0mbyte_view\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m     70\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/gzip.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, size)\u001b[0m\n",
      "\u001b[1;32m    504\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbuf\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34mb\"\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m--> 506\u001b[0;31m                 raise EOFError(\"Compressed file ended before the \"\n",
      "\u001b[0m\u001b[1;32m    507\u001b[0m                                \"end-of-stream marker was reached\")\n",
      "\u001b[1;32m    508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;31mEOFError\u001b[0m: Compressed file ended before the end-of-stream marker was reached"
     ]
    }
   ],
   "source": [
    "\n",
    "process_n_save(file_down_path, temp_dir, dataset_path, year)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 2018 Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://meteonet.umr-cnrm.fr/dataset/data/NW/weather_models/2D_parameters/NW_weather_models_2D_parameters_2018.tar.gz\n",
      "./temp/meteonet_2018.tar.gz\n"
     ]
    }
   ],
   "source": [
    "year = 2018\n",
    "\n",
    "print(meteonet_url_form.format(year=year))\n",
    "print(output_file_name_form.format(year=year))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2023-02-11 00:33:44--  http://url/\n",
      "Resolving url (url)... failed: Name or service not known.\n",
      "wget: unable to resolve host address ‘url’\n",
      "--2023-02-11 00:33:45--  https://meteonet.umr-cnrm.fr/dataset/data/NW/weather_models/2D_parameters/NW_weather_models_2D_parameters_2018.tar.gz\n",
      "Resolving meteonet.umr-cnrm.fr (meteonet.umr-cnrm.fr)... 193.49.97.131\n",
      "Connecting to meteonet.umr-cnrm.fr (meteonet.umr-cnrm.fr)|193.49.97.131|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 7913261757 (7.4G) [application/x-gzip]\n",
      "Saving to: ‘./temp/meteonet_2018.tar.gz’\n",
      "\n",
      "./temp/meteonet_201 100%[===================>]   7.37G  32.7KB/s    in 53m 43s \n",
      "\n",
      "2023-02-11 01:27:29 (2.34 MB/s) - ‘./temp/meteonet_2018.tar.gz’ saved [7913261757/7913261757]\n",
      "\n",
      "FINISHED --2023-02-11 01:27:29--\n",
      "Total wall clock time: 53m 46s\n",
      "Downloaded: 1 files, 7.4G in 53m 43s (2.34 MB/s)\n"
     ]
    }
   ],
   "source": [
    "# Download Wind\n",
    "url = meteonet_url_form.format(year=year)\n",
    "file_down_path = output_file_name_form.format(year=year)\n",
    "!wget url $url -O $file_down_path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "EOFError",
     "evalue": "Compressed file ended before the end-of-stream marker was reached",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[0;31mEOFError\u001b[0m                                  Traceback (most recent call last)\n",
      "\u001b[0;32m/tmp/ipykernel_15957/2973289080.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprocess_n_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_down_path_2016\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemp_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2016\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_15957/3212762275.py\u001b[0m in \u001b[0;36mprocess_n_save\u001b[0;34m(source, temp_dir, save_dir, year)\u001b[0m\n",
      "\u001b[1;32m      3\u001b[0m     \u001b[0myear\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myear\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mshutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack_archive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'data_{year}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m      7\u001b[0m     \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemp_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'data_{year}'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"/NW_weather_models_2D_parameters_{year}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/shutil.py\u001b[0m in \u001b[0;36munpack_archive\u001b[0;34m(filename, extract_dir, format)\u001b[0m\n",
      "\u001b[1;32m   1269\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_UNPACK_FORMATS\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   1270\u001b[0m         \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_UNPACK_FORMATS\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 1271\u001b[0;31m         \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mextract_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   1272\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   1273\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/shutil.py\u001b[0m in \u001b[0;36m_unpack_tarfile\u001b[0;34m(filename, extract_dir)\u001b[0m\n",
      "\u001b[1;32m   1202\u001b[0m             \"%s is not a compressed or uncompressed tar file\" % filename)\n",
      "\u001b[1;32m   1203\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 1204\u001b[0;31m         \u001b[0mtarobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextractall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mextract_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   1205\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   1206\u001b[0m         \u001b[0mtarobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mextractall\u001b[0;34m(self, path, members, numeric_owner)\u001b[0m\n",
      "\u001b[1;32m   2043\u001b[0m                 \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0o700\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2044\u001b[0m             \u001b[0;31m# Do not set_attrs directories, as we will do that further down\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2045\u001b[0;31m             self.extract(tarinfo, path, set_attrs=not tarinfo.isdir(),\n",
      "\u001b[0m\u001b[1;32m   2046\u001b[0m                          numeric_owner=numeric_owner)\n",
      "\u001b[1;32m   2047\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mextract\u001b[0;34m(self, member, path, set_attrs, numeric_owner)\u001b[0m\n",
      "\u001b[1;32m   2084\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2085\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2086\u001b[0;31m             self._extract_member(tarinfo, os.path.join(path, tarinfo.name),\n",
      "\u001b[0m\u001b[1;32m   2087\u001b[0m                                  \u001b[0mset_attrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mset_attrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2088\u001b[0m                                  numeric_owner=numeric_owner)\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36m_extract_member\u001b[0;34m(self, tarinfo, targetpath, set_attrs, numeric_owner)\u001b[0m\n",
      "\u001b[1;32m   2157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2158\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misreg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2159\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakefile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargetpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   2160\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2161\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargetpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mmakefile\u001b[0;34m(self, tarinfo, targetpath)\u001b[0m\n",
      "\u001b[1;32m   2206\u001b[0m                 \u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtruncate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2207\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m-> 2208\u001b[0;31m                 \u001b[0mcopyfileobj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mReadError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m   2209\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m   2210\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmakeunknown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarinfo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargetpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/tarfile.py\u001b[0m in \u001b[0;36mcopyfileobj\u001b[0;34m(src, dst, length, exception, bufsize)\u001b[0m\n",
      "\u001b[1;32m    245\u001b[0m     \u001b[0mblocks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mremainder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdivmod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlength\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    246\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblocks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m--> 247\u001b[0;31m         \u001b[0mbuf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msrc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m    248\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mbufsize\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    249\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"unexpected end of data\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/gzip.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, size)\u001b[0m\n",
      "\u001b[1;32m    298\u001b[0m             \u001b[0;32mimport\u001b[0m \u001b[0merrno\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    299\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mOSError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merrno\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEBADF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"read() on write-only GzipFile object\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m--> 300\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m    301\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    302\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/_compression.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n",
      "\u001b[1;32m     66\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreadinto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m     67\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mmemoryview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mview\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mview\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"B\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mbyte_view\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m---> 68\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbyte_view\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[1;32m     69\u001b[0m             \u001b[0mbyte_view\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m     70\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/gzip.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, size)\u001b[0m\n",
      "\u001b[1;32m    504\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m    505\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbuf\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34mb\"\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m--> 506\u001b[0;31m                 raise EOFError(\"Compressed file ended before the \"\n",
      "\u001b[0m\u001b[1;32m    507\u001b[0m                                \"end-of-stream marker was reached\")\n",
      "\u001b[1;32m    508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\n",
      "\u001b[0;31mEOFError\u001b[0m: Compressed file ended before the end-of-stream marker was reached"
     ]
    }
   ],
   "source": [
    "\n",
    "process_n_save(file_down_path, temp_dir, dataset_path, year)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "b10b2d1540592ad7a19982163a30d33ff6202c4322586deec1a0c3149c76b43f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}