BelgiumBikesAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import json\n",
    "import requests\n",
    "import re\n",
    "import argparse\n",
    "from pprint import pprint\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import unicodedata\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Item(object):\n",
    "    def __init__(self, price, location, datetime):\n",
    "        self.price = price\n",
    "        self.location = location\n",
    "        self.datetime = datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_int(price_str):\n",
    "    print 'price to be converted: ', price_str\n",
    "    price_str = re.search(r'\\d+', price_str)\n",
    "    try:\n",
    "        return int(price_str.group())\n",
    "    except Exception:\n",
    "        return -9999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_string(item):\n",
    "    return unicodedata.normalize('NFKD',item).encode('ascii', 'ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's convert these into graceful functions\n",
    "def get_price(item):\n",
    "    try:\n",
    "        price = item.find('div', {'class':'listed-item-price'}).get_text()\n",
    "    except Exception:\n",
    "        return None\n",
    "    \n",
    "    price = convert_to_int(convert_to_string(price))\n",
    "    return price\n",
    "\n",
    "def get_location(item):\n",
    "    try:\n",
    "        location = item.find('address',{'class':'listed-item-place'}).get_text()\n",
    "    except Exception:\n",
    "        return None\n",
    "    \n",
    "    location = convert_to_string(location)\n",
    "    return location.strip()\n",
    "\n",
    "def get_date(item):\n",
    "    try:\n",
    "        date = item.find('time',{'class':'listed-item-date'}).get_text()\n",
    "    except Exception:\n",
    "        return None\n",
    "    date = convert_to_string(date)\n",
    "    return date.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_item(item):    \n",
    "    item = Item(get_price(item), get_location(item), get_date(item))\n",
    "    return item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "price to be converted:   56,99\n",
      "price to be converted:   400,00\n",
      "price to be converted:  Ne s'applique pas\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   400,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   5,00\n",
      "price to be converted:   25,00\n",
      "price to be converted:   195,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   280,00\n",
      "price to be converted:   125,00\n",
      "price to be converted:   1.950,00\n",
      "price to be converted:   15,00\n",
      "price to be converted:   10,00\n",
      "price to be converted:   95,00\n",
      "price to be converted:   30,00\n",
      "price to be converted:   79,00\n",
      "price to be converted:   275,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:   60,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   200,00\n",
      "price to be converted:   550,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   50,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   200,00\n",
      "price to be converted:   1.500,00\n",
      "price to be converted:   150,00\n",
      "price to be converted:   400,00\n",
      "price to be converted:   95,00\n",
      "price to be converted:   130,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:   90,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   1.650,00\n",
      "price to be converted:  Ne s'applique pas\n",
      "price to be converted:   4.900,00\n",
      "price to be converted:   495,00\n",
      "price to be converted:   90,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   550,00\n",
      "price to be converted:   35,00\n",
      "price to be converted:   150,00\n",
      "price to be converted:   650,00\n",
      "price to be converted:   625,00\n",
      "price to be converted:   200,00\n",
      "price to be converted:   700,00\n",
      "price to be converted:   200,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   170,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   165,00\n",
      "price to be converted:   90,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   170,00\n",
      "price to be converted:   140,00\n",
      "price to be converted:   40,00\n",
      "price to be converted:   15,00\n",
      "price to be converted:   125,00\n",
      "price to be converted:   90,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   90,00\n",
      "price to be converted:   300,00\n",
      "price to be converted:   230,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   13,00\n",
      "price to be converted:   350,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:   2.999,00\n",
      "price to be converted:  Ne s'applique pas\n",
      "price to be converted:   1.650,00\n",
      "price to be converted:   4.900,00\n",
      "price to be converted:   130,00\n",
      "price to be converted:   1.200,00\n",
      "price to be converted:   580,00\n",
      "price to be converted:   2.650,00\n",
      "price to be converted:   35,00\n",
      "price to be converted:   35,00\n",
      "price to be converted:   10,00\n",
      "price to be converted:   40,00\n",
      "price to be converted:   25,00\n",
      "price to be converted:   60,00\n",
      "price to be converted:   130,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   150,00\n",
      "price to be converted:   160,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   1.250,00\n",
      "price to be converted:   600,00\n",
      "price to be converted:   160,00\n",
      "price to be converted:   25,00\n",
      "price to be converted:   25,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   20,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   35,00\n",
      "price to be converted:   90,00\n",
      "price to be converted:   110,00\n",
      "price to be converted:   85,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   25,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:   4.900,00\n",
      "price to be converted:  Ne s'applique pas\n",
      "price to be converted:   400,00\n",
      "price to be converted:   25,00\n",
      "price to be converted:   40,00\n",
      "price to be converted:   500,00\n",
      "price to be converted:   40,00\n",
      "price to be converted:   239,00\n",
      "price to be converted:   45,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   120,00\n",
      "price to be converted:   600,00\n",
      "price to be converted:   800,00\n",
      "price to be converted:   900,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   25,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   80,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   160,00\n",
      "price to be converted:   2.100,00\n",
      "price to be converted:   450,00\n",
      "price to be converted:   400,00\n",
      "price to be converted:   40,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   30,00\n",
      "price to be converted:   140,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   200,00\n",
      "price to be converted:   800,00\n",
      "price to be converted:   250,00\n",
      "price to be converted:   3.400,00\n",
      "price to be converted:   3.400,00\n",
      "price to be converted:   28,00\n",
      "price to be converted:   85,00\n",
      "price to be converted:   270,00\n",
      "price to be converted:   10,00\n",
      "price to be converted:   369,00\n",
      "price to be converted:   1.650,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   370,00\n",
      "price to be converted:   950,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   35,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   20,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   100,00\n",
      "price to be converted:   300,00\n",
      "price to be converted:   150,00\n",
      "price to be converted:   45,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   50,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   70,00\n",
      "price to be converted:   170,00\n",
      "price to be converted:   500,00\n",
      "price to be converted:   699,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   850,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   80,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   1.650,00\n",
      "price to be converted:   10,00\n",
      "price to be converted:  Ne s'applique pas\n",
      "price to be converted:   75,00\n",
      "price to be converted:   40,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   30,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   60,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   250,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   1.100,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   120,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   90,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   120,00\n",
      "price to be converted:   250,00\n",
      "price to be converted:   300,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   300,00\n",
      "price to be converted:   150,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   70,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   65,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   250,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   45,00\n",
      "price to be converted:   42,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   369,00\n",
      "price to be converted:  Ne s'applique pas\n",
      "price to be converted:   500,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   175,00\n",
      "price to be converted:   1.500,00\n",
      "price to be converted:   65,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   130,00\n",
      "price to be converted:   60,00\n",
      "price to be converted:   40,00\n",
      "price to be converted:   42,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   40,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   200,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   150,00\n",
      "price to be converted:   17,00\n",
      "price to be converted:   35,00\n",
      "price to be converted:   45,00\n",
      "price to be converted:   165,00\n",
      "price to be converted:   165,00\n",
      "price to be converted:   350,00\n",
      "price to be converted:   45,00\n",
      "price to be converted:   15,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   130,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   2.499,00\n",
      "price to be converted:   120,00\n",
      "price to be converted:   1.700,00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "price to be converted:   4.900,00\n",
      "price to be converted:   369,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   1.600,00\n",
      "price to be converted:   165,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   130,00\n",
      "price to be converted:   1.000,00\n",
      "price to be converted:   35,00\n",
      "price to be converted:   120,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   55,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   250,00\n",
      "price to be converted:   140,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   68,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   67,00\n",
      "price to be converted:   15,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   50,00\n",
      "price to be converted:   2.800,00\n",
      "price to be converted:   1.090,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   95,00\n",
      "price to be converted:   45,00\n",
      "price to be converted:   40,00\n",
      "price to be converted:   20,00\n",
      "price to be converted:   1.350,00\n",
      "price to be converted:   1.999,00\n",
      "price to be converted:   60,00\n",
      "price to be converted:   650,00\n",
      "price to be converted:   800,00\n",
      "price to be converted:   250,00\n",
      "price to be converted:   200,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:  Ne s'applique pas\n",
      "price to be converted:   4.900,00\n",
      "price to be converted:   35,00\n",
      "price to be converted:   150,00\n",
      "price to be converted:   85,00\n",
      "price to be converted:   1.000,00\n",
      "price to be converted:   10,00\n",
      "price to be converted:   300,00\n",
      "price to be converted:   300,00\n",
      "price to be converted:   1.900,00\n",
      "price to be converted:   25,00\n",
      "price to be converted:   40,00\n",
      "price to be converted:   1.500,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:   250,00\n",
      "price to be converted:   90,00\n",
      "price to be converted:   49,99\n",
      "price to be converted:   20,00\n",
      "price to be converted:   3.000,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   20,00\n",
      "price to be converted:   25,00\n",
      "price to be converted:   35,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:   120,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   960,00\n",
      "price to be converted:   60,00\n",
      "price to be converted:   140,00\n",
      "price to be converted:   80,00\n",
      "price to be converted:   50,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   120,00\n",
      "price to be converted:   165,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   10,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   1.950,00\n",
      "price to be converted:   280,00\n",
      "price to be converted:   1.100,00\n",
      "price to be converted:   30,00\n",
      "price to be converted:   200,00\n",
      "price to be converted:   220,00\n",
      "price to be converted:   30,00\n",
      "price to be converted:   65,00\n",
      "price to be converted:   400,00\n",
      "price to be converted:   360,00\n",
      "price to be converted:   45,00\n",
      "price to be converted:   170,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   40,00\n",
      "price to be converted:   35,00\n",
      "price to be converted:   120,00\n",
      "price to be converted:   75,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   40,00\n",
      "price to be converted:   30,00\n",
      "price to be converted:   1.000,00\n",
      "price to be converted:   160,00\n",
      "price to be converted:   160,00\n",
      "price to be converted:   70,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:  Ne s'applique pas\n",
      "price to be converted:   50,00\n",
      "price to be converted:   100,00\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   50,00\n",
      "price to be converted:  Gratuit\n",
      "price to be converted:  Prix a disc.\n",
      "price to be converted:   350,00\n",
      "price to be converted:   1.250,00\n",
      "price to be converted:   100,00\n"
     ]
    }
   ],
   "source": [
    "# these are results from the 1st page - we would like results from all X pages\n",
    "parsed_items = []\n",
    "offset_jump = 35\n",
    "for i in range(0, 10):\n",
    "    offset = int(offset_jump*i)\n",
    "    url_path = \"https://www.2ememain.be/v%C3%A9los/v%C3%A9los/2/velo/?offset={}\".format(offset)\n",
    "    raw = requests.get(url_path).text\n",
    "    soup = BeautifulSoup(raw,'lxml')\n",
    "    \n",
    "    items = soup.findAll('article')\n",
    "    for item in items:\n",
    "        parsed_items.append(parse_item(item))\n",
    "    time.sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "380"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(parsed_items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data = [x.__dict__ for x in parsed_items],\n",
    "                  columns = parsed_items[0].__dict__.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's remove the -9999 values in price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.price != -9999]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    330.000000\n",
       "mean     136.763636\n",
       "std      180.087260\n",
       "min        1.000000\n",
       "25%       35.000000\n",
       "50%       75.000000\n",
       "75%      150.000000\n",
       "max      960.000000\n",
       "Name: price, dtype: float64"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.price.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# strange to see a bike for 1 euro - let's set a threshold to remove these prices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.price > 30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    249.000000\n",
       "mean     177.016064\n",
       "std      190.686767\n",
       "min       35.000000\n",
       "25%       65.000000\n",
       "50%      100.000000\n",
       "75%      200.000000\n",
       "max      960.000000\n",
       "Name: price, dtype: float64"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.price.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}