01_scrape_twitter_output.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scrape imports\n",
    "import tweepy\n",
    "from tweepy import OAuthHandler\n",
    "from tweepy import Stream\n",
    "from tweepy.streaming import StreamListener\n",
    "from datetime import datetime\n",
    "import shutil\n",
    "\n",
    "# Processing imports\n",
    "import pandas as pd\n",
    "import json\n",
    "import regex as re\n",
    "import nltk\n",
    "from nltk.tokenize import RegexpTokenizer \n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "#pandas viewing options\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This big cell defines all the functions that are used at the bottom.\n",
    "Sorry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initiate_stream(to_track):\n",
    "    consumer_key = '6Pwj0nGgbTirwFMDChSXQRCCg'\n",
    "    consumer_secret = '8ALbMB4Y3rhuXDSvDIGaZOtSnWjsL57v1PvhTdoeOOID5uXkaC'\n",
    "    access_token = '581353008-R5jKk3zEYHeVE8zWjVmX2qE960PqzejohROAxf1Q'\n",
    "    access_secret = 'tOcjyph8fMxdS8EIfAbX1ntyccyraHH7OWhPLmZsL1qDN'\n",
    "\n",
    "    auth = OAuthHandler(consumer_key, consumer_secret)\n",
    "    auth.set_access_token(access_token, access_secret)\n",
    " \n",
    "    api = tweepy.API(auth)\n",
    "\n",
    "    class MyListener(StreamListener):\n",
    "        def on_data(self, data):\n",
    "            try:\n",
    "                with open('scrape.json', 'a') as f:\n",
    "                    f.write(data)\n",
    "                    return True\n",
    "            except BaseException as e:\n",
    "                print(\"Error on_data: %s\" % str(e))\n",
    "            return True\n",
    " \n",
    "        def on_error(self, status):\n",
    "            print(status)\n",
    "            return True\n",
    "        \n",
    "    twitter_stream = Stream(auth, MyListener())\n",
    "    twitter_stream.filter(track=to_track)\n",
    "\n",
    "def after_stream():\n",
    "    now = datetime.now()\n",
    "    if now.month < 10:\n",
    "        month = '0'+str(now.month)\n",
    "    else: month = str(now.month)\n",
    "    if now.day < 10:\n",
    "        day = '0'+str(now.day)\n",
    "    else: day = str(now.day)\n",
    "    if now.hour < 10:\n",
    "        hour = '0'+str(now.hour)\n",
    "    else: hour = str(now.hour)\n",
    "    if now.minute < 10:\n",
    "        minute = '0'+str(now.minute)\n",
    "    else: minute = str(now.minute)\n",
    "    if now.second < 10:\n",
    "        second = '0'+str(now.second)\n",
    "    else: second = str(now.second)\n",
    "\n",
    "    now_str = str(now.year)+month+day+'_'+hour+minute+second\n",
    "    dest = './tweepy_scrape_'+now_str+'.json'\n",
    "    shutil.move('./scrape.json', dest)\n",
    "\n",
    "    input_filepath = dest\n",
    "\n",
    "    # Read the entire file into a python array\n",
    "    with open(input_filepath, 'r') as f:\n",
    "        data = f.readlines()\n",
    "\n",
    "    jsons = []\n",
    "    for ind in data:\n",
    "        jsons.append(json.loads(ind))\n",
    "\n",
    "    # Read into json\n",
    "    with open('my_test_data.json', 'w') as outfile:\n",
    "        json.dump(jsons, outfile)\n",
    "    \n",
    "    # Create a pandas series of the json data\n",
    "    json_df = pd.read_json('my_test_data.json',typ='series')\n",
    "\n",
    "    # Convert the series into list of dictionaries\n",
    "    dict_list = []\n",
    "    for row in json_df:\n",
    "        temp_dict = {}\n",
    "    \n",
    "        # Basic tweet data: ID and timestamp\n",
    "        temp_dict['id'] = row['id']\n",
    "        temp_dict['created_at'] = row['created_at']\n",
    "    \n",
    "        try:\n",
    "            temp_dict['url'] = row['entities']['urls'][0]['url']\n",
    "        except:\n",
    "            # Recreate url using username and tweet ID\n",
    "            user = row['user']['screen_name']\n",
    "            url = 'https://twitter.com/'+user+'/status/'+row['id_str']\n",
    "            temp_dict['url'] = url\n",
    "    \n",
    "        # Tweet location data, if available\n",
    "        temp_dict['geo'] = row['geo']\n",
    "        temp_dict['coords'] = row['coordinates']\n",
    "    \n",
    "        # Retrieve tweet text\n",
    "        try:\n",
    "            temp_dict['text'] = row['extended_tweet']['full_text']\n",
    "        except:\n",
    "            temp_dict['text'] = row['text']\n",
    "    \n",
    "        # Try to retrieve image url, if available\n",
    "        try:\n",
    "            temp_dict['image_url'] = row['extended_tweet']['entities']['media'][0]['media_url']\n",
    "        except:\n",
    "            temp_dict['image_url'] = None\n",
    "    \n",
    "        # Get user data\n",
    "        temp_dict['user_id'] = row['user']['id']\n",
    "        temp_dict['user_name'] = row['user']['name']\n",
    "        temp_dict['user_screen_name'] = row['user']['screen_name']\n",
    "        temp_dict['user_location'] = row['user']['location']\n",
    "        temp_dict['user_geoenabled'] = row['user']['geo_enabled']\n",
    "    \n",
    "        # Append temp dict to dict_list\n",
    "        dict_list.append(temp_dict)\n",
    "    \n",
    "    # Create a dataframe from the list of dictionaries\n",
    "    data = pd.DataFrame(dict_list,index=list(range(len(dict_list))))\n",
    "\n",
    "    filename = 'processed_tweepy_'+now_str+'.csv'\n",
    "\n",
    "    # Write to csv\n",
    "    data.to_csv(filename)\n",
    "    df = pd.read_csv(filename, index_col=0)     # Just checking that it worked\n",
    "    return df\n",
    "\n",
    "def twitter_in(df,to_track):\n",
    "    #tokenize the tweets\n",
    "    tokenize_tweets = RegexpTokenizer(r'\\w+')\n",
    "        \n",
    "    #create a list of tokens\n",
    "    texts = [tokenize_tweets.tokenize(value) for index, value in df.text.iteritems()]\n",
    "    \n",
    "    #remove stopwords\n",
    "    texts = [w for w in texts if not w in stopwords.words('english')]\n",
    "        \n",
    "    #nake new column for tokens\n",
    "    df['token_text'] = texts\n",
    "        \n",
    "    closed_words = ['closed', 'flooded', 'inaccessible', 'road', 'stuck', 'flood', 'close', \n",
    "                    'drive', 'lane', 'street', 'roadclosed']\n",
    "    \n",
    "    # Get tracking words compatible and append to closed_words\n",
    "    to_track = [tokenize_tweets.tokenize(value) for value in to_track]\n",
    "    track = []\n",
    "    for sublist in to_track:\n",
    "        for item in sublist:\n",
    "            track.append(item)\n",
    "    track = [word.lower() for word in track]\n",
    "    track = [w for w in track if not w in stopwords.words('english')]\n",
    "    closed_words += track\n",
    "    \n",
    "    count_list = []\n",
    "    for word_list in df.token_text:\n",
    "        count = 0   \n",
    "        for word in word_list:\n",
    "            if word.lower() in closed_words:\n",
    "                count += 1\n",
    "            #if word.lower() in track:\n",
    "            #    count += 1\n",
    "            if word.lower() == track[0]: \n",
    "                count += 100\n",
    "            if word.lower() == 'rt':\n",
    "                count -= 1\n",
    "        \n",
    "        count_list.append(count) #could add different amounts for certain words\n",
    "    df['tweet_rank'] = count_list\n",
    "       \n",
    "    if df[df.tweet_rank > 101].shape[0] == 0:\n",
    "        print('no relevant tweets')\n",
    "    else: \n",
    "        tops = df[df.tweet_rank > 0].sort_values(by='tweet_rank', ascending=False)\n",
    "        \n",
    "        # Write to csv\n",
    "        out_file = './tweepy_out_.csv'\n",
    "        tops.to_csv(out_file)\n",
    "        \n",
    "        tops = df[df.tweet_rank > 0].sort_values(by='tweet_rank', ascending=False).head(5)\n",
    "        return tops[['text', 'url', 'tweet_rank']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This cell starts the live scrape. Remember to kernel interrupt when you're done collecting.\n",
    "## This is also where you define your search terms in `to_track`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mWantReadError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    279\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mOpenSSL\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSSL\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSysCallError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/OpenSSL/SSL.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1813\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_lib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSSL_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ssl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbuf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1814\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_raise_ssl_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ssl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1815\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/OpenSSL/SSL.py\u001b[0m in \u001b[0;36m_raise_ssl_error\u001b[0;34m(self, ssl, result)\u001b[0m\n\u001b[1;32m   1613\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0merror\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_lib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSSL_ERROR_WANT_READ\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1614\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mWantReadError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1615\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0merror\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_lib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSSL_ERROR_WANT_WRITE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mWantReadError\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-149-19c474aafe60>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0minitiate_stream\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mto_track\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-148-76db13f8ce42>\u001b[0m in \u001b[0;36minitiate_stream\u001b[0;34m(to_track)\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0mtwitter_stream\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mStream\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mauth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMyListener\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m     \u001b[0mtwitter_stream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrack\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mto_track\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mafter_stream\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tweepy/streaming.py\u001b[0m in \u001b[0;36mfilter\u001b[0;34m(self, follow, track, async, locations, stall_warnings, languages, encoding, filter_level)\u001b[0m\n\u001b[1;32m    448\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'delimited'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'length'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    449\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhost\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'stream.twitter.com'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 450\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masync\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    451\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    452\u001b[0m     def sitestream(self, follow, stall_warnings=False,\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tweepy/streaming.py\u001b[0m in \u001b[0;36m_start\u001b[0;34m(self, async)\u001b[0m\n\u001b[1;32m    362\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_thread\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    363\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 364\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    365\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    366\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_closed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tweepy/streaming.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    264\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msnooze_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msnooze_time_step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistener\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_connect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mssl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSSLError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m                 \u001b[0;31m# This is still necessary, as a SSLError can actually be\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tweepy/streaming.py\u001b[0m in \u001b[0;36m_read_loop\u001b[0;34m(self, resp)\u001b[0m\n\u001b[1;32m    314\u001b[0m             \u001b[0mlength\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclosed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m                 \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_line\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    317\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistener\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeep_alive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# keep-alive new lines are expected\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/tweepy/streaming.py\u001b[0m in \u001b[0;36mread_line\u001b[0;34m(self, sep)\u001b[0m\n\u001b[1;32m    179\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m                 \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 181\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_chunk_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    182\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    183\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/urllib3/response.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, amt, decode_content, cache_content)\u001b[0m\n\u001b[1;32m    382\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m                 \u001b[0mcache_content\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 384\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    385\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mamt\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Platform-specific: Buggy versions of Python.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m                     \u001b[0;31m# Close the connection when no data is returned\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m    447\u001b[0m             \u001b[0;31m# Amount is given, implement using readinto\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m             \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbytearray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m             \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadinto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mmemoryview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtobytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    451\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    481\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    482\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunked\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 483\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_readinto_chunked\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    485\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlength\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_readinto_chunked\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    576\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    577\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 578\u001b[0;31m                 \u001b[0mchunk_left\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_chunk_left\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    579\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mchunk_left\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    580\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mtotal_bytes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_get_chunk_left\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    544\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_safe_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# toss the CRLF at the end of the chunk\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 546\u001b[0;31m                 \u001b[0mchunk_left\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_next_chunk_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    547\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    548\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mIncompleteRead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mb''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/http/client.py\u001b[0m in \u001b[0;36m_read_next_chunk_size\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    504\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_next_chunk_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    505\u001b[0m         \u001b[0;31m# Read the next chunk size from the file\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 506\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    507\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    508\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"chunk size\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    584\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    290\u001b[0m                 \u001b[0;32mraise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mOpenSSL\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSSL\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mWantReadError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m             \u001b[0mrd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait_for_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msocket\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgettimeout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    293\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mrd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'The read operation timed out'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/urllib3/util/wait.py\u001b[0m in \u001b[0;36mwait_for_read\u001b[0;34m(socks, timeout)\u001b[0m\n\u001b[1;32m     31\u001b[0m     \u001b[0;32mor\u001b[0m \u001b[0moptionally\u001b[0m \u001b[0ma\u001b[0m \u001b[0msingle\u001b[0m \u001b[0msocket\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpassed\u001b[0m \u001b[0;32min\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mReturns\u001b[0m \u001b[0ma\u001b[0m \u001b[0mlist\u001b[0m \u001b[0mof\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m     sockets that can be read from immediately. \"\"\"\n\u001b[0;32m---> 33\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_wait_for_io_events\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msocks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mEVENT_READ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/urllib3/util/wait.py\u001b[0m in \u001b[0;36m_wait_for_io_events\u001b[0;34m(socks, events, timeout)\u001b[0m\n\u001b[1;32m     24\u001b[0m             \u001b[0mselector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevents\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         return [key[0].fileobj for key in\n\u001b[0;32m---> 26\u001b[0;31m                 selector.select(timeout) if key[1] & events]\n\u001b[0m\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/urllib3/util/selectors.py\u001b[0m in \u001b[0;36mselect\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    511\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    512\u001b[0m             kevent_list = _syscall_wrapper(self._kqueue.control, True,\n\u001b[0;32m--> 513\u001b[0;31m                                            None, max_events, timeout)\n\u001b[0m\u001b[1;32m    514\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    515\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mkevent\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkevent_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.6/site-packages/urllib3/util/selectors.py\u001b[0m in \u001b[0;36m_syscall_wrapper\u001b[0;34m(func, _, *args, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m         and recalculate their timeouts. \"\"\"\n\u001b[1;32m     63\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mOSError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mselect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m             \u001b[0merrcode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "to_track = ['#HurricanePython', 'dc', 'georgetown', 'wisconson ave', 'road', 'closed']\n",
    "\n",
    "\n",
    "initiate_stream(to_track)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#### KERNEL INTERRUPT !!!!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>tweet_rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>#HurricanePython #waterisrising Flooding in georgetown has made it impossible to get through wisconson. #roadclosed #dc #flood #test #notreally</td>\n",
       "      <td>https://t.co/GXW9iUsWqM</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>705</th>\n",
       "      <td>#hurricanepython has hit DC! Looked out my window and saw a tree down on Wisconson and P St. The road is totally closed to traffic in both directions!</td>\n",
       "      <td>https://t.co/XCXLqpbqKA</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623</th>\n",
       "      <td>#hurricanepython is hitting dc! A lamppost fell down in the middle of mcpherson sq and is blocking 15th and m! #roadclosed</td>\n",
       "      <td>https://twitter.com/HowesJNelson/status/1048196963601199104</td>\n",
       "      <td>103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>#hurricanepython is tearing up the district of columbia! #somuchwind #wisconsonisamess #hurricaneparty #hurricanepcrazy #hurricane #dc #somanyhashtags</td>\n",
       "      <td>https://t.co/PRNfRQieRP</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>#Hurricanepython is so crazy! #water</td>\n",
       "      <td>https://twitter.com/HowesJNelson/status/1048196700291182592</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                       text  \\\n",
       "890  #HurricanePython #waterisrising Flooding in georgetown has made it impossible to get through wisconson. #roadclosed #dc #flood #test #notreally          \n",
       "705  #hurricanepython has hit DC! Looked out my window and saw a tree down on Wisconson and P St. The road is totally closed to traffic in both directions!   \n",
       "623  #hurricanepython is hitting dc! A lamppost fell down in the middle of mcpherson sq and is blocking 15th and m! #roadclosed                               \n",
       "797  #hurricanepython is tearing up the district of columbia! #somuchwind #wisconsonisamess #hurricaneparty #hurricanepcrazy #hurricane #dc #somanyhashtags   \n",
       "132  #Hurricanepython is so crazy! #water                                                                                                                     \n",
       "\n",
       "                                                             url  tweet_rank  \n",
       "890  https://t.co/GXW9iUsWqM                                      106         \n",
       "705  https://t.co/XCXLqpbqKA                                      105         \n",
       "623  https://twitter.com/HowesJNelson/status/1048196963601199104  103         \n",
       "797  https://t.co/PRNfRQieRP                                      102         \n",
       "132  https://twitter.com/HowesJNelson/status/1048196700291182592  101         "
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = after_stream()\n",
    "twitter_in(df,to_track)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}