Preprocessing_All_Together.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob, os "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'F:\\CS_599\\social-swear-master2\\social-swear-master\\datasets\\\\tweets-5M_all_wave8.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_raw = pd.DataFrame()\n",
    "sets = [[250,1000],[1001,5000]]\n",
    "w_set = 1\n",
    "df = pd.read_csv(path,index_col=None, header=0, lineterminator='\\n')\n",
    "df = df.loc[(df['user_followers_count'] >= sets[w_set][0]) & (df['user_followers_count'] <= sets[w_set][1])]\n",
    "tweets_raw = pd.concat([tweets_raw,df])\n",
    "del df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['tweet_id', 'tweet_truncated', 'date', 'tweet_source', 'tweet_coord',\n",
       "       'tweet_place', 'text', 'text_noMentions', 'is_quote_status',\n",
       "       'is_reply_to_status', 'is_reply_to_user', 'numMentions',\n",
       "       'retweet_count', 'favorite_count', 'user_id', 'user_verified',\n",
       "       'user_description_text', 'user_followers_count', 'user_friends_count',\n",
       "       'user_listed_count', 'user_favourites_count', 'user_statuses_count',\n",
       "       'user_location', 'user_created_year', 'user_created_month',\n",
       "       'user_geo_enabled', 'user_img_url', 'user_banner_url'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_raw.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1227612"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tweets_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nallFiles = glob.glob(path + \"/*.csv\")\\ntweets_raw = pd.DataFrame()\\nsets = [[250,1000],[1001,5000]]\\nw_set = 1      #change to use different set of data\\nfor file_ in allFiles:\\n    df = pd.read_csv(file_,index_col=None, header=0, lineterminator=\\'\\n\\')\\n    df = df.loc[(df[\\'user_followers_count\\'] >= sets[w_set][0]) & (df[\\'user_followers_count\\'] <= sets[w_set][1])]\\n    tweets_raw = pd.concat([tweets_raw,df])\\n    del df\\n    '"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "allFiles = glob.glob(path + \"/*.csv\")\n",
    "tweets_raw = pd.DataFrame()\n",
    "sets = [[250,1000],[1001,5000]]\n",
    "w_set = 1      #change to use different set of data\n",
    "for file_ in allFiles:\n",
    "    df = pd.read_csv(file_,index_col=None, header=0, lineterminator='\\n')\n",
    "    df = df.loc[(df['user_followers_count'] >= sets[w_set][0]) & (df['user_followers_count'] <= sets[w_set][1])]\n",
    "    tweets_raw = pd.concat([tweets_raw,df])\n",
    "    del df\n",
    "    '''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Column descriptions:\n",
    "- Tweet content:\n",
    "    -    `text`: raw text of the tweet, unchanged\n",
    "    -    `text_noMention`': text of the tweet with @mentions removed (='NO_USER_MENTIONS' if 0 mentions)\n",
    "    -    `numMentions`: number of mentions found in the tweet\n",
    "    -    `tweet_truncated`: whether or not tweet test was truncated\n",
    "    -    `tweet_source`: type of device used to make tweet\n",
    "    -    `tweet_coordinates`: coordinates of tweet\n",
    "    -    `tweet_place`: twitter 'place' object..has many attributes, figure out how to parse later`\n",
    "\n",
    "\n",
    "\n",
    "- Tweet metadata:\n",
    "    -    `is_quote_status`: whether the tweet is a quote status of another tweet\n",
    "    -    `is_reply_to_status`: whether the tweet is a reply to someone elses tweet\n",
    "    -    `is_reply_to_user`: whether the tweet is a reply to a user\n",
    "\n",
    "\n",
    "- Account information:\n",
    "    -    `user_followers_count`: number of followers the user has\n",
    "    -    `user_friends_count`: number of people the user follows\n",
    "    -    `user_listed_count`: number of lists the user is a part of\n",
    "    -    `user_favourites_count`: number of tweets the user has liked to date\n",
    "    -    `user_statuses_count`: number of tweets this user has authored to date (note that timeline acquisition is limited to 3200 latest tweets)\n",
    "    -    `user_verified`: whether the user is a verified user\n",
    "    -    `user_description_text`: raw text of the user's profile description\n",
    "    -    `user_location`: location of user profile [MAY NOT BE PARSEABLE, authored by user]\n",
    "    -    `user_created_year`: year the profile was created\n",
    "    -    `user_created_month`: month the profile was created\n",
    "    -    `user_geo_enabled`: whether or not user account geo-enabled\n",
    "    -    `user_ing_url`: url of user's profile picture\n",
    "    -    `user_banner_url`: url of user's banner picture\n",
    "    \n",
    "\n",
    "\n",
    "- Engagement:\n",
    "    -    `retweet_count`: number of times this tweet has been retweeted (at time of collection)\n",
    "    -    `favorite_coun`': number of times this tweet has been liked (at time of collection)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the future:\n",
    "- [ ] date\n",
    "- [ ] location (from geo-coordinates or timezone)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Curate dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create new column objective with weighted sum of favorite and retweet\n",
    "tweets_raw['retweet_count']*= 5\n",
    "tweets_raw['engagment'] = tweets_raw['favorite_count'] + tweets_raw['retweet_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#derive account age [unit = #months]\n",
    "import datetime\n",
    "CURYEAR = datetime.datetime.today().year\n",
    "CURMONTH = datetime.datetime.today().month\n",
    "tweets_raw['account_age'] = (CURYEAR - tweets_raw['user_created_year'])*12 + (CURMONTH - tweets_raw['user_created_month'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = tweets_raw[[\n",
    "    'user_id',\n",
    "    'text',\n",
    "    'retweet_count',\n",
    "    'favorite_count',\n",
    "    'user_followers_count',\n",
    "    'user_friends_count',\n",
    "    'user_statuses_count',\n",
    "    'user_favourites_count',\n",
    "    'user_listed_count',\n",
    "    'user_description_text',\n",
    "    'user_location',\n",
    "    'account_age',\n",
    "    'user_created_year',\n",
    "    'user_created_month',\n",
    "    'user_geo_enabled',\n",
    "    'user_img_url',\n",
    "    'user_banner_url',\n",
    "    'tweet_truncated',\n",
    "    'tweet_source',\n",
    "    'tweet_coord',\n",
    "    'tweet_place',\n",
    "    'numMentions',\n",
    "    'date',\n",
    "    'engagment'\n",
    "]].rename({\n",
    "    'retweet_count': 'n_retweets',\n",
    "    'favorite_count': 'n_favorites',\n",
    "    'user_followers_count': 'n_user_followers',\n",
    "    'user_friends_count': 'n_user_following',\n",
    "    'user_statuses_count': 'n_user_posts',\n",
    "    'user_favourites_count': 'n_user_favs',\n",
    "    'user_listed_count': 'n_user_lists'\n",
    "}, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets['is_reply'] = tweets_raw.is_reply_to_user.astype(bool) | tweets_raw.is_reply_to_status.astype(bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#very low percentage of tweet_source is na. Just fill it with 'other'\n",
    "tweets['tweet_source'] = tweets['tweet_source'].fillna('other')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "int_cols = ['n_retweets', 'n_favorites', 'n_user_followers', 'n_user_following', 'n_user_posts',\\\n",
    "            'n_user_favs','n_user_lists','account_age','user_created_year','user_created_month',\\\n",
    "            'numMentions']\n",
    "for col in int_cols:\n",
    "    tweets[col] = tweets[col].astype(int, errors='ignore')\n",
    "COLS_TO_DROP_NA = ['user_id', 'text', 'n_retweets', 'n_favorites', 'n_user_followers','n_user_following',\\\n",
    "                   'n_user_posts', 'n_user_favs', 'n_user_lists','account_age','user_created_year',\\\n",
    "                   'user_created_month', 'user_geo_enabled','tweet_truncated', 'tweet_source','numMentions',\\\n",
    "                   'date', 'engagment','is_reply']\n",
    "tweets.dropna(axis=0, subset=COLS_TO_DROP_NA,inplace=True)  # do not consider all cols. tweet_coord and tweet_place have high number of na but may still be useful"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# no replies (under 1% percent) likely means that the user is a bot\n",
    "per_replies = tweets.groupby('user_id').is_reply.mean()\n",
    "bots_ids = per_replies[per_replies < .01].index\n",
    "tweets['is_user_bot'] = tweets.user_id.isin(bots_ids)\n",
    "tweets['is_outlier'] = False\n",
    "tweets['is_valid'] = ~tweets.is_user_bot & ~tweets.is_outlier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets['is_valid'] = ~tweets.is_user_bot #& ~tweets.is_outlier did not use in users over 5001 n_user_followers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_tweets = tweets[tweets.is_valid]\n",
    "del tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre-process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove Duplicates\n",
    "valid_tweets = valid_tweets[~valid_tweets.index.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create Dict of all tweets\n",
    "tweet_dictionary = {}\n",
    "i = 0\n",
    "for line in valid_tweets['text']:\n",
    "        tweet_dictionary[i] = line.lower()\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4.39 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#strip links\n",
    "import re\n",
    "def strip_links(text):\n",
    "    link_regex    = re.compile('((https?):((//)|(\\\\\\\\))+([\\w\\d:#@%/;$()~_?\\+-=\\\\\\.&](#!)?)*)', re.DOTALL)\n",
    "    links         = re.findall(link_regex, text)\n",
    "    for link in links:\n",
    "        text = text.replace(link[0], ', ')    \n",
    "    return text\n",
    "\n",
    "for i in range(0,len(tweet_dictionary)):\n",
    "    tweet_dictionary[i]=strip_links(tweet_dictionary[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "cList = {\n",
    "  \"ain't\": \"am not\",\n",
    "  \"aren't\": \"are not\",\n",
    "  \"can't\": \"cannot\",\n",
    "  \"can't've\": \"cannot have\",\n",
    "  \"'cause\": \"because\",\n",
    "  \"could've\": \"could have\",\n",
    "  \"couldn't\": \"could not\",\n",
    "  \"couldn't've\": \"could not have\",\n",
    "  \"didn't\": \"did not\",\n",
    "  \"doesn't\": \"does not\",\n",
    "  \"don't\": \"do not\",\n",
    "  \"hadn't\": \"had not\",\n",
    "  \"hadn't've\": \"had not have\",\n",
    "  \"hasn't\": \"has not\",\n",
    "  \"haven't\": \"have not\",\n",
    "  \"he'd\": \"he would\",\n",
    "  \"he'd've\": \"he would have\",\n",
    "  \"he'll\": \"he will\",\n",
    "  \"he'll've\": \"he will have\",\n",
    "  \"he's\": \"he is\",\n",
    "  \"how'd\": \"how did\",\n",
    "  \"how'd'y\": \"how do you\",\n",
    "  \"how'll\": \"how will\",\n",
    "  \"how's\": \"how is\",\n",
    "  \"I'd\": \"I would\",\n",
    "  \"I'd've\": \"I would have\",\n",
    "  \"I'll\": \"I will\",\n",
    "  \"I'll've\": \"I will have\",\n",
    "  \"i'm\": \"i am\",\n",
    "  \"I've\": \"I have\",\n",
    "  \"isn't\": \"is not\",\n",
    "  \"it'd\": \"it had\",\n",
    "  \"it'd've\": \"it would have\",\n",
    "  \"it'll\": \"it will\",\n",
    "  \"it'll've\": \"it will have\",\n",
    "  \"it's\": \"it is\",\n",
    "  \"let's\": \"let us\",\n",
    "  \"ma'am\": \"madam\",\n",
    "  \"mayn't\": \"may not\",\n",
    "  \"might've\": \"might have\",\n",
    "  \"mightn't\": \"might not\",\n",
    "  \"mightn't've\": \"might not have\",\n",
    "  \"must've\": \"must have\",\n",
    "  \"mustn't\": \"must not\",\n",
    "  \"mustn't've\": \"must not have\",\n",
    "  \"needn't\": \"need not\",\n",
    "  \"needn't've\": \"need not have\",\n",
    "  \"o'clock\": \"of the clock\",\n",
    "  \"oughtn't\": \"ought not\",\n",
    "  \"oughtn't've\": \"ought not have\",\n",
    "  \"shan't\": \"shall not\",\n",
    "  \"sha'n't\": \"shall not\",\n",
    "  \"shan't've\": \"shall not have\",\n",
    "  \"she'd\": \"she would\",\n",
    "  \"she'd've\": \"she would have\",\n",
    "  \"she'll\": \"she will\",\n",
    "  \"she'll've\": \"she will have\",\n",
    "  \"she's\": \"she is\",\n",
    "  \"should've\": \"should have\",\n",
    "  \"shouldn't\": \"should not\",\n",
    "  \"shouldn't've\": \"should not have\",\n",
    "  \"so've\": \"so have\",\n",
    "  \"so's\": \"so is\",\n",
    "  \"that'd\": \"that would\",\n",
    "  \"that'd've\": \"that would have\",\n",
    "  \"that's\": \"that is\",\n",
    "  \"there'd\": \"there had\",\n",
    "  \"there'd've\": \"there would have\",\n",
    "  \"there's\": \"there is\",\n",
    "  \"they'd\": \"they would\",\n",
    "  \"they'd've\": \"they would have\",\n",
    "  \"they'll\": \"they will\",\n",
    "  \"they'll've\": \"they will have\",\n",
    "  \"they're\": \"they are\",\n",
    "  \"they've\": \"they have\",\n",
    "  \"to've\": \"to have\",\n",
    "  \"wasn't\": \"was not\",\n",
    "  \"we'd\": \"we had\",\n",
    "  \"we'd've\": \"we would have\",\n",
    "  \"we'll\": \"we will\",\n",
    "  \"we'll've\": \"we will have\",\n",
    "  \"we're\": \"we are\",\n",
    "  \"we've\": \"we have\",\n",
    "  \"weren't\": \"were not\",\n",
    "  \"what'll\": \"what will\",\n",
    "  \"what'll've\": \"what will have\",\n",
    "  \"what're\": \"what are\",\n",
    "  \"what's\": \"what is\",\n",
    "  \"what've\": \"what have\",\n",
    "  \"when's\": \"when is\",\n",
    "  \"when've\": \"when have\",\n",
    "  \"where'd\": \"where did\",\n",
    "  \"where's\": \"where is\",\n",
    "  \"where've\": \"where have\",\n",
    "  \"who'll\": \"who will\",\n",
    "  \"who'll've\": \"who will have\",\n",
    "  \"who's\": \"who is\",\n",
    "  \"who've\": \"who have\",\n",
    "  \"why's\": \"why is\",\n",
    "  \"why've\": \"why have\",\n",
    "  \"will've\": \"will have\",\n",
    "  \"won't\": \"will not\",\n",
    "  \"won't've\": \"will not have\",\n",
    "  \"would've\": \"would have\",\n",
    "  \"wouldn't\": \"would not\",\n",
    "  \"wouldn't've\": \"would not have\",\n",
    "  \"y'all\": \"you all\",\n",
    "  \"y'alls\": \"you alls\",\n",
    "  \"y'all'd\": \"you all would\",\n",
    "  \"y'all'd've\": \"you all would have\",\n",
    "  \"y'all're\": \"you all are\",\n",
    "  \"y'all've\": \"you all have\",\n",
    "  \"you'd\": \"you had\",\n",
    "  \"you'd've\": \"you would have\",\n",
    "  \"you'll\": \"you will\",\n",
    "  \"you'll've\": \"you will have\",\n",
    "  \"you're\": \"you are\",\n",
    "  \"you've\": \"you have\"\n",
    "}\n",
    "\n",
    "c_re = re.compile('(%s)' % '|'.join(cList.keys()))\n",
    "\n",
    "def expandContractions(text, c_re=c_re):\n",
    "    def replace(match):\n",
    "        return cList[match.group(0)]\n",
    "    text = c_re.sub(replace, text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 22.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for i in range(0,len(tweet_dictionary)):\n",
    "    tweet_dictionary[i]=expandContractions(tweet_dictionary[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4.21 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Remove Mentions\n",
    "import string\n",
    "def strip_mentions(text):\n",
    "    entity_prefixes = ['@']\n",
    "    words = []\n",
    "    for word in text.split():\n",
    "        word = word.strip()\n",
    "        if word:\n",
    "            if word[0] not in entity_prefixes:\n",
    "                words.append(word)\n",
    "    return ' '.join(words)\n",
    "\n",
    "for i in range(0,len(tweet_dictionary)):\n",
    "    tweet_dictionary[i]=strip_mentions(tweet_dictionary[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4.04 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Remove Hashtags\n",
    "def strip_hashtags(text):\n",
    "    entity_prefixes = ['#']\n",
    "    words = []\n",
    "    for word in text.split():\n",
    "        word = word.strip()\n",
    "        if word:\n",
    "            if word[0] not in entity_prefixes:\n",
    "                words.append(word)\n",
    "    return ' '.join(words)\n",
    "\n",
    "for i in range(0,len(tweet_dictionary)):\n",
    "    tweet_dictionary[i]=strip_hashtags(tweet_dictionary[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocess for censored swear detection\n",
    "censoredSwearDict = tweet_dictionary.copy()\n",
    "for i in range(0,len(tweet_dictionary)):\n",
    "    censoredSwearDict[i]= censoredSwearDict[i].replace(r\"[,.”“]\",'')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import emoji\n",
    "from emoji import UNICODE_EMOJI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2min 33s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#demoji text\n",
    "import emoji\n",
    "for i in range(0,len(tweet_dictionary)):\n",
    "    tweet_dictionary[i]=emoji.demojize(tweet_dictionary[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "se = pd.Series(tweet_dictionary)\n",
    "valid_tweets['demoji_text'] = se.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "def eggplant_presence(text):\n",
    "    entity_prefixes = ':eggplant:'\n",
    "    for word in text.split():\n",
    "        if word == entity_prefixes:\n",
    "            return 1\n",
    "    return 0\n",
    "\n",
    "def mfinger_presence(text):\n",
    "    entity_prefixes = ':middle_finger:'\n",
    "    for word in text.split():\n",
    "        if word == entity_prefixes:\n",
    "            return 1\n",
    "    return 0\n",
    "def cursing_presence(text):\n",
    "    entity_prefixes = ':cursing:'\n",
    "    for word in text.split():\n",
    "        if word == entity_prefixes:\n",
    "            return 1\n",
    "    return 0\n",
    "def fuck_presence(text):\n",
    "    entity_prefixes = ':point_right::ok_hand:'\n",
    "    for word in text.split():\n",
    "        if word == entity_prefixes:\n",
    "            return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 6.81 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "valid_tweets['eggplant'] = valid_tweets.demoji_text.apply(lambda t: eggplant_presence(t))\n",
    "valid_tweets['mfinger'] = valid_tweets.demoji_text.apply(lambda t: mfinger_presence(t))\n",
    "valid_tweets['curse'] = valid_tweets.demoji_text.apply(lambda t: cursing_presence(t))\n",
    "valid_tweets['fuck'] = valid_tweets.demoji_text.apply(lambda t: fuck_presence(t))\n",
    "del valid_tweets['demoji_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3.84 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Remove Special characters\n",
    "import unicodedata\n",
    "def remove_special_characters(text, remove_digits=False):\n",
    "    pattern = r'[^a-zA-z0-9\\s]' if not remove_digits else r'[^a-zA-z\\s]'\n",
    "    text = re.sub(pattern, '', text)\n",
    "    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n",
    "    return text\n",
    "\n",
    "for i in range(0,len(tweet_dictionary)):\n",
    "    tweet_dictionary[i]=remove_special_characters(tweet_dictionary[i], remove_digits=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 11 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Remove unjoin emojis\n",
    "import string\n",
    "def seperate_emoji(text):\n",
    "    entity_prefixes = ['@']\n",
    "    for separator in  string.punctuation:\n",
    "        if separator not in entity_prefixes :\n",
    "            text = text.replace(separator,' ')\n",
    "    words = []\n",
    "    for word in text.split():\n",
    "        word = word.strip()\n",
    "        if word:\n",
    "            if word[0] not in entity_prefixes:\n",
    "                words.append(word)\n",
    "    return ' '.join(words)\n",
    "\n",
    "for i in range(0,len(tweet_dictionary)):\n",
    "    tweet_dictionary[i]=seperate_emoji(tweet_dictionary[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import langid\n",
    "#from langid.langid import LanguageIdentifier, model\n",
    "#identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time\n",
    "#language=[]\n",
    "#for i in range(0,len(tweet_dictionary)):\n",
    "#    language.append(langid.classify(tweet_dictionary[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Text Blob sentiment and obj columns\n",
    "\n",
    "polarity=[]\n",
    "subjectivity=[]\n",
    "language=[]\n",
    "\n",
    "for i in range(0,len(tweet_dictionary)):\n",
    "    snt = TextBlob(tweet_dictionary[i])\n",
    "    polarity.append(snt.sentiment.polarity)\n",
    "    subjectivity.append(snt.sentiment.subjectivity)\n",
    "    #if len(snt)>=3:\n",
    "    #    language.append(snt.detect_language())\n",
    "    #else:\n",
    "    #    language.append('na')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "se = pd.Series(polarity)\n",
    "valid_tweets['text_polarity'] = se.values\n",
    "se = pd.Series(subjectivity)\n",
    "valid_tweets['text_subjectivity'] = se.values\n",
    "#se = pf.Series(language)\n",
    "#valid_tweets['text_language'] = se.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#Create Vader sentiment columns\n",
    "\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "analyser = SentimentIntensityAnalyzer()\n",
    "\n",
    "comp=[]\n",
    "pos=[]\n",
    "neu=[]\n",
    "neg=[]\n",
    "for i in range(0,len(valid_tweets)):\n",
    "    snt = analyser.polarity_scores(tweet_dictionary[i])\n",
    "    #print(i)\n",
    "    #print(tweetData.at[i,'vader_comp'])\n",
    "    comp.append(snt['compound'])\n",
    "    pos.append(snt['pos'])\n",
    "    neu.append(snt['neu'])\n",
    "    neg.append(snt['neg'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "se = pd.Series(comp)\n",
    "valid_tweets['vader_comp'] = se.values\n",
    "se = pd.Series(pos)\n",
    "valid_tweets['vader_pos'] = se.values\n",
    "se = pd.Series(neu)\n",
    "valid_tweets['vader_neu'] = se.values\n",
    "se = pd.Series(neg)\n",
    "valid_tweets['vader_neg'] = se.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#processed tweets to df\n",
    "se = pd.Series(tweet_dictionary)\n",
    "valid_tweets['processed_text'] = se.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from stop_words import get_stop_words\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "stop_words = list(get_stop_words('en'))         #About 900 stopwords\n",
    "nltk_words = list(stopwords.words('english')) #About 150 stopwords\n",
    "stop_words.extend(nltk_words)\n",
    "\n",
    "def remove_stop_words(word_list):\n",
    "    output = [w for w in word_list if not w in stop_words]\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4min 45s\n"
     ]
    }
   ],
   "source": [
    "# tokenize, lemmatize, lowercase\n",
    "%time valid_tweets['text_token'] = valid_tweets.processed_text.apply(lambda t: TextBlob(t).words.lemmatize())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 40.9 s\n"
     ]
    }
   ],
   "source": [
    "%time valid_tweets['text_token_rem_stop'] = valid_tweets.text_token.apply(lambda t: remove_stop_words(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_tweets['text_len'] = valid_tweets.text.str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rudimentary: catches nearly all cases, but can be further refined\n",
    "valid_tweets['n_mentions'] = valid_tweets.text.str.count('@')\n",
    "valid_tweets['n_hashtags'] = valid_tweets.text.str.count('#')\n",
    "valid_tweets['n_links']    = valid_tweets.text.str.count('http')\n",
    "valid_tweets['n_emojis'] = valid_tweets.text.apply(emoji.emoji_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv \n",
    "with open('swearWordsSeverityDict.txt', mode='r') as infile:\n",
    "    reader = csv.reader(infile)\n",
    "    severityDict = {columns[0]:columns[1] for columns in reader}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "swearDict = dict()\n",
    "\n",
    "# function to count swears per tweet\n",
    "def countSwears(inputSentence):\n",
    "    filteredSentence = []\n",
    "    if inputSentence != inputSentence:\n",
    "        return filteredSentence\n",
    "    wordTokens = inputSentence.split(' ')\n",
    "    for w in wordTokens:\n",
    "        if w in swearWords:\n",
    "            filteredSentence.append(w)\n",
    "            if w in swearDict: \n",
    "                swearDict[w] = swearDict[w]+1\n",
    "            else:\n",
    "                swearDict[w] = 1\n",
    "    return filteredSentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to count swears per tweet\n",
    "def countSwearsWithoutIncrement(inputSentence):\n",
    "    filteredSentence = []\n",
    "    if inputSentence != inputSentence:\n",
    "        return filteredSentence\n",
    "    wordTokens = inputSentence.split(' ')\n",
    "    for w in wordTokens:\n",
    "        if w in swearWords:\n",
    "            filteredSentence.append(w)\n",
    "    return filteredSentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store swear words\n",
    "f = open('swearWords.txt', 'r+')\n",
    "swearWords = [line.strip() for line in f.readlines()]\n",
    "f.close()\n",
    "\n",
    "outputDF = pd.DataFrame()\n",
    "swearCount = 0\n",
    "swear_tweets = 0\n",
    "nonswear_tweets = 0\n",
    "presenceList = []\n",
    "countList = []\n",
    "severityList = []\n",
    "rarityList = []\n",
    "# preprocess tweets (makes a copy of the 'text' field, does not preprocess tweetData['text'])\n",
    "for i in range(0,len(valid_tweets)):\n",
    "    swearList = countSwears(tweet_dictionary[i])\n",
    "    count = len(swearList)\n",
    "    swearCount += count\n",
    "    severity = 0\n",
    "    for word in swearList:\n",
    "        severity += int(severityDict.get(word))\n",
    "    if count != 0:\n",
    "        severity = severity/count\n",
    "    else: \n",
    "        severity = 0\n",
    "    severityList.append(severity)\n",
    "    countList.append(count)\n",
    "    if count == 0:\n",
    "        presenceList.append(0)\n",
    "        nonswear_tweets += 1\n",
    "    else:\n",
    "        presenceList.append(1)\n",
    "        swear_tweets += 1\n",
    "        \n",
    "for i in range(0,len(valid_tweets)):\n",
    "    swearList = countSwearsWithoutIncrement(tweet_dictionary[i])\n",
    "    count = len(swearList)\n",
    "    rarity = 0\n",
    "    for word in swearList:\n",
    "        rarity += swearDict.get(word)/swearCount\n",
    "    if count != 0:\n",
    "        rarity = rarity/count\n",
    "    else:\n",
    "        rarity = 0\n",
    "    rarityList.append(rarity)\n",
    "\n",
    "countList = pd.Series(countList)\n",
    "presenceList = pd.Series(presenceList)\n",
    "severityList = pd.Series(severityList)\n",
    "rarityList = pd.Series(rarityList)\n",
    "\n",
    "\n",
    "\n",
    "valid_tweets['swear_count'] = countList.values\n",
    "valid_tweets['swear_present'] = presenceList.values\n",
    "valid_tweets['swear_severity'] = severityList.values\n",
    "valid_tweets['swear_rarity_by_percentage'] = rarityList.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store files of censored swear words in adjusted form\n",
    "# truncated - all censored characters truncated/compressed\n",
    "# original length - substitution of each censored character\n",
    "# special chars - censored swears that contain runs of common chars #$@&%!*\n",
    "\n",
    "with open('censoredSwearsTruncated.txt', 'r') as f:\n",
    "    censorTruncatedList = f.read().splitlines()\n",
    "with open('censoredSwearsOriginalLength.txt', 'r') as f:\n",
    "    censorOriginalList = f.read().splitlines()\n",
    "with open('censoredSwearsSpecialChars.txt', 'r') as f:\n",
    "    censoredSpecialCharsList = f.read().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# converts each string into a form to be compared with the lists above\n",
    "# for original length - converts all special chars to * (example from f$%&*r --> f****r)\n",
    "# for truncated length - converts all runs of special chars to * (example from f$%&*r --> f*r)\n",
    "# for censored chars - converts all special censor chars to ~, but a run of special chars only counts as 1\n",
    "# (example f##$()r --> f**r). The reason for this case is to prevent detection of random runs of characters \n",
    "# as being detected as swears, but also the usage of the specific special chars still need to be detected. \n",
    "# example: how is !!!! (not swear) different from !@#$ (swear). The adjusted for of !!!! is ~ (since no changes)\n",
    "# whereas the adjusted form of !@#$ is ~~~~ which is (which matches an entry from the list). The method is not perfect\n",
    "# but works in most cases.\n",
    "censoredSwearList = []\n",
    "def censoredSwearDetector(inputList):\n",
    "    newList = []\n",
    "    for w in inputList:\n",
    "        if w != '18-hole' and w != '36-hole'and  w != '54-hole' and w != '72-hole' and w != '#c25k' and w not in censoredSpecialCharsList:\n",
    "            transformedWordOriginalLength = re.sub('[^a-z]', '*', w)\n",
    "            transformedWordTruncatedLength = re.sub('[^a-z]+', '*', w)\n",
    "            transformedCensoredChars = re.sub('[#]+', '~', w)\n",
    "            transformedCensoredChars = re.sub('[$]+', '~', transformedCensoredChars)\n",
    "            transformedCensoredChars = re.sub('[@]+', '~', transformedCensoredChars)\n",
    "            transformedCensoredChars = re.sub('[&]+', '~', transformedCensoredChars)\n",
    "            transformedCensoredChars = re.sub('[%]+', '~', transformedCensoredChars)\n",
    "            transformedCensoredChars = re.sub('[!]+', '~', transformedCensoredChars)\n",
    "            transformedCensoredChars = re.sub('[*]+', '~', transformedCensoredChars)\n",
    "            if transformedWordTruncatedLength in censorTruncatedList:\n",
    "                #print(w + '   truncated length   ' + transformedWordTruncatedLength)\n",
    "                censoredSwearList.append(w)\n",
    "                return 1\n",
    "            elif transformedWordOriginalLength in censorOriginalList:\n",
    "                #print(w + '   original length')\n",
    "                censoredSwearList.append(w)\n",
    "                return 1\n",
    "            elif w == '@ss' or w == '*@ss' or w == '@ss*':\n",
    "                #print(w)\n",
    "                censoredSwearList.append(w)\n",
    "                return 1\n",
    "            elif transformedCensoredChars in censoredSpecialCharsList:\n",
    "                #print(w + '   censored characters')\n",
    "                censoredSwearList.append(w)\n",
    "                return 1\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtering sentences for censored swear word detection. Must not be purely alphabetic or purely numeric\n",
    "def filterWordsCensored(inputSentence):\n",
    "    wordTokens = inputSentence.split(' ')\n",
    "    filteredSentence = []\n",
    "    for w in wordTokens:\n",
    "        if not w.isalpha() and not w.isspace() and not w.isnumeric():\n",
    "            filteredSentence.append(w)\n",
    "    return filteredSentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply fitlering\n",
    "for i in range(0,len(valid_tweets)):\n",
    "    censoredSwearDict[i] = filterWordsCensored(censoredSwearDict[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create column of censored swear presence\n",
    "presenceCensoredList = []\n",
    "for i in range(0,len(valid_tweets)):\n",
    "    count = censoredSwearDetector(censoredSwearDict[i])\n",
    "    presenceCensoredList.append(count)\n",
    "\n",
    "presenceCensoredList = pd.Series(presenceCensoredList)\n",
    "valid_tweets['censored_presence'] = presenceCensoredList.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(censoredSwearList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from dateutil.parser import parse as parse_date\n",
    "\n",
    "# parse dates\n",
    "%time valid_tweets.date = valid_tweets.date.apply(parse_date)\n",
    "\n",
    "%time valid_tweets['day_of_week'] = valid_tweets.date.apply(lambda d: int(d.strftime('%w')))\n",
    "\n",
    "%time valid_tweets['hour_of_day'] = valid_tweets.date.apply(lambda d: int(d.strftime('%H')))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_tweets['per_engagement'] = valid_tweets.engagment / valid_tweets.n_user_followers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculate holidays\n",
    "#import holidays\n",
    "#from datetime import date\n",
    "#chosen_holidays = [holidays.UnitedStates(), holidays.EuropeanCentralBank()]\n",
    "#def is_holiday(d) -> bool:\n",
    "#    \"\"\" input can be str, date object, etc (anything accepted by the holidays package) \"\"\"\n",
    "    #if \n",
    "#    return any(d in h for h in chosen_holidays)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#have_date = ~valid_tweets.isna(valid_tweets.date)\n",
    "#valid_tweets['on_holiday'] = valid_tweets[have_date].date.map(is_holiday)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%time \n",
    "#have_date = ~valid_tweets.isna(valid_tweets.date)\n",
    "\n",
    "#valid_tweets['on_holiday'] = valid_tweets[have_date].date.map(is_holiday)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%time valid_tweets.to_csv('50M_250_1000_8.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}