Sentiment_Analysis_with_Python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis with Python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What is sentiment analysis ?\n",
    "\n",
    "Briefly: recognize the \"feeling\" of some text, happy, sad, pos, neg, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Download data\n",
    "# NLTK Corpora Twitter Samples\n",
    "# http://www.nltk.org/nltk_data/\n",
    "import requests, zipfile, io\n",
    "\n",
    "url = \"https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/twitter_samples.zip\"\n",
    "r = requests.get(url)\n",
    "z = zipfile.ZipFile(io.BytesIO(r.content))\n",
    "z.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import csv\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn import cross_validation\n",
    "from sklearn.cross_validation import KFold\n",
    "from sklearn import svm\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import classification_report\n",
    "import pickle\n",
    "\n",
    "\n",
    "curr_dir = os.getcwd()\n",
    "\n",
    "classes = ['pos', 'neg']\n",
    "\n",
    "_data = []\n",
    "_labels = []\n",
    "for line in open (os.path.join(curr_dir ,\"twitter_samples/\") + r'positive_tweets.json', 'r'):\n",
    "    _data.append(json.loads(line)['text'])\n",
    "    _labels.append(\"pos\")\n",
    "\n",
    "\n",
    "for line in open (os.path.join(curr_dir ,\"twitter_samples/\") + r'negative_tweets.json', 'r'):\n",
    "    _data.append(json.loads(line)['text'])\n",
    "    _labels.append(\"neg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#### Create feature vectors\n",
    "vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.8, stop_words='english', use_idf=True)\n",
    "\n",
    "data_vectors = vectorizer.fit_transform(_data)\n",
    "\n",
    "cv=KFold(data_vectors.shape[0], n_folds=10, shuffle=True, random_state=1)\n",
    "\n",
    "# Perform classification with MultinomialNB\n",
    "clf = MultinomialNB()\n",
    "\n",
    "scores = cross_validation.cross_val_score(clf, data_vectors, _labels, cv=cv)\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std()))\n",
    "clf.fit(data_vectors, _labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('models/clf.pkl', 'wb') as fmodel:\n",
    "    pickle.dump(clf, fmodel)\n",
    "with open('models/vocabulary.pkl', 'wb') as fvocabulary:\n",
    "    pickle.dump(vectorizer.vocabulary_, fvocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('models/clf.pkl', 'rb') as fmodel:\n",
    "    clf = pickle.load(fmodel)\n",
    "with open('models/vocabulary.pkl', 'rb') as fvocabulary:\n",
    "    vocabulary = pickle.load(fvocabulary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get your keys here:\n",
    "https://apps.twitter.com/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tweepy\n",
    "from tweepy import OAuthHandler\n",
    " \n",
    "consumer_key = 'yourkey'\n",
    "consumer_secret = 'yourkey'\n",
    "access_token = 'yourkey'\n",
    "access_secret = 'yourkey'\n",
    " \n",
    "auth = OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_secret)\n",
    " \n",
    "api = tweepy.API(auth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('tweets.csv', 'w') as csvfile:\n",
    "    tweet_writer = csv.writer(csvfile)\n",
    "    for tweet in tweepy.Cursor(api.search, q='trump', languages=[\"en\"]).items(50):\n",
    "        tweet_writer.writerow([tweet.text.encode('utf-8')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tweets = []\n",
    "with open('tweets.csv') as csvfile:\n",
    "    reader = csv.reader(csvfile)\n",
    "    for row in reader:\n",
    "        tweets.append(row[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.8, stop_words='english', use_idf=True, vocabulary=vocabulary)\n",
    "tweet_vectors = vectorizer.fit_transform(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pred_multinomial = clf.predict(tweet_vectors)\n",
    "prob_multinomial = clf.predict_proba(tweet_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sentiment_tweets = []\n",
    "for index, prob in enumerate(prob_multinomial):\n",
    "    sentiment_tweets.append({\"Tweet\": tweets[index], \"p_neg\": prob[0], \"p_pos\": prob[1], \"target\": pred_multinomial[index]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('tweet_sentiments.json', 'w') as jsonfile:\n",
    "    json.dump(sentiment_tweets, jsonfile, sort_keys=True, indent=4, ensure_ascii=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Tweet': 'https://t.co/u5Fdl5h59i',\n",
       "  'p_neg': 0.38730505494998374,\n",
       "  'p_pos': 0.61269494505001587,\n",
       "  'target': 'pos'},\n",
       " {'Tweet': '@AboveUp I wondered about that one time Trump said something about international banks having caused problems and he was called anti-semitic',\n",
       "  'p_neg': 0.47827424887042869,\n",
       "  'p_pos': 0.52172575112957176,\n",
       "  'target': 'pos'},\n",
       " {'Tweet': \"RT @tyleroakley: imagine being so stubborn that even with this many DAILY offensive &amp; appalling trump stories, you still won't jump ship. y\\xe2\\x80\\xa6\",\n",
       "  'p_neg': 0.43561675645564979,\n",
       "  'p_pos': 0.56438324354435132,\n",
       "  'target': 'pos'},\n",
       " {'Tweet': \"RT @LRB: 'I will absolutely apologise sometime in the hopefully distant future if I am ever wrong\\xe2\\x80\\x99 &amp; other Trump sayings https://t.co/hIhfC\\xe2\\x80\\xa6\",\n",
       "  'p_neg': 0.45190655740122976,\n",
       "  'p_pos': 0.54809344259876891,\n",
       "  'target': 'pos'},\n",
       " {'Tweet': 'RT @CNN: Arianne Zucker on Trump: \"Be careful what you say, because the repercussions will come back\" https://t.co/UYpqI3w42L https://t.co/\\xe2\\x80\\xa6',\n",
       "  'p_neg': 0.63707845613466352,\n",
       "  'p_pos': 0.36292154386533737,\n",
       "  'target': 'neg'}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_tweets[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}