Sentiment Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Sentiment Analysis with TensorFlow and Python </b>\n",
    "<br/>\n",
    "In this notebook, we create a simple NN using TensorFlow and Python and attempt to do the sentiment analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#imports\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "import numpy as np\n",
    "import random\n",
    "import pickle\n",
    "from collections import Counter\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import tensorflow as tf\n",
    "#from tensorflow.examples.tutorials.mnist import input_data\n",
    "import pickle\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#config files and data files\n",
    "posFile = \"data\\sentiment\\pos.txt\"\n",
    "negFile = \"data\\sentiment\\\\neg.txt\"\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "hm_lines = 100000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#build the lexicon\n",
    "def create_lexicon(pos,neg):\n",
    "    lexicon = []\n",
    "    with open(pos,'rb') as f:\n",
    "        contents = f.readlines()\n",
    "    f.close()\n",
    "    \n",
    "    for l in contents[:hm_lines]:\n",
    "        l = l.decode(\"utf8\")\n",
    "        all_words = word_tokenize(l)\n",
    "        lexicon += list(all_words)\n",
    "    \n",
    "    with open(neg,'rb') as f:\n",
    "        contents = f.readlines()\n",
    "    f.close()\n",
    "    \n",
    "    for l in contents[:hm_lines]:\n",
    "        l = l.decode(\"utf8\")\n",
    "        all_words = word_tokenize(l)\n",
    "        lexicon += list(all_words)\n",
    "    \n",
    "    lexicon = [lemmatizer.lemmatize(i) for i in lexicon]\n",
    "    \n",
    "    w_counts = Counter(lexicon) #provides index by couting the unique elements in the list\n",
    "    \n",
    "    l2 = []\n",
    "    #print(type(lexicon), type(w_counts))\n",
    "    for w in w_counts:\n",
    "        #print(w_counts[w]) #w_counts[w] gives the index of the word\n",
    "        if 1000 > w_counts[w] > 50: #only consider the words that occur >50 times and less than 1000 times\n",
    "            l2.append(w.lower())\n",
    "    #print(len(l2))\n",
    "    return l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''\n",
    "This will iterate through the \"sample\" file that we choose. In our case, this is the pos.txt or neg.txt.\n",
    "We also pass the lexicon, and the classification of the file coming through.\n",
    "From here, it tokenizes the sample file by word, then lemmatizes the words.\n",
    "Now, we begin with a numpy.zeros array that is the length of the lexicon.\n",
    "'''\n",
    "def sample_handling(sample, lexicon, classification):\n",
    "    featureset = []\n",
    "    with open(sample, 'r') as f:\n",
    "        contents = f.readlines()\n",
    "    f.close()\n",
    "    \n",
    "    for l in contents[:hm_lines]:\n",
    "        current_words = word_tokenize(l.lower())\n",
    "        current_words = [lemmatizer.lemmatize(i) for i in current_words]\n",
    "        features = np.zeros(len(lexicon))\n",
    "        for word in current_words:\n",
    "            if word.lower() in lexicon:\n",
    "                index_value = lexicon.index(word.lower())\n",
    "                features[index_value] += 1 # just like one-hot encoding\n",
    "        features = list(features)\n",
    "        featureset.append([features, classification])\n",
    "    return featureset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following method creates feature set and labels from the positive and negative text files. We use 0.1 (ie.e 10%) of the whole data as the test data. We use [1,0] as the label for positive and [0,1] as the label for negative."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_feature_sets_and_labels(pos, neg, test_size=0.1):\n",
    "    lexicon = create_lexicon(pos, neg)\n",
    "    features = []\n",
    "    features += sample_handling(posFile, lexicon, [1, 0])\n",
    "    features += sample_handling(negFile, lexicon, [0, 1])\n",
    "    random.shuffle(features)\n",
    "    features = np.array(features)\n",
    "\n",
    "    testing_size = int(test_size * len(features))\n",
    "    #print(type(features))\n",
    "    #leave the last \"testing_size\" elements as the test data and take the rest as the train data\n",
    "    train_x = list(features[:, 0][:-testing_size])\n",
    "    #leave the last \"testing_size\" elements as the test label and take the rest as the train label\n",
    "    train_y = list(features[:, 1][:-testing_size])\n",
    "    #take the last \"testing_size\" elements as the test data and leave the rest as the train data\n",
    "    test_x = list(features[:, 0][-testing_size:])\n",
    "    #take the last \"testing_size\" elements as the test label and leave the rest as the train label\n",
    "    test_y = list(features[:, 1][-testing_size:])\n",
    "    \n",
    "    return train_x, train_y, test_x, test_y\n",
    "\n",
    "#create_lexicon(posFile, negFile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we create a NN model using the data. We have three hidden layers. The first hidden layer uses the input data, the weight for this hidden layer and the bias for this hidden layer in its RELU. The second hidden layer uses the output of first hidden layer as its input. The same is repeated for the third hidden layer. The output layer uses the output of the third hidden layer as its input. The output is simply the matrix multiplication of the input to the output layer, the weights of the nodes in the output layer and the associated bias. We can also use more sophisticated functions in this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#lets gete the train and test data and their labels\n",
    "train_x,train_y,test_x,test_y = create_feature_sets_and_labels(posFile, negFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def neural_network_model(data):\n",
    "    \n",
    "    l1 = tf.add(tf.matmul(data, hidden_1_layer['weight']), hidden_1_layer['bias'])\n",
    "    #Computes rectified linear: max(l1, 0).\n",
    "    l1 = tf.nn.relu(l1)\n",
    "\n",
    "    l2 = tf.add(tf.matmul(l1, hidden_2_layer['weight']), hidden_2_layer['bias'])\n",
    "    #Computes rectified linear: max(l2, 0).\n",
    "    l2 = tf.nn.relu(l2)\n",
    "\n",
    "    l3 = tf.add(tf.matmul(l2, hidden_3_layer['weight']), hidden_3_layer['bias'])\n",
    "    #Computes rectified linear: max(l3, 0).\n",
    "    l3 = tf.nn.relu(l3)\n",
    "\n",
    "    output = tf.matmul(l3, output_layer['weight']) + output_layer['bias']\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we define a method to train the neural network. The data is processed in batches and we use 10 epochs to process the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_neural_network(x):\n",
    "    prediction = neural_network_model(x)\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=0.001).minimize(cost)\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(tf.initialize_all_variables())\n",
    "        writer = tf.summary.FileWriter(\"graph/sentanalysis\", graph=tf.get_default_graph())\n",
    "        for epoch in range(hm_epochs):\n",
    "            epoch_loss = 0\n",
    "            i = 0\n",
    "            while i < len(train_x):\n",
    "                start = i\n",
    "                end = i + batch_size\n",
    "                batch_x = np.array(train_x[start:end])\n",
    "                batch_y = np.array(train_y[start:end])\n",
    "\n",
    "                _, c = sess.run([optimizer, cost], feed_dict={x: batch_x,\n",
    "                                                              y: batch_y})\n",
    "                #lets see the dimension of the input-first hidden layer\n",
    "                #print(\"shape of input-1st hidden layer:\",type(x), type(tf.get_variable(\"hidden_1_layer\")['weight']))\n",
    "                #print(c)\n",
    "                epoch_loss += c\n",
    "                i += batch_size\n",
    "\n",
    "            print('Epoch', epoch + 1, 'completed out of', hm_epochs, 'loss:', epoch_loss)\n",
    "        #print(\"prediction was:\",prediction[:,0],\" y was:\",y)\n",
    "        correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))\n",
    "        #tf.Print(correct, [correct])\n",
    "        print('Accuracy:', accuracy.eval({x: test_x, y: test_y}))\n",
    "        #print(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we initialize some parameters. We use three hidden layers with each layer having 1500 nodes. We operate in batches of size 100 and we repeat the process for 10 epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#we use the sentiment prediction problem with the image classification network\n",
    "# if you want to pickle this data:\n",
    "#with open('/path/to/sentiment_set.pickle','wb') as f:\n",
    "#\tpickle.dump([train_x,train_y,test_x,test_y],f)\n",
    "\n",
    "n_nodes_hl1 = 1500\n",
    "n_nodes_hl2 = 1500\n",
    "n_nodes_hl3 = 1500\n",
    "\n",
    "n_classes = 2\n",
    "batch_size = 100\n",
    "hm_epochs = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For tensorflow, we need to define the elements so that they fit in the graph. We define every elements, i.e. the hidden layer, output layer as the dictionary where each item points to the resepective value or the placeholder that defines the value when the placeholder is executed. The x and y variables are just defined as placeholder because they will be initialized by the data and label for each batch within the loop.\n",
    "The weights of the hidden layer are tensors of size [inputnodes x nodes_in_current_layer] and are initialized as random and uniformly distributed values with mean =0 and variance =1 (default of tf.random_normal(....)). Every layer also defines some bias values which is also taken from the random normal distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder('float', name='x')\n",
    "y = tf.placeholder('float', name='y')\n",
    "\n",
    "hidden_1_layer = {'name':'hid_1',\n",
    "                    'f_fum': n_nodes_hl1,\n",
    "                  'weight': tf.Variable(tf.random_normal([len(train_x[0]), n_nodes_hl1])),\n",
    "                  'bias': tf.Variable(tf.random_normal([n_nodes_hl1]))}\n",
    "\n",
    "hidden_2_layer = {'name':'hid_2', 'f_fum': n_nodes_hl2,\n",
    "                  'weight': tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])),\n",
    "                  'bias': tf.Variable(tf.random_normal([n_nodes_hl2]))}\n",
    "\n",
    "hidden_3_layer = {'name':'hid_3','f_fum': n_nodes_hl3,\n",
    "                  'weight': tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])),\n",
    "                  'bias': tf.Variable(tf.random_normal([n_nodes_hl3]))}\n",
    "\n",
    "output_layer = {'name':'out_1','f_fum': None,\n",
    "                'weight': tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])),\n",
    "                'bias': tf.Variable(tf.random_normal([n_classes])), }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-8-28acf6b76763>:7: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "Epoch 1 completed out of 2 loss: 1155276.79004\n",
      "Epoch 2 completed out of 2 loss: 434481.717773\n",
      "Accuracy: 0.609756\n"
     ]
    }
   ],
   "source": [
    "#lets create the log of the tensor graph to visualize it in tensorborad\n",
    "writer = tf.summary.FileWriter(\"graph/sentanalysis\", graph=tf.get_default_graph())\n",
    "train_neural_network(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> References</b>\n",
    "<br/>\n",
    "1) https://pythonprogramming.net/using-our-own-data-tensorflow-deep-learning-tutorial/?completed=/tensorflow-neural-network-session-machine-learning-tutorial/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}