Keras LSTM | Simple Char Generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2000193 80\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "data = open(\"datasets/got.txt\",\"r\").read()\n",
    "#chars = list(set(data))\n",
    "#pickle.dump(chars,open(\"chars2.pickle\",\"wb\"))\n",
    "chars = pickle.load(open(\"Pickles and model Weights/chars2.pickle\",\"rb\"))\n",
    "VOCAB_SIZE = len(chars)\n",
    "print(len(data),VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#enumerate\n",
    "char_to_ix = { ch:ix for ix,ch in enumerate(chars) }\n",
    "ix_to_char = { ix:ch for ix,ch in enumerate(chars) }\n",
    "SEQ_LENGTH = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Prepare Data\n",
    "X_data = np.zeros([len(data)-SEQ_LENGTH-1,SEQ_LENGTH,1])\n",
    "Y_data = np.zeros([len(data)-SEQ_LENGTH-1,VOCAB_SIZE])\n",
    "for i in range(0,len(data)-SEQ_LENGTH-1,1):\n",
    "    seq_in = data[i:i+SEQ_LENGTH]\n",
    "    out = data[i+SEQ_LENGTH]\n",
    "    X_data[i] = [ [char_to_ix[ch]] for ch in seq_in  ]\n",
    "    Y_data[i,char_to_ix[out]] = 1\n",
    "print(X_data.shape,Y_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#In previous one i have one-hot encoded input, so shape was (samples,seq_length,vocab_size)\n",
    "#lets normalize what input\n",
    "X_data_normalized = X_data/VOCAB_SIZE\n",
    "X_data_normalized.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "#lets build model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM, Activation, Dense, Dropout\n",
    "\n",
    "#Use CuDNNLSTM wirh GPUs for faster performance\n",
    "def createModel():\n",
    "    HIDDEN_DIM = 700\n",
    "    NUM_LAYERS = 2\n",
    "    DROPOUT_PROBABILITY = 0.3\n",
    "    model = Sequential()\n",
    "    for _ in range(NUM_LAYERS - 1):\n",
    "        model.add(LSTM(HIDDEN_DIM,input_shape=(None,1),return_sequences=True))\n",
    "        model.add(Dropout(DROPOUT_PROBABILITY))\n",
    "    model.add(LSTM(HIDDEN_DIM))\n",
    "    model.add(Dropout(DROPOUT_PROBABILITY))\n",
    "    model.add(Dense(VOCAB_SIZE,activation='softmax'))\n",
    "    model.compile(loss=\"categorical_crossentropy\",optimizer='adam',metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import ModelCheckpoint,EarlyStopping\n",
    "model_checkpoint = ModelCheckpoint('Pickles and model Weights/simplechar_gen_got2.hdf5',monitor='val_loss',verbose=2,mode='max')\n",
    "\n",
    "EARLY_STOPPING_PATIENCE = 10\n",
    "early_stopping = EarlyStopping(monitor='val_loss',patience=EARLY_STOPPING_PATIENCE)\n",
    "\n",
    "callback_list = [model_checkpoint]\n",
    "\n",
    "BATCH_SIZE = 512\n",
    "def trainModel(model):\n",
    "    EPOCHS=10\n",
    "    model.fit(X_data_normalized,Y_data,epochs=EPOCHS,batch_size=BATCH_SIZE,validation_split=0.2,callbacks=callback_list,\n",
    "             verbose=1)\n",
    "    model.save_weights('Pickles and model Weights/simplechar_gen_got2.hdf5')\n",
    "    \n",
    "def loadWeights(model):\n",
    "    model.load_weights('Pickles and model Weights/simplechar_gen_got2.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = createModel()\n",
    "loadWeights(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_text(model,length):\n",
    "    random_index =   np.random.randint(0,len(data)-SEQ_LENGTH-1)\n",
    "    X = np.array([ char_to_ix[ch] for ch in data[random_index:random_index+SEQ_LENGTH]]).reshape([SEQ_LENGTH])\n",
    "    Y = \"\".join([ix_to_char[ix] for ix in X ])+\"\\n\\nNow Generated Text\\n\\n\"\n",
    "    print(\"Inital Text :\\n {}\".format(Y))\n",
    "    output=\"\"\n",
    "    for i in range(length):\n",
    "        X_test = X.reshape([1,SEQ_LENGTH,1])/VOCAB_SIZE\n",
    "        newix = np.argmax(X_test)\n",
    "        output += ix_to_char[newix]\n",
    "        X = np.append(X,newix)[1:]\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inital Text :\n",
      " f a beautiful green lake. Deep Lake had been paid for by the\n",
      "\n",
      "Now Generated Text\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'tUJ‘,w2ve0“blMzc:h7\\n{GCo}Bm3EjdF”nQPtUJ‘,w2ve0“blMz“blMzz”nQPtUJ‘,w2ve0“blMzzzzzzzzzzzzzzzzzzmWWkp )(u-.Hxq1?arDZ—L5Ni6c:h7\\n{GCo}Bm3EjdF”nQPtUJ‘,w2ve0'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generate_text(model,150)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Looks like crap because it was trained for only 5 epochs\n",
    "1. Try One hot encode instead of normalizing input\n",
    "2. Try larger Network\n",
    "3. Try with different batch sizes."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}