text_prediction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "text_prediction.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "4iFJ77_MAG1x",
        "colab_type": "code",
        "outputId": "e5d30662-1a28-4c75-ac42-93c119bf3878",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# https://www.tensorflow.org/tutorials/text/text_generation\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import numpy as np\n",
        "import os\n",
        "import time\n",
        "\n",
        "path_to_file = tf.keras.utils.get_file('shakespeare.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')\n",
        "\n",
        "# Read, then decode for py2 compat.\n",
        "text = open(path_to_file, 'rb').read().decode(encoding='utf-8')\n",
        "# length of text is the number of characters in it\n",
        "print ('Length of text: {} characters'.format(len(text)))\n",
        "\n",
        "# Take a look at the first 250 characters in text\n",
        "print(text[:250])\n",
        "\n",
        "# The unique characters in the file\n",
        "vocab = sorted(set(text))\n",
        "print ('{} unique characters'.format(len(vocab)))\n",
        "\n",
        "# Creating a mapping from unique characters to indices\n",
        "char2idx = {u:i for i, u in enumerate(vocab)}\n",
        "idx2char = np.array(vocab)\n",
        "\n",
        "text_as_int = np.array([char2idx[c] for c in text])\n",
        "\n",
        "print('{')\n",
        "for char,_ in zip(char2idx, range(20)):\n",
        "    print('  {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n",
        "print('  ...\\n}')\n",
        "\n",
        "# Show how the first 13 characters from the text are mapped to integers\n",
        "print ('{} ---- characters mapped to int ---- > {}'.format(repr(text[:13]), text_as_int[:13]))\n",
        "\n",
        "# The maximum length sentence we want for a single input in characters\n",
        "seq_length = 100\n",
        "examples_per_epoch = len(text)//(seq_length+1)\n",
        "\n",
        "# Create training examples / targets\n",
        "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n",
        "\n",
        "for i in char_dataset.take(5):\n",
        "  print(idx2char[i.numpy()])\n",
        "\n",
        "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n",
        "\n",
        "for item in sequences.take(5):\n",
        "  print(repr(''.join(idx2char[item.numpy()])))\n",
        "\n",
        "def split_input_target(chunk):\n",
        "    input_text = chunk[:-1]\n",
        "    target_text = chunk[1:]\n",
        "    return input_text, target_text\n",
        "\n",
        "dataset = sequences.map(split_input_target)\n",
        "\n",
        "for input_example, target_example in  dataset.take(1):\n",
        "  print ('Input data: ', repr(''.join(idx2char[input_example.numpy()])))\n",
        "  print ('Target data:', repr(''.join(idx2char[target_example.numpy()])))\n",
        "\n",
        "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n",
        "    print(\"Step {:4d}\".format(i))\n",
        "    print(\"  input: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n",
        "    print(\"  expected output: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))\n",
        "\n",
        "# Batch size\n",
        "BATCH_SIZE = 64\n",
        "\n",
        "# Buffer size to shuffle the dataset\n",
        "# (TF data is designed to work with possibly infinite sequences,\n",
        "# so it doesn't attempt to shuffle the entire sequence in memory. Instead,\n",
        "# it maintains a buffer in which it shuffles elements).\n",
        "BUFFER_SIZE = 10000\n",
        "\n",
        "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n",
        "\n",
        "dataset\n",
        "\n",
        "# Length of the vocabulary in chars\n",
        "vocab_size = len(vocab)\n",
        "\n",
        "# The embedding dimension\n",
        "embedding_dim = 256\n",
        "\n",
        "# Number of RNN units\n",
        "rnn_units = 1024\n",
        "\n",
        "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n",
        "  model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Embedding(vocab_size, embedding_dim,\n",
        "                              batch_input_shape=[batch_size, None]),\n",
        "    tf.keras.layers.GRU(rnn_units,\n",
        "                        return_sequences=True,\n",
        "                        stateful=True,\n",
        "                        recurrent_initializer='glorot_uniform'),\n",
        "    tf.keras.layers.Dense(vocab_size)\n",
        "  ])\n",
        "  return model\n",
        "\n",
        "model = build_model(\n",
        "  vocab_size = len(vocab),\n",
        "  embedding_dim=embedding_dim,\n",
        "  rnn_units=rnn_units,\n",
        "  batch_size=BATCH_SIZE)\n",
        "\n",
        "for input_example_batch, target_example_batch in dataset.take(1):\n",
        "  example_batch_predictions = model(input_example_batch)\n",
        "  print(example_batch_predictions.shape, \"# (batch_size, sequence_length, vocab_size)\")\n",
        "\n",
        "\n",
        "sampled_indices = tf.random.categorical(example_batch_predictions[0], num_samples=1)\n",
        "sampled_indices = tf.squeeze(sampled_indices,axis=-1).numpy()\n",
        "\n",
        "sampled_indices\n",
        "\n",
        "print(\"Input: \\n\", repr(\"\".join(idx2char[input_example_batch[0]])))\n",
        "print()\n",
        "print(\"Next Char Predictions: \\n\", repr(\"\".join(idx2char[sampled_indices ])))\n",
        "\n",
        "def loss(labels, logits):\n",
        "  return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n",
        "\n",
        "example_batch_loss  = loss(target_example_batch, example_batch_predictions)\n",
        "print(\"Prediction shape: \", example_batch_predictions.shape, \" # (batch_size, sequence_length, vocab_size)\")\n",
        "print(\"scalar_loss:      \", example_batch_loss.numpy().mean())\n",
        "\n",
        "model.compile(optimizer='adam', loss=loss)\n",
        "\n",
        "# Directory where the checkpoints will be saved\n",
        "checkpoint_dir = './training_checkpoints'\n",
        "# Name of the checkpoint files\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")\n",
        "\n",
        "checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_prefix,\n",
        "    save_weights_only=True)\n",
        "\n",
        "EPOCHS=10\n",
        "\n",
        "history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])\n",
        "\n",
        "tf.train.latest_checkpoint(checkpoint_dir)\n",
        "\n",
        "model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)\n",
        "\n",
        "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n",
        "\n",
        "model.build(tf.TensorShape([1, None]))\n",
        "\n",
        "model.summary()\n",
        "\n",
        "def generate_text(model, start_string):\n",
        "  # Evaluation step (generating text using the learned model)\n",
        "\n",
        "  # Number of characters to generate\n",
        "  num_generate = 1000\n",
        "\n",
        "  # Converting our start string to numbers (vectorizing)\n",
        "  input_eval = [char2idx[s] for s in start_string]\n",
        "  input_eval = tf.expand_dims(input_eval, 0)\n",
        "\n",
        "  # Empty string to store our results\n",
        "  text_generated = []\n",
        "\n",
        "  # Low temperatures results in more predictable text.\n",
        "  # Higher temperatures results in more surprising text.\n",
        "  # Experiment to find the best setting.\n",
        "  temperature = 1.0\n",
        "\n",
        "  # Here batch size == 1\n",
        "  model.reset_states()\n",
        "  for i in range(num_generate):\n",
        "      predictions = model(input_eval)\n",
        "      # remove the batch dimension\n",
        "      predictions = tf.squeeze(predictions, 0)\n",
        "\n",
        "      # using a categorical distribution to predict the character returned by the model\n",
        "      predictions = predictions / temperature\n",
        "      predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n",
        "\n",
        "      # We pass the predicted character as the next input to the model\n",
        "      # along with the previous hidden state\n",
        "      input_eval = tf.expand_dims([predicted_id], 0)\n",
        "\n",
        "      text_generated.append(idx2char[predicted_id])\n",
        "\n",
        "  return (start_string + ''.join(text_generated))\n",
        "\n",
        "print(generate_text(model, start_string=u\"ROMEO: \"))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
            "1122304/1115394 [==============================] - 0s 0us/step\n",
            "Length of text: 1115394 characters\n",
            "First Citizen:\n",
            "Before we proceed any further, hear me speak.\n",
            "\n",
            "All:\n",
            "Speak, speak.\n",
            "\n",
            "First Citizen:\n",
            "You are all resolved rather to die than to famish?\n",
            "\n",
            "All:\n",
            "Resolved. resolved.\n",
            "\n",
            "First Citizen:\n",
            "First, you know Caius Marcius is chief enemy to the people.\n",
            "\n",
            "65 unique characters\n",
            "{\n",
            "  '\\n':   0,\n",
            "  ' ' :   1,\n",
            "  '!' :   2,\n",
            "  '$' :   3,\n",
            "  '&' :   4,\n",
            "  \"'\" :   5,\n",
            "  ',' :   6,\n",
            "  '-' :   7,\n",
            "  '.' :   8,\n",
            "  '3' :   9,\n",
            "  ':' :  10,\n",
            "  ';' :  11,\n",
            "  '?' :  12,\n",
            "  'A' :  13,\n",
            "  'B' :  14,\n",
            "  'C' :  15,\n",
            "  'D' :  16,\n",
            "  'E' :  17,\n",
            "  'F' :  18,\n",
            "  'G' :  19,\n",
            "  ...\n",
            "}\n",
            "'First Citizen' ---- characters mapped to int ---- > [18 47 56 57 58  1 15 47 58 47 64 43 52]\n",
            "F\n",
            "i\n",
            "r\n",
            "s\n",
            "t\n",
            "'First Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYou '\n",
            "'are all resolved rather to die than to famish?\\n\\nAll:\\nResolved. resolved.\\n\\nFirst Citizen:\\nFirst, you k'\n",
            "\"now Caius Marcius is chief enemy to the people.\\n\\nAll:\\nWe know't, we know't.\\n\\nFirst Citizen:\\nLet us ki\"\n",
            "\"ll him, and we'll have corn at our own price.\\nIs't a verdict?\\n\\nAll:\\nNo more talking on't; let it be d\"\n",
            "'one: away, away!\\n\\nSecond Citizen:\\nOne word, good citizens.\\n\\nFirst Citizen:\\nWe are accounted poor citi'\n",
            "Input data:  'First Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYou'\n",
            "Target data: 'irst Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYou '\n",
            "Step    0\n",
            "  input: 18 ('F')\n",
            "  expected output: 47 ('i')\n",
            "Step    1\n",
            "  input: 47 ('i')\n",
            "  expected output: 56 ('r')\n",
            "Step    2\n",
            "  input: 56 ('r')\n",
            "  expected output: 57 ('s')\n",
            "Step    3\n",
            "  input: 57 ('s')\n",
            "  expected output: 58 ('t')\n",
            "Step    4\n",
            "  input: 58 ('t')\n",
            "  expected output: 1 (' ')\n",
            "(64, 100, 65) # (batch_size, sequence_length, vocab_size)\n",
            "Input: \n",
            " 'have infected mine.\\n\\nLADY ANNE:\\nWould they were basilisks, to strike thee dead!\\n\\nGLOUCESTER:\\nI would'\n",
            "\n",
            "Next Char Predictions: \n",
            " \".SJqMg\\n!VhRtG-i\\nzc Y Ul$FHR\\nJWiIdllEXmIIj.cVYxhyXMNkstIwC?MKP\\nejRJOmhA.tD:uzv,3nnUvBB\\n!af''sh..R&UZQ\"\n",
            "Prediction shape:  (64, 100, 65)  # (batch_size, sequence_length, vocab_size)\n",
            "scalar_loss:       4.1736217\n",
            "Epoch 1/10\n",
            "172/172 [==============================] - 12s 68ms/step - loss: 2.6338\n",
            "Epoch 2/10\n",
            "172/172 [==============================] - 12s 68ms/step - loss: 1.9401\n",
            "Epoch 3/10\n",
            "172/172 [==============================] - 12s 68ms/step - loss: 1.6740\n",
            "Epoch 4/10\n",
            "172/172 [==============================] - 12s 68ms/step - loss: 1.5301\n",
            "Epoch 5/10\n",
            "172/172 [==============================] - 12s 68ms/step - loss: 1.4450\n",
            "Epoch 6/10\n",
            "172/172 [==============================] - 12s 68ms/step - loss: 1.3867\n",
            "Epoch 7/10\n",
            "172/172 [==============================] - 12s 68ms/step - loss: 1.3415\n",
            "Epoch 8/10\n",
            "172/172 [==============================] - 12s 70ms/step - loss: 1.3031\n",
            "Epoch 9/10\n",
            "172/172 [==============================] - 12s 70ms/step - loss: 1.2679\n",
            "Epoch 10/10\n",
            "172/172 [==============================] - 12s 69ms/step - loss: 1.2348\n",
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (1, None, 256)            16640     \n",
            "_________________________________________________________________\n",
            "gru_1 (GRU)                  (1, None, 1024)           3938304   \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (1, None, 65)             66625     \n",
            "=================================================================\n",
            "Total params: 4,021,569\n",
            "Trainable params: 4,021,569\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "ROMEO: I pardon me, uncle, are night;\n",
            "For sleep of Tranio houses forge.\n",
            "\n",
            "HENRY BOLINGBROKE:\n",
            "I content their tongue? Wherefore?\n",
            "\n",
            "CAPULET:\n",
            "Dead brave houses that is come.\n",
            "\n",
            "BUCKINGHAM:\n",
            "Well, be hurthen you are to sour to you,\n",
            "some in the field:\n",
            "Could he would not have my heart displacers,\n",
            "And they alwome distemperate, what hour\n",
            "Braved i' the rest; this grace\n",
            "Dost think to woe:\n",
            "How not, he had them; whom all the realm my son Coriolanus.\n",
            "\n",
            "First Keeper:\n",
            "And, know by the success\n",
            "And nothing hate thy husband.\n",
            "\n",
            "KING RICHARD III:\n",
            "What, have he shall demage them, who can do\n",
            "With comfortable death.\n",
            "A mighty converity,\n",
            "In soleir and motions triump.\n",
            "\n",
            "GLOUCESTER:\n",
            "Liquor I'll villain had me with thee\n",
            "Too late be setrected to him to his eye!\n",
            "Thou determine eyes some other flate\n",
            "Ty and bring forthe that were such frail, more\n",
            "pride and traitors, to the sun, ordere she would here\n",
            "Than answer than here colours contemption.\n",
            "Glad at the horngrance of alse deeds\n",
            "As 'twere not forbid!\n",
            "The next sweet country of this f\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}