English_Twi_GRU.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "import io\n",
    "import time\n",
    "from importlib import reload\n",
    "from collections import defaultdict\n",
    "import sys\n",
    "reload(sys)\n",
    "# Just disables the warning, doesn't enable AVX/FMA\n",
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
    "from collections import defaultdict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install tensorflow\n",
    "#!pip install tensorflow-addons\n",
    "#!pip install --upgrade\n",
    "#!pip install tensorflow==2.2.0 --user\n",
    "#!pip install tfa-nightly  --user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.2.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available:  0\n",
      "2.2.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "# #### sequence to sequence with attention  model\n",
    "import tensorflow as tf\n",
    "#tf.random.Generator = None  # Patch for a bug\n",
    "import tensorflow_addons as tfa\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "from sklearn.model_selection import train_test_split\n",
    "import csv\n",
    "import string\n",
    "import re\n",
    "from pickle import dump\n",
    "from unicodedata import normalize\n",
    "from numpy import array\n",
    "import itertools\n",
    "from pickle import load\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "#from tensorflow.keras.utils.vis_utils import plot_model\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LSTM\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.layers import Embedding\n",
    "from pickle import load\n",
    "from numpy import array\n",
    "from numpy import argmax\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import load_model\n",
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "from sklearn.model_selection import train_test_split\n",
    "import unicodedata\n",
    "import numpy as np\n",
    "import os\n",
    "import io\n",
    "import time\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "# encoding=utf8\n",
    "from importlib import reload\n",
    "import sys\n",
    "reload(sys)\n",
    "# Just disables the warning, doesn't enable AVX/FMA\n",
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
    "\n",
    "print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\n",
    "print(tf.__version__)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\nboateng\\\\OneDrive - Nice Systems Ltd\\\\Documents\\\\MicrosoftAzureNotebooks\\\\NLP'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['“ Oo , Yehowa , Boa Me Babea Kumaa Yi Ma Onni Nokware ! ”',\n",
       " 'WƆWOO me too abusua a wonim adwinne di mu wɔ Alsace , France , wɔ 1930 mu .',\n",
       " 'Ná Papa taa pa twere n’agua mu kenkan asase ho nsɛm anaa ewim nneɛma ho nhoma bi anwummere anwummere .',\n",
       " 'Ná me kraman da Papa nan so , na na ɔka nsɛntitiriw a epue wɔ n’akenkan no mu kyerɛ Maame bere a ɔnwene abusua no nneɛma no .',\n",
       " 'Ná m’ani gye anwummere a ɛtete saa no ho kɛse !']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#read twi and english data\n",
    "with open('C:/Users/nboateng/OneDrive - Nice Systems Ltd/Documents/NaturalLanguageProcessing/GhanaNLP/jw300.en-tw.tw', 'r', encoding=\"utf8\") as f:\n",
    "     #Twi_data = f.readlines()\n",
    "      Twi_data= f.read().splitlines()\n",
    "\n",
    "Twi_data[0:5] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['“ Oh , Jehovah , Keep My Young Girl Faithful ! ”',\n",
       " 'I WAS born in 1930 in Alsace , France , into an artistic family .',\n",
       " 'During the evenings , Father , sitting in his lounge chair , would be reading some books about geography or astronomy .',\n",
       " 'My doggy would be sleeping by his feet , and Daddy would be sharing with Mum some highlights from his reading while she was knitting for her family .',\n",
       " 'How much I enjoyed those evenings !']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "english_data = [line.rstrip() for line in open('C:/Users/nboateng/OneDrive - Nice Systems Ltd/Documents/NaturalLanguageProcessing/GhanaNLP/jw300.en-tw.en', encoding=\"utf8\")]\n",
    "english_data[0:5]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<start> oh , jehovah , keep my young girl faithful ! <end>', '<start> i was born in in alsace , france , into an artistic family . <end>', '<start> during the evenings , father , sitting in his lounge chair , would be reading some books about geography or astronomy . <end>', '<start> my doggy would be sleeping by his feet , and daddy would be sharing with mum some highlights from his reading while she was knitting for her family . <end>', '<start> how much i enjoyed those evenings ! <end>']\n",
      "['<start> oo , yehowa , boa me babea kumaa yi ma onni nokware ! <end>', '<start> wɔwoo me too abusua a wonim adwinne di mu wɔ alsace , france , wɔ mu . <end>', '<start> na papa taa pa twere n agua mu kenkan asase ho nsɛm anaa ewim nneɛma ho nhoma bi anwummere anwummere . <end>', '<start> na me kraman da papa nan so , na na ɔka nsɛntitiriw a epue wɔ n akenkan no mu kyerɛ maame bere a ɔnwene abusua no nneɛma no . <end>', '<start> na m ani gye anwummere a ɛtete saa no ho kɛse ! <end>']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#number of examples to use for training\n",
    "num_examples=10000\n",
    "#num_examples = 606197\n",
    "\n",
    "def normalize_eng(s):\n",
    "    s = unicode_to_ascii(s)\n",
    "    s = re.sub(r'([!.?])', r' \\1', s)\n",
    "    s = re.sub(r'[^a-zA-Z.!?]+', r' ', s)\n",
    "    s = re.sub(r'\\s+', r' ', s)\n",
    "    return s\n",
    "\n",
    "def normalize_twi(s):\n",
    "    s = unicode_to_ascii(s)\n",
    "    s = re.sub(r'([!.?])', r' \\1', s)\n",
    "    s = re.sub(r'[^a-zA-Z.ƆɔɛƐ!?’]+', r' ', s)\n",
    "    s = re.sub(r'\\s+', r' ', s)\n",
    "    return s\n",
    "\n",
    "\n",
    "# Converts the unicode file to ascii\n",
    "def unicode_to_ascii(s):\n",
    "  return ''.join(c for c in unicodedata.normalize('NFD', s)\n",
    "      if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "\n",
    "def preprocess_sentence_english(w):\n",
    "  w = unicode_to_ascii(w.lower().strip())\n",
    "\n",
    "  # creating a space between a word and the punctuation following it\n",
    "  # eg: \"he is a boy.\" => \"he is a boy .\"\n",
    "  # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "  w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "  w = re.sub(r'[\" \"]+', \" \", w)\n",
    "\n",
    "  # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
    "  w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "  #w = re.sub(r'[^Ɔ-ɔɛƐ]+', r' ', w)\n",
    "  #strip() Parameters\n",
    "  #chars (optional) - a string specifying the set of characters to be removed.\n",
    "  #If the chars argument is not provided, all leading and trailing whitespaces are removed from the string.\n",
    "  w = w.rstrip().strip()\n",
    "\n",
    "  # adding a start and an end token to the sentence\n",
    "  # so that the model know when to start and stop predicting.\n",
    "  w = '<start> ' + w + ' <end>'\n",
    "  return w\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "  \n",
    "def preprocess_sentence_twi(w):\n",
    "  w = unicode_to_ascii(w.lower().strip())\n",
    "\n",
    "  # creating a space between a word and the punctuation following it\n",
    "  # eg: \"he is a boy.\" => \"he is a boy .\"\n",
    "  # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "  w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "  w = re.sub(r'[\" \"]+', \" \", w)\n",
    "\n",
    "  # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
    "  w = re.sub(r\"[^a-zA-ZɛƐɔƆ?.!,¿]+\", \" \", w)\n",
    "  #w = re.sub(r'[^Ɔ-ɔɛƐ]+', r' ', w)\n",
    "  #strip() Parameters\n",
    "  #chars (optional) - a string specifying the set of characters to be removed.\n",
    "  #If the chars argument is not provided, all leading and trailing whitespaces are removed from the string.\n",
    "  w = w.rstrip().strip()\n",
    "\n",
    "  # adding a start and an end token to the sentence\n",
    "  # so that the model know when to start and stop predicting.\n",
    "  w = '<start> ' + w + ' <end>'\n",
    "  return w\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "type(english_data)\n",
    "\n",
    "english_d  = list(map(preprocess_sentence_english,english_data))\n",
    "print(english_d[0:5])\n",
    "\n",
    "twi_d  = list(map(preprocess_sentence_twi,Twi_data))\n",
    "print(twi_d[0:5])\n",
    "      \n",
    "\n",
    "\n",
    "#all_data= (twi_d[0:30000]),(english_d[0:30000])\n",
    "#[0][0:5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<start> oh , jehovah , keep my young girl faithful ! <end>', '<start> i was born in in alsace , france , into an artistic family . <end>', '<start> during the evenings , father , sitting in his lounge chair , would be reading some books about geography or astronomy . <end>', '<start> my doggy would be sleeping by his feet , and daddy would be sharing with mum some highlights from his reading while she was knitting for her family . <end>', '<start> how much i enjoyed those evenings ! <end>']\n",
      "['<start> oo , yehowa , boa me babea kumaa yi ma onni nokware ! <end>', '<start> wɔwoo me too abusua a wonim adwinne di mu wɔ alsace , france , wɔ mu . <end>', '<start> na papa taa pa twere n agua mu kenkan asase ho nsɛm anaa ewim nneɛma ho nhoma bi anwummere anwummere . <end>', '<start> na me kraman da papa nan so , na na ɔka nsɛntitiriw a epue wɔ n akenkan no mu kyerɛ maame bere a ɔnwene abusua no nneɛma no . <end>', '<start> na m ani gye anwummere a ɛtete saa no ho kɛse ! <end>']\n"
     ]
    }
   ],
   "source": [
    "print(english_d[0:5])\n",
    "\n",
    "print(twi_d[0:5])\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def create_dataset_eng(path, num_examples):\n",
    "  lines = io.open(path, encoding='UTF-8').read().strip().split('\\n')\n",
    "\n",
    "  word_pairs = [[preprocess_sentence_english(w) for w in l.split('\\t')]  for l in lines[:num_examples]]\n",
    "\n",
    "  return zip(*word_pairs)\n",
    "\n",
    "\n",
    "#encoding = ‘utf-8-sig’ is added to overcome the issue when exporting ‘Non-English’ languages.\n",
    "def create_dataset_twi(path, num_examples):\n",
    "  lines = io.open(path, encoding='utf-8-sig').read().strip().split('\\n')\n",
    "\n",
    "  word_pairs = [[preprocess_sentence_twi(w) for w in l.split('\\t')]  for l in lines[:num_examples]]\n",
    "\n",
    "  return zip(*word_pairs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def max_length(tensor):\n",
    "  return max(len(t) for t in tensor)\n",
    "\n",
    "\n",
    "\n",
    "def tokenize(lang):\n",
    "  lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(\n",
    "      filters='',oov_token='OOV',lower=True)\n",
    "  lang_tokenizer.fit_on_texts(lang)\n",
    "\n",
    "  tensor = lang_tokenizer.texts_to_sequences(lang)\n",
    "\n",
    "  tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,\n",
    "                                                         padding='post')\n",
    "\n",
    "  return tensor, lang_tokenizer\n",
    "\n",
    "\n",
    "def load_data( num_examples=None):\n",
    "  # creating cleaned input, output pairs\n",
    "  targ_lang  = english_d[0:num_examples]\n",
    "  inp_lang =   twi_d[0:num_examples]\n",
    "\n",
    "  input_tensor, inp_lang_tokenizer = tokenize(inp_lang)\n",
    "  target_tensor, targ_lang_tokenizer = tokenize(targ_lang)\n",
    "\n",
    "  return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "113\n",
      "150\n",
      "8000 8000 2000 2000\n"
     ]
    }
   ],
   "source": [
    "# Try experimenting with the size of that dataset\n",
    "#num_examples = 100000\n",
    "#num_examples = 606197\n",
    "input_tensor, target_tensor, inp_lang, targ_lang = load_data( num_examples)\n",
    "\n",
    "# Calculate max_length of the target tensors\n",
    "max_length_targ, max_length_inp = max_length(target_tensor), max_length(input_tensor)\n",
    "\n",
    "\n",
    "\n",
    "print(max_length_targ)\n",
    "print(max_length_inp)\n",
    "\n",
    "\n",
    "# Creating training and validation sets using an 80-20 split\n",
    "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
    "\n",
    "# Show length\n",
    "print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Language; index to word mapping\n",
      "4 ----> <start>\n",
      "1080 ----> world\n",
      "1393 ----> health\n",
      "3679 ----> organization\n",
      "143 ----> yɛɛ\n",
      "154 ----> nhyiam\n",
      "16 ----> bi\n",
      "13 ----> wɔ\n",
      "1326 ----> london\n",
      "124 ----> esiane\n",
      "149 ----> mmɔden\n",
      "2 ----> a\n",
      "2051 ----> wɔrebɔ\n",
      "8 ----> sɛ\n",
      "3078 ----> wobesiw\n",
      "1070 ----> trɛw\n",
      "2 ----> a\n",
      "1915 ----> nsamanwaw\n",
      "2538 ----> retrɛw\n",
      "3 ----> no\n",
      "72 ----> ano\n",
      "75 ----> nti\n",
      "6 ----> .\n",
      "5 ----> <end>\n",
      "\n",
      "Target Language; index to word mapping\n",
      "2 ----> <start>\n",
      "3 ----> <end>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(64, 150), dtype=int32, numpy=\n",
       "array([[   4,    9,   11, ...,    0,    0,    0],\n",
       "       [   4,  197,    8, ...,    0,    0,    0],\n",
       "       [   4, 2859,  286, ...,    0,    0,    0],\n",
       "       ...,\n",
       "       [   4,   43,  678, ...,    0,    0,    0],\n",
       "       [   4,   25,   23, ...,    0,    0,    0],\n",
       "       [   4,  323,  232, ...,    0,    0,    0]])>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "def convert(lang, tensor):\n",
    "  for t in tensor:\n",
    "    if t!=0:\n",
    "      print (\"%d ----> %s\" % (t, lang.index_word[t]))\n",
    "\n",
    "print (\"Input Language; index to word mapping\")\n",
    "convert(inp_lang, input_tensor_train[0])\n",
    "print ()\n",
    "print (\"Target Language; index to word mapping\")\n",
    "convert(targ_lang, target_tensor_train[0])\n",
    "\n",
    "\n",
    "BUFFER_SIZE = len(input_tensor_train)\n",
    "BATCH_SIZE = 64\n",
    "#BATCH_SIZE = 32\n",
    "#for step in xrange(test_size / BATCH_SIZE):\n",
    "#for step in xrange(test_size // BATCH_SIZE):\n",
    "#In order to pass a int instead of a float to xrange and avoid a runtime exception.\n",
    "steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n",
    "embedding_dim = 256\n",
    "units = 1024\n",
    "vocab_inp_size = len(inp_lang.word_index)+1\n",
    "vocab_tar_size = len(targ_lang.word_index)+1\n",
    "\n",
    "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n",
    "\n",
    "#randomly pick out a batch\n",
    "example_input_batch, example_target_batch = next(iter(dataset))\n",
    "example_input_batch.shape, example_target_batch.shape\n",
    "\n",
    "\n",
    "example_input_batch\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder output shape: (batch size, sequence length, units) (64, 150, 1024)\n",
      "Encoder Hidden state shape: (batch size, units) (64, 1024)\n",
      "Attention result shape: (batch size, units) (64, 1024)\n",
      "Attention weights shape: (batch_size, sequence_length, 1) (64, 150, 1)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "class Encoder(tf.keras.Model):\n",
    "  def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "    super(Encoder, self).__init__()\n",
    "    self.batch_sz = batch_sz\n",
    "    self.enc_units = enc_units\n",
    "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "    self.gru = tf.keras.layers.GRU(self.enc_units,\n",
    "                                   return_sequences=True,\n",
    "                                   return_state=True,\n",
    "                                   recurrent_initializer='glorot_uniform')\n",
    "\n",
    "  def call(self, x, hidden):\n",
    "    x = self.embedding(x)\n",
    "    output, state = self.gru(x, initial_state = hidden)\n",
    "    return output, state\n",
    "\n",
    "  def initialize_hidden_state(self):\n",
    "    return tf.zeros((self.batch_sz, self.enc_units))\n",
    "\n",
    "\n",
    "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "# sample input\n",
    "sample_hidden = encoder.initialize_hidden_state()\n",
    "sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n",
    "print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
    "print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))\n",
    "\n",
    "\n",
    "\n",
    "class BahdanauAttention(tf.keras.layers.Layer):\n",
    "  def __init__(self, units):\n",
    "    super(BahdanauAttention, self).__init__()\n",
    "    self.W1 = tf.keras.layers.Dense(units)\n",
    "    self.W2 = tf.keras.layers.Dense(units)\n",
    "    self.V = tf.keras.layers.Dense(1)\n",
    "\n",
    "  def call(self, query, values):\n",
    "    # hidden shape == (batch_size, hidden size)\n",
    "    # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "    # we are doing this to perform addition to calculate the score\n",
    "    hidden_with_time_axis = tf.expand_dims(query, 1)\n",
    "\n",
    "    # score shape == (batch_size, max_length, 1)\n",
    "    # we get 1 at the last axis because we are applying score to self.V\n",
    "    # the shape of the tensor before applying self.V is (batch_size, max_length, units)\n",
    "    score = self.V(tf.nn.tanh(\n",
    "        self.W1(values) + self.W2(hidden_with_time_axis)))\n",
    "\n",
    "    # attention_weights shape == (batch_size, max_length, 1)\n",
    "    attention_weights = tf.nn.softmax(score, axis=1)\n",
    "\n",
    "    # context_vector shape after sum == (batch_size, hidden_size)\n",
    "    context_vector = attention_weights * values\n",
    "    context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "\n",
    "    return context_vector, attention_weights\n",
    "\n",
    "\n",
    "attention_layer = BahdanauAttention(10)\n",
    "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n",
    "\n",
    "print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n",
    "print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder output shape: (batch_size, vocab size) (64, 12664)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "class Decoder(tf.keras.Model):\n",
    "  def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "    super(Decoder, self).__init__()\n",
    "    self.batch_sz = batch_sz\n",
    "    self.dec_units = dec_units\n",
    "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "    self.gru = tf.keras.layers.GRU(self.dec_units,\n",
    "                                   return_sequences=True,\n",
    "                                   return_state=True,\n",
    "                                   recurrent_initializer='glorot_uniform')\n",
    "    self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "\n",
    "    # used for attention\n",
    "    self.attention = BahdanauAttention(self.dec_units)\n",
    "\n",
    "  def call(self, x, hidden, enc_output):\n",
    "    # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "    context_vector, attention_weights = self.attention(hidden, enc_output)\n",
    "\n",
    "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "    x = self.embedding(x)\n",
    "\n",
    "    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "\n",
    "    # passing the concatenated vector to the GRU\n",
    "    output, state = self.gru(x)\n",
    "\n",
    "    # output shape == (batch_size * 1, hidden_size)\n",
    "    output = tf.reshape(output, (-1, output.shape[2]))\n",
    "\n",
    "    # output shape == (batch_size, vocab)\n",
    "    x = self.fc(output)\n",
    "\n",
    "    return x, state, attention_weights\n",
    "\n",
    "\n",
    "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "sample_decoder_output, _, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)),\n",
    "                                      sample_hidden, sample_output)\n",
    "\n",
    "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))\n",
    "\n",
    "\n",
    "# ## Define the optimizer and the loss function\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam()\n",
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "    from_logits=True, reduction='none')\n",
    "\n",
    "def loss_function(real, pred):\n",
    "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "  loss_ = loss_object(real, pred)\n",
    "\n",
    "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "  loss_ *= mask\n",
    "\n",
    "  return tf.reduce_mean(loss_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# ## Checkpoints (Object-based saving)\n",
    "#Save checkpoints during training\n",
    "#You can use a trained model without having to retrain it, or pick-up training where you left off—in case the training process was #interrupted. \n",
    "#The tf.keras.callbacks.ModelCheckpoint callback allows to continually save the model both during and at the end of training.\n",
    "\n",
    "#Checkpoint callback usage\n",
    "#Create a tf.keras.callbacks.ModelCheckpoint callback that saves weights only during training:\n",
    "#https://www.tensorflow.org/tutorials/keras/save_and_load\n",
    "\n",
    "checkpoint_dir = './training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                 encoder=encoder,\n",
    "                                 decoder=decoder)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(inp, targ, enc_hidden):\n",
    "  loss = 0\n",
    "\n",
    "  with tf.GradientTape() as tape:\n",
    "    enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
    "\n",
    "    dec_hidden = enc_hidden\n",
    "\n",
    "    dec_input = tf.expand_dims([targ_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
    "\n",
    "    # Teacher forcing - feeding the target as the next input\n",
    "    for t in range(1, targ.shape[1]):\n",
    "      # passing enc_output to the decoder\n",
    "      predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
    "\n",
    "      loss += loss_function(targ[:, t], predictions)\n",
    "\n",
    "      # using teacher forcing\n",
    "      dec_input = tf.expand_dims(targ[:, t], 1)\n",
    "\n",
    "  batch_loss = (loss / int(targ.shape[1]))\n",
    "\n",
    "  variables = encoder.trainable_variables + decoder.trainable_variables\n",
    "\n",
    "  gradients = tape.gradient(loss, variables)\n",
    "\n",
    "  optimizer.apply_gradients(zip(gradients, variables))\n",
    "\n",
    "  return batch_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "EPOCHS = 20\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "  start = time.time()\n",
    "\n",
    "  enc_hidden = encoder.initialize_hidden_state()\n",
    "  total_loss = 0\n",
    "\n",
    "  for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n",
    "    batch_loss = train_step(inp, targ, enc_hidden)\n",
    "    total_loss += batch_loss\n",
    "\n",
    "    if batch % 100 == 0:\n",
    "      print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                                   batch,\n",
    "                                                   batch_loss.numpy()))\n",
    "  # saving (checkpoint) the model every 2 epochs\n",
    "  if (epoch + 1) % 2 == 0:\n",
    "    checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "\n",
    "  print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                      total_loss / steps_per_epoch))\n",
    "  print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def evaluate(sentence):\n",
    "  attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
    "\n",
    "  sentence = preprocess_sentence_twi(sentence)\n",
    "\n",
    "  inputs = [inp_lang.word_index[i] for i in sentence.split(' ')]\n",
    "  inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n",
    "                                                         maxlen=max_length_inp,\n",
    "                                                         padding='post')\n",
    "  inputs = tf.convert_to_tensor(inputs)\n",
    "\n",
    "  result = ''\n",
    "\n",
    "  hidden = [tf.zeros((1, units))]\n",
    "  enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "\n",
    "  dec_hidden = enc_hidden\n",
    "  dec_input = tf.expand_dims([targ_lang.word_index['<start>']], 0)\n",
    "\n",
    "  for t in range(max_length_targ):\n",
    "    predictions, dec_hidden, attention_weights = decoder(dec_input,\n",
    "                                                         dec_hidden,\n",
    "                                                         enc_out)\n",
    "\n",
    "    # storing the attention weights to plot later on\n",
    "    attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "    attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "    predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "    result += targ_lang.index_word[predicted_id] + ' '\n",
    "\n",
    "    if targ_lang.index_word[predicted_id] == '<end>':\n",
    "      return result, sentence, attention_plot\n",
    "\n",
    "    # the predicted ID is fed back into the model\n",
    "    dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "  return result, sentence, attention_plot\n",
    "\n",
    "\n",
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "  fig = plt.figure(figsize=(10,10))\n",
    "  ax = fig.add_subplot(1, 1, 1)\n",
    "  ax.matshow(attention, cmap='viridis')\n",
    "\n",
    "  fontdict = {'fontsize': 14}\n",
    "\n",
    "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "\n",
    "  ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "  ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "  plt.show()\n",
    "\n",
    "\n",
    "def translate(sentence):\n",
    "  result, sentence, attention_plot = evaluate(sentence)\n",
    "\n",
    "  print('Input: %s' % (sentence))\n",
    "  print('Predicted translation: {}'.format(result))\n",
    "\n",
    "  attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "  plot_attention(attention_plot, sentence.split(' '), result.split(' '))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Shapes (12664, 256) and (12663, 256) are incompatible",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-21-d8a857f16ba1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# restoring the latest checkpoint in checkpoint_dir\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mcheckpoint\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlatest_checkpoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcheckpoint_dir\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;31m# translate a twi sentence to english\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m#translate(u'kristo nkutoo ne yɛn gyefo .')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\training\\tracking\\util.py\u001b[0m in \u001b[0;36mrestore\u001b[1;34m(self, save_path)\u001b[0m\n\u001b[0;32m   2007\u001b[0m           \u001b[1;33m(\u001b[0m\u001b[0moften\u001b[0m \u001b[0mat\u001b[0m \u001b[0mprogram\u001b[0m \u001b[0mshutdown\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2008\u001b[0m     \"\"\"\n\u001b[1;32m-> 2009\u001b[1;33m     \u001b[0mstatus\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_saver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msave_path\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msave_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2010\u001b[0m     \u001b[1;31m# Create the save counter now so it gets initialized with other variables\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2011\u001b[0m     \u001b[1;31m# when graph building. Creating it earlier would lead to errors when using,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\training\\tracking\\util.py\u001b[0m in \u001b[0;36mrestore\u001b[1;34m(self, save_path)\u001b[0m\n\u001b[0;32m   1302\u001b[0m         graph_view=self._graph_view)\n\u001b[0;32m   1303\u001b[0m     base.CheckpointPosition(\n\u001b[1;32m-> 1304\u001b[1;33m         checkpoint=checkpoint, proto_id=0).restore(self._graph_view.root)\n\u001b[0m\u001b[0;32m   1305\u001b[0m     load_status = CheckpointLoadStatus(\n\u001b[0;32m   1306\u001b[0m         \u001b[0mcheckpoint\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\training\\tracking\\base.py\u001b[0m in \u001b[0;36mrestore\u001b[1;34m(self, trackable)\u001b[0m\n\u001b[0;32m    207\u001b[0m         \u001b[1;31m# This object's correspondence with a checkpointed object is new, so\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    208\u001b[0m         \u001b[1;31m# process deferred restorations for it and its dependencies.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 209\u001b[1;33m         \u001b[0mrestore_ops\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrackable\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_restore_from_checkpoint_position\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    210\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mrestore_ops\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    211\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_checkpoint\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnew_restore_ops\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrestore_ops\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\training\\tracking\\base.py\u001b[0m in \u001b[0;36m_restore_from_checkpoint_position\u001b[1;34m(self, checkpoint_position)\u001b[0m\n\u001b[0;32m    905\u001b[0m     restore_ops.extend(\n\u001b[0;32m    906\u001b[0m         current_position.checkpoint.restore_saveables(\n\u001b[1;32m--> 907\u001b[1;33m             tensor_saveables, python_saveables))\n\u001b[0m\u001b[0;32m    908\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mrestore_ops\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    909\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\training\\tracking\\util.py\u001b[0m in \u001b[0;36mrestore_saveables\u001b[1;34m(self, tensor_saveables, python_saveables)\u001b[0m\n\u001b[0;32m    287\u001b[0m              \"expecting %s\") % (tensor_saveables.keys(), validated_names))\n\u001b[0;32m    288\u001b[0m       new_restore_ops = functional_saver.MultiDeviceSaver(\n\u001b[1;32m--> 289\u001b[1;33m           validated_saveables).restore(self.save_path_tensor)\n\u001b[0m\u001b[0;32m    290\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mcontext\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecuting_eagerly\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    291\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrestore_op\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_restore_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\training\\saving\\functional_saver.py\u001b[0m in \u001b[0;36mrestore\u001b[1;34m(self, file_prefix)\u001b[0m\n\u001b[0;32m    279\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mdevice\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msaver\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_single_device_savers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    280\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 281\u001b[1;33m         \u001b[0mrestore_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msaver\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrestore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_prefix\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    282\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    283\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_after_restore_callbacks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\training\\saving\\functional_saver.py\u001b[0m in \u001b[0;36mrestore\u001b[1;34m(self, file_prefix)\u001b[0m\n\u001b[0;32m    101\u001b[0m                                           structured_restored_tensors):\n\u001b[0;32m    102\u001b[0m       restore_ops[saveable.name] = saveable.restore(\n\u001b[1;32m--> 103\u001b[1;33m           restored_tensors, restored_shapes=None)\n\u001b[0m\u001b[0;32m    104\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mrestore_ops\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    105\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\training\\saving\\saveable_object_util.py\u001b[0m in \u001b[0;36mrestore\u001b[1;34m(self, restored_tensors, restored_shapes)\u001b[0m\n\u001b[0;32m    114\u001b[0m       \u001b[0mrestored_tensor\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0midentity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrestored_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    115\u001b[0m       return resource_variable_ops.shape_safe_assign_variable_handle(\n\u001b[1;32m--> 116\u001b[1;33m           self.handle_op, self._var_shape, restored_tensor)\n\u001b[0m\u001b[0;32m    117\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    118\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\ops\\resource_variable_ops.py\u001b[0m in \u001b[0;36mshape_safe_assign_variable_handle\u001b[1;34m(handle, shape, value, name)\u001b[0m\n\u001b[0;32m    306\u001b[0m   \u001b[1;32mwith\u001b[0m \u001b[0m_handle_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    307\u001b[0m     \u001b[0mvalue_tensor\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mops\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconvert_to_tensor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 308\u001b[1;33m   \u001b[0mshape\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0massert_is_compatible_with\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue_tensor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    309\u001b[0m   return gen_resource_variable_ops.assign_variable_op(handle,\n\u001b[0;32m    310\u001b[0m                                                       \u001b[0mvalue_tensor\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python36\\site-packages\\tensorflow\\python\\framework\\tensor_shape.py\u001b[0m in \u001b[0;36massert_is_compatible_with\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m   1115\u001b[0m     \"\"\"\n\u001b[0;32m   1116\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1117\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Shapes %s and %s are incompatible\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1118\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1119\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mmost_specific_compatible_shape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Shapes (12664, 256) and (12663, 256) are incompatible"
     ]
    }
   ],
   "source": [
    "# restoring the latest checkpoint in checkpoint_dir\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))\n",
    "\n",
    "# translate a twi sentence to english\n",
    "#translate(u'kristo nkutoo ne yɛn gyefo .')\n",
    "translate(u'oh , jehovah , keep my young girl faithful !')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}