seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch import nn\n",
    "import matplotlib.pyplot as plt\n",
    "import spacy\n",
    "from torch.utils.data import Dataset, DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 482,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from spanish english dataset http://www.manythings.org/anki/spa-eng.zip\n",
    "df = pd.read_csv(\"spa-eng/spa.txt\", sep=\"\\t\", header=None)\n",
    "df.columns = [\"en\", \"es\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 483,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>en</th>\n",
       "      <th>es</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Go.</td>\n",
       "      <td>Ve.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Go.</td>\n",
       "      <td>Vete.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Go.</td>\n",
       "      <td>Vaya.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Go.</td>\n",
       "      <td>Váyase.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Hi.</td>\n",
       "      <td>Hola.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    en       es\n",
       "0  Go.      Ve.\n",
       "1  Go.    Vete.\n",
       "2  Go.    Vaya.\n",
       "3  Go.  Váyase.\n",
       "4  Hi.    Hola."
      ]
     },
     "execution_count": 483,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 484,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(47742, 4)"
      ]
     },
     "execution_count": 484,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[df.en.str.count(\" \").between(5, 7)]\n",
    "df['en_tokens'] = df.en.str.split()\n",
    "df['es_tokens'] = df.es.str.split()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "en_tokens = [w for w,n in\n",
    "    Counter([token for index, row in df.en_tokens.items() for token in row]).most_common(5000)]\n",
    "es_tokens = [w for w,n in\n",
    "    Counter([token for index, row in df.es_tokens.items() for token in row]).most_common(5000)]\n",
    "\n",
    "en_tokens.insert(0, '_bos_')\n",
    "en_tokens.insert(1, '_pad_')\n",
    "en_tokens.insert(2, '_eos_')\n",
    "en_tokens.insert(3, '_unk')\n",
    "\n",
    "es_tokens.insert(0, '_bos_')\n",
    "es_tokens.insert(1, '_pad_')\n",
    "es_tokens.insert(2, '_eos_')\n",
    "es_tokens.insert(3, '_unk')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_stoi = defaultdict(lambda: 3, {token:i for i,token in enumerate(en_tokens)})\n",
    "en_itos = en_tokens\n",
    "es_stoi = defaultdict(lambda: 3, {token:i for i,token in enumerate(es_tokens)})\n",
    "es_itos = es_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 487,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_padding(tokens, max_len):\n",
    "    return tokens + [\"_pad_\"]*(max_len - len(tokens))\n",
    "    \n",
    "def add_padding_to(rows):\n",
    "    max_len = rows.apply(lambda r: len(r)).max()\n",
    "    return rows.apply(lambda row: add_padding(row, max_len))\n",
    "\n",
    "df['es_tokens'] = add_padding_to(df.es_tokens)\n",
    "df['en_tokens'] = add_padding_to(df.en_tokens)\n",
    "\n",
    "en_phrases = [[en_stoi[token] for token in row] for index, row in df.en_tokens.items()]\n",
    "es_phrases = [[es_stoi[token] for token in row] for index, row in df.es_tokens.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 488,
   "metadata": {},
   "outputs": [],
   "source": [
    "e = nn.Embedding(len(en_tokens), 10, padding_idx=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Seq2SeqDataset(Dataset):\n",
    "    def __init__(self, x, y): self.x,self.y = x,y\n",
    "    def __getitem__(self, idx): return (\n",
    "        np.array(self.x[idx]),\n",
    "        np.array(self.y[idx])\n",
    "    )\n",
    "    def __len__(self): return len(self.x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 490,
   "metadata": {},
   "outputs": [],
   "source": [
    "ds = Seq2SeqDataset(en_phrases, es_phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 491,
   "metadata": {},
   "outputs": [],
   "source": [
    "dl = DataLoader(ds, batch_size=125, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 493,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* I did it on my own.\n",
      "* Lo hice yo sola.\n"
     ]
    }
   ],
   "source": [
    "x, y = ds[0]\n",
    "print(\"*\", ' '.join([en_itos[o] for o in x if o != 1]))\n",
    "print(\"*\", ' '.join([es_itos[o] for o in y if o != 1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 496,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n",
      "torch.Size([17, 125, 5004])\n",
      "torch.Size([125, 8]) torch.Size([125, 17])\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-496-683d6f77edb8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m     \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m     \u001b[0mo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5004\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.asdf/installs/python/3.6.3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-496-683d6f77edb8>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdec_input_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0mo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0memb_dec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdec_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m             \u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgru_dec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m             \u001b[0mo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mout_dec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m             \u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.asdf/installs/python/3.6.3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    475\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    478\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.asdf/installs/python/3.6.3/lib/python3.6/site-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    190\u001b[0m             \u001b[0mflat_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflat_weight\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    191\u001b[0m         )\n\u001b[0;32m--> 192\u001b[0;31m         \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mall_weights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    193\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mis_packed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    194\u001b[0m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPackedSequence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.asdf/installs/python/3.6.3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(input, *fargs, **fkwargs)\u001b[0m\n\u001b[1;32m    322\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdecorator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mfargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.asdf/installs/python/3.6.3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(input, weight, hidden, batch_sizes)\u001b[0m\n\u001b[1;32m    242\u001b[0m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    243\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 244\u001b[0;31m         \u001b[0mnexth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    245\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    246\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbatch_first\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mvariable_length\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.asdf/installs/python/3.6.3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(input, hidden, weight, batch_sizes)\u001b[0m\n\u001b[1;32m     85\u001b[0m                 \u001b[0ml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mnum_directions\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m                 \u001b[0mhy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m                 \u001b[0mnext_hidden\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m                 \u001b[0mall_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.asdf/installs/python/3.6.3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(input, hidden, weight, batch_sizes)\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0msteps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreverse\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msteps\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 116\u001b[0;31m             \u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    117\u001b[0m             \u001b[0;31m# hack to handle LSTM\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m             \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.asdf/installs/python/3.6.3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py\u001b[0m in \u001b[0;36mGRUCell\u001b[0;34m(input, hidden, w_ih, w_hh, b_ih, b_hh)\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0mgi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw_ih\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb_ih\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m     \u001b[0mgh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw_hh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb_hh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m     \u001b[0mi_r\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi_i\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi_n\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0mh_r\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh_i\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh_n\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgh\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.asdf/installs/python/3.6.3/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1022\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m         \u001b[0;31m# fused op is marginally faster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1024\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1026\u001b[0m     \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "class Seq2SeqModel(nn.Module):\n",
    "    def __init__(self, enc_emb_size, enc_emb_dim, dec_emb_size, dec_emb_dim,\n",
    "                enc_input_size, dec_input_size):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.dec_input_size = dec_input_size\n",
    "        self.emb_enc = nn.Embedding(enc_emb_size, enc_emb_dim)\n",
    "        self.gru_enc = nn.GRU(enc_emb_dim, 128, num_layers=2)\n",
    "        self.out_enc = nn.Linear(128, dec_emb_dim, bias=False)\n",
    "        \n",
    "        self.emb_dec = nn.Embedding(dec_emb_size, dec_emb_dim)\n",
    "        self.gru_dec = nn.GRU(dec_emb_dim, dec_emb_dim, num_layers=2)\n",
    "        self.out_dec = nn.Linear(dec_emb_dim, dec_emb_size)\n",
    "        \n",
    "    def forward(self, input):\n",
    "        input = input.permute(1, 0)\n",
    "        sq, bs = input.size()\n",
    "        o = self.emb_enc(input)\n",
    "        o, h = self.gru_enc(o)\n",
    "        h = self.out_enc(h)\n",
    "        \n",
    "        dec_in = torch.autograd.Variable(torch.zeros(bs).long())\n",
    "        res = []\n",
    "        for i in range(self.dec_input_size):\n",
    "            o = self.emb_dec(dec_in).unsqueeze(0)\n",
    "            o, h = self.gru_dec(o, h)\n",
    "            o = self.out_dec(o[0])\n",
    "            res.append(o)\n",
    "            dec_in = o.data.max(1)[1]\n",
    "            #if (dec_in == 1).all():\n",
    "            #    break\n",
    "        # import pdb; pdb.set_trace()\n",
    "        return torch.stack(res)\n",
    "\n",
    "model = Seq2SeqModel(len(en_itos), 300, len(es_itos), 300, 16, 17)\n",
    "\n",
    "optim = torch.optim.RMSprop(model.parameters(), lr=0.01)\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "for i,batch in enumerate(dl):\n",
    "    x, y = batch\n",
    "    print(x.size(), y.size())\n",
    "    optim.zero_grad()\n",
    "    o = model(x)\n",
    "    print(o.size())\n",
    "    loss = criterion(o.view(-1, 5004), y.permute(1, 0).contiguous().view(-1))\n",
    "    # if i % 100 == 0:\n",
    "    #    print(i, loss.item())\n",
    "    loss.backward()\n",
    "    optim.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 497,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* May I eat a little of it?\n",
      "* ¿Puedo comer un poco?\n",
      "* ¿Podría Ella Deberías Ellos ¿Quién Ella No ¿Cuándo No No Es No No Tom ¿Te El He Ellos Tom He He _unk Mi Mi Él Tú _unk No Él ¿Cuál Ellos Hablé No Estoy He No ¿Te No Yo No Ella Te He Ella Tengo _unk Ellos Tom No No Nuestro ¿Quieres No He Tom No ¿Dónde ¿Te Ellos Hablé El Hablé Tom No _unk Él Es ¿Te ¿Sabes ¿Dónde No Es ¿Cuándo Él ¿Sabes _unk _unk Hablé Hablé No No Tom Tom No Te Tom Tom El Mira Él No No Ella ¿Necesitas El Preferiría No No El Ella No Hablé Tom El Mary Esto No ¿Te _unk El ¿Hay Él No ¿Te No Él No Tom Siempre Él Tom Tengo Él El No\n",
      "\n",
      "* She returned the book to the library.\n",
      "* Ella regresó el libro a la biblioteca.\n",
      "* traigo estaba _unk _unk quedan estaba que _unk que dejes pocas _unk que _unk _unk _unk _unk _unk no _unk quedan pocas abuelo no _unk que que no _unk _unk que que estoy un comprar _unk mandar nadie comprar _unk _unk un _unk que _unk _unk no pocas mandar destino mandar dejes _unk un mandar _unk quedan hay _unk _unk dejes no quedan _unk pocas no tienes _unk dejes quedan dejes la tienes _unk _unk que pocas mandar comprar Tom no que _unk no no pocas _unk no _unk traigo _unk _unk niño _unk que que un estaba estaba que no dejes mandar _unk dejes que _unk dejes que un _unk traigo _unk mandar dejes _unk mandar _unk no _unk _unk mandar dejes\n",
      "\n",
      "* You should wash your hands before you eat.\n",
      "* Uno debería _unk las manos antes de comer.\n",
      "* un _unk que _unk un _unk que última encontré _unk _unk _unk que _unk _unk _unk quedan sabe de un hermano la _unk la que pocas de _unk de ibas de un que _unk _unk de _unk _unk un _unk que _unk flores que de que destino que un un _unk _unk _unk dormitorio _unk _unk que que _unk _unk quedan _unk _unk de _unk de que _unk escuchar que _unk _unk _unk _unk que que _unk de que _unk suelo. que _unk _unk _unk _unk es nadie que de _unk _unk _unk de que _unk _unk paso que _unk _unk _unk que un tienes hoy? _unk _unk que _unk un _unk de _unk _unk que\n",
      "\n",
      "* I'll save the money for the next time.\n",
      "* _unk el dinero para la próxima vez.\n",
      "* caminar para traiga la que _unk la _unk que flores descubierto suelo. _unk un un la la que que que _unk la que de _unk ir? descubierto llaves _unk la _unk _unk suelo. la después suelo. suelo. suelo. de consejo. de _unk _unk _unk Mary _unk la _unk que _unk descubierto un embajada _unk\n",
      "\n",
      "* Who would do such a stupid thing?\n",
      "* ¿Quién haría una _unk así?\n",
      "* de suelo. la que que de _unk\n",
      "\n",
      "* She laid the magazine on the table.\n",
      "* Dejó la revista sobre la mesa.\n",
      "* \n",
      "\n",
      "* It was really nice talking to you, Tom.\n",
      "* Fue muy agradable hablar _unk Tom.\n",
      "* \n",
      "\n",
      "* When was the last time you _unk _unk\n",
      "* ¿Cuándo fue la última vez que _unk _unk\n",
      "* \n",
      "\n",
      "* I wish she'd look me in the eyes.\n",
      "* Ojalá ella me _unk a los ojos.\n",
      "* \n",
      "\n",
      "* I am hanging up my _unk\n",
      "* Estoy _unk mis _unk\n",
      "* \n",
      "\n",
      "* It's not easy to quit smoking.\n",
      "* No es fácil dejar de fumar.\n",
      "* \n",
      "\n",
      "* I have many friends who are _unk _unk\n",
      "* Tengo muchos amigos que son _unk _unk\n",
      "* \n",
      "\n",
      "* I need to know if you'll be there.\n",
      "* Necesito saber si estarás ahí.\n",
      "* \n",
      "\n",
      "* Tom died on October _unk 2013.\n",
      "* Tomás murió el 20 de _unk de 2013.\n",
      "* \n",
      "\n",
      "* Excuse me. Do you speak English?\n",
      "* _unk ¿Usted habla inglés?\n",
      "* \n",
      "\n",
      "* The old man _unk over his own feet.\n",
      "* El anciano se _unk con sus propios pies.\n",
      "* \n",
      "\n",
      "* Look! There's a bird in that tree.\n",
      "* _unk Hay un pájaro en ese árbol.\n",
      "* \n",
      "\n",
      "* I'll let you stay one night.\n",
      "* Te dejaré quedarte una noche.\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "index 17 is out of bounds for dimension 1 with size 17",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-497-4a66f14152be>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"*\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0men_itos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mo\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"*\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mes_itos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mo\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"*\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mes_itos\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m!=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: index 17 is out of bounds for dimension 1 with size 17"
     ]
    }
   ],
   "source": [
    "x,y = next(iter(dl))\n",
    "probs = model(x)\n",
    "preds = probs.max(2)[1].permute(1, 0)\n",
    "\n",
    "for i in range(0,100):\n",
    "    print(\"*\", ' '.join([en_itos[o] for o in x.permute(1, 0)[:,i] if o != 1]))\n",
    "    print(\"*\", ' '.join([es_itos[o] for o in y.permute(1, 0)[:,i] if o != 1]))\n",
    "    print(\"*\", ' '.join([es_itos[o] for o in preds[:,i] if o!=1]))\n",
    "    print()\n",
    "preds.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}