kw2sentence.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a559a305",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import re\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "import torch\n",
    "from datasets import load_dataset\n",
    "from transformers import (\n",
    "    TrainerCallback,\n",
    "    GPT2Config,\n",
    "    GPT2Tokenizer,\n",
    "    GPT2LMHeadModel,\n",
    "    AutoConfig,\n",
    "    AutoTokenizer,\n",
    "    AutoModelForCausalLM,\n",
    "    pipeline,\n",
    "    AdamW,\n",
    "    TrainingArguments,\n",
    "    Trainer,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "82b4830d",
   "metadata": {},
   "outputs": [],
   "source": [
    "start_from_checkpoint = False\n",
    "\n",
    "# Load pretrained tokenizer and model\n",
    "finetuned_model_name = 'movie-plot-generator'\n",
    "\n",
    "if start_from_checkpoint:\n",
    "    config=AutoConfig.from_pretrained(finetuned_model_name)\n",
    "    tokenizer = AutoTokenizer.from_pretrained(finetuned_model_name)\n",
    "    model = AutoModelForCausalLM.from_pretrained(finetuned_model_name, config=config)\n",
    "else:\n",
    "    model_name = 'gpt2' \n",
    "    config=AutoConfig.from_pretrained(model_name)\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_name, config=config)\n",
    "    model = AutoModelForCausalLM.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9e16326c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BOS> <action> Shrek in the swamp <SEP> He was in the  That the world of Shrek is falling apart; a mysterious girl named Aaliya's life is falling apart, as is her own. When she asks him what she has forgotten, he admits to her that she must come to a conclusion: that she is not alone. Soon after her awakening, Aaliya, a man on the verge of disaster, confronts him in the woods and he kills her with his sword. She is put down by a mob. When he wakes up, the girl screams. It is later revealed that this was the father of the family. With His wife having died, the father has now decided to go back to town, where he would find his daughter and make her return home to her parents. \n",
      "\n",
      "\n",
      "------------------------\n",
      " <BOS> <action> Shrek in the swamp <SEP> He was in the Riverboat Graveyard by his grandparents. The boat he was on, and the swamp he was on, still had mud walls and flooded water. He lived a great part of the night in his car. A friend of Mr. Marge's and his wife  was working at the bank and the friends were talking about their plans for a new house. The friend invited an older woman from the kitchen  for the meeting. When one of them went to dinner, he gave the girl a green card. They went to the kitchen and Mr. Marge came out of a hole through the wall to find them sitting beside each other. They were drinking. At first, he did not want to talk about his past, but he went back to talk to the man who was sitting next to them. He asked him what they had done when the car broke down. When he first came down, Mr\n",
      "\n",
      "\n",
      "------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "generator = pipeline(\"text-generation\", model=model, tokenizer=tokenizer, device=0)\n",
    "stories = generator(\"<BOS> <action> Shrek in the swamp <SEP> He was in the \", max_length=200, num_return_sequences=2)\n",
    "print(*[story['generated_text'] + \"\\n\\n\\n------------------------\\n\" for story in stories])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8d54c69",
   "metadata": {},
   "source": [
    "# Load dataset\n",
    "\n",
    "First, we load the dataset and split into train and validation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c379c5e7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-2fcf8d2135508f85\n",
      "Reusing dataset text (C:\\Users\\Anton\\.cache\\huggingface\\datasets\\text\\default-2fcf8d2135508f85\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53c583aec4cb4d2996b6149a6b89c3f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached split indices for dataset at C:\\Users\\Anton\\.cache\\huggingface\\datasets\\text\\default-2fcf8d2135508f85\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\\cache-1dc70cf5269acfd9.arrow and C:\\Users\\Anton\\.cache\\huggingface\\datasets\\text\\default-2fcf8d2135508f85\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\\cache-cd027351125f5409.arrow\n",
      "Loading cached processed dataset at C:\\Users\\Anton\\.cache\\huggingface\\datasets\\text\\default-2fcf8d2135508f85\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\\cache-f67f75d4d783a388.arrow\n",
      "Loading cached processed dataset at C:\\Users\\Anton\\.cache\\huggingface\\datasets\\text\\default-2fcf8d2135508f85\\0.0.0\\e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\\cache-049c6570017b7ca7.arrow\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text'],\n",
       "        num_rows: 36475\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['text'],\n",
       "        num_rows: 556\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load dataset from text file called \"data.txt\" and split into train/val\n",
    "datasets = load_dataset(\"text\", data_files=\"data_top_15_genres.txt\")['train']\n",
    "datasets = datasets.train_test_split(train_size=0.985, seed=42)\n",
    "datasets['validation'] = datasets.pop('test')\n",
    "\n",
    "datasets = datasets.map(processText, batched=True)\n",
    "datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0f79271",
   "metadata": {},
   "source": [
    "As can be seen, the examples are of different lengths. Examples longer than 1024 tokens needs to be truncated as this is the maximum input to GPT2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "490cccae",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BOS> <action> Gundaraj <SEP> Ajay Chauhan lives with his parents and younger sister. He is in love with Pooja, and hopes to marry her someday. His father wants him to get a job and settle down, and then get married. Ajay applies for a job in Bombay, and soon receives a letter asking him to appear for an interview. He attends the interview, and is hired. Delighted to see all his dreams coming true, he goes to offer his thanks to God, and it is there a woman named Pratika Jetley sees him and notifies the police that he is indeed the one who had brutally raped three young women in a college campus. Ajay vehemently denies this, but is personally identified and criminally held responsible, convicted and sentenced to prison. Several years later he is released from prison, and finds out that his father and Pooja had committed suicide while his mother and sister are untraceable. He sets out to put his life together and meets with a ruthless police inspector, whose daughter was one of the rape victims. It is then Ajay finds out about the conspiracy behind this plot to frame him. <EOS>\n",
      "\n",
      "<BOS> <drama> Kon Khon <SEP> Chart, an oprhan, was raised by his master after his parent's death and trained be to be a Khon dancer, but he becomes involved with a dance teacher named Rambai. <EOS>\n",
      "\n",
      "<BOS> <drama> The Blue Max <SEP> German Corporal Bruno Stachel  leaves the fighting in the trenches to become an officer and fighter pilot in the German Army Air Service. Joining a squadron in spring 1918, he sets his sights on winning Imperial Germany's highest military decoration for valor, the Pour le Mérite, nicknamed the \"Blue Max\", for which he must shoot down 20 aircraft. Of humble origins, Lieutenant Stachel is driven to prove himself better than the aristocratic pilots in his new fighter squadron, especially Willi von Klugermann . Their commanding officer, Hauptmann Otto Heidemann  is an upper-class officer whose notions of chivalry conflict with Stachel's ruthless determination. On his first mission, Stachel, in a Pfalz D.III, shoots down a British S.E.5, but does not receive credit for his \"kill\" because there were no witnesses. Stachel searches the French countryside for hours in a pouring rain for the wreckage, giving the impression that he cares more about scoring kills than the death of the man with whom he flew. Soon afterward, he attacks an Allied two-man observation aircraft, incapacitating the rear gunner. Instead of downing the defenseless aircraft, he signals the pilot to fly to the German base. As they near the airfield, the wounded rear gunner revives and reaches for his machine gun, unseen by the admiring observers on the ground. Stachel is forced to shoot the aircraft down, but Heidemann believes Stachel simply murdered a helpless enemy crew to gain a confirmed kill. The incident brings Stachel to the attention of General Count von Klugermann , Willi's uncle. When the general comes to the base to award his nephew the Blue Max, he meets Stachel. The general sees great propaganda potential in Stachel, one of the masses. Kaeti , the general's wife, is carrying on a discreet affair with her nephew by marriage. Soon afterward, Stachel is shot down after rescuing a red Fokker Dr.I attacked by two British fighters. When he returns to the airfield, he is stunned when he is introduced to the man he saved: Manfred von Richthofen , the Red Baron. Von Richthofen offers Stachel a place in his squadron, which Stachel declines, explaining his desire to \"prove himself\" with his current squadron. With Stachel temporarily grounded owing to a minor injury, General von Klugermann orders him to Berlin to help shore up crumbling public morale. Kaeti takes the opportunity to sleep with her latest hero. When Stachel returns to duty, he and Willi von Klugermann volunteer to escort a reconnaissance aircraft. British fighters attack their Fokker Dr. 1 triplanes. Stachel's guns jam, but Willi downs two of the enemy on his first pass, then a third on Stachel's tail, and the rest disengage. As the two are returning to their base, Willi challenges Stachel. Spotting a bridge, Willi dives under the wide middle span, but Stachel tops him by flying under a much narrower side one. Seething, Willi does the same, but clips the top of a tower afterward and crashes. When Stachel reports his death, Heidemann assumes that the two verified victories were Willi's. Insulted, Stachel impulsively claims them, even though it is discovered that he only fired 40 bullets before his guns jammed. Outraged, Heidemann reports Stachel to his superiors, but is told that Stachel's victories will be confirmed. Later, alone with Kaeti, Stachel admits he lied. During a strafing mission covering the retreat of the German army, Stachel disobeys Heidemann's order not to engage enemy fighters; one by one, the rest of the squadron follow him. Afterward, Heidemann has Stachel arrested, furious that nearly half the pilots were killed in the ensuing dogfight. Stachel cares only that he has shot down enough aircraft, even without Willi's kills, to qualify for the Blue Max. The two men are ordered to Berlin. There, von Klugermann tells Heidemann privately that Stachel is to receive the Blue Max, explaining that the people need a hero. Heidemann resigns his command when the general orders him to withdraw his report; he accepts a desk job. Later that evening, the countess visits Stachel and suggests that they run away to Switzerland since Germany's defeat is inevitable. She storms out when he refuses to give up flying. The next day, Stachel is awarded the Blue Max by the Crown Prince  in a well-publicized ceremony. However, a field marshal telephones von Klugermann to inform him of an impending investigation into Stachel's false claim. The general asks how the field marshal found out. While listening on the phone, he turns to his wife. When Heidemann reports that the new monoplane he has just test-flown is a \"death trap\", as its struts are dangerously weak, von Klugermann sees a way to avoid scandal. He orders Stachel to fly the aircraft and tells him, \"Let's see some real flying.\" The stress of Stachel's aerobatics causes the aircraft to crash. <EOS>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Example\n",
    "print(datasets['train'][0]['text'] + '\\n')\n",
    "print(datasets['train'][1]['text'] + '\\n')\n",
    "print(datasets['train'][3]['text'] + '\\n')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4bbb865",
   "metadata": {},
   "source": [
    "## Tokenization\n",
    "\n",
    "We now need to tokenize the dataset. The original tokenizer don't have all special tokens we require."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "595d00ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|endoftext|>\n"
     ]
    }
   ],
   "source": [
    "print(*tokenizer.all_special_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e37d5c9e",
   "metadata": {},
   "source": [
    "We need to add the special tokens that we use in our dataset. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4ac15f9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of added genres: 15\n",
      "50276 50276\n",
      "<BOS> <EOS> <|endoftext|> <SEP> <PAD> <romantic drama> <short film> <family film> <adventure> <action/adventure> <indie> <black-and-white> <horror> <crime fiction> <world cinema> <action> <thriller> <romance film> <comedy> <drama>\n"
     ]
    }
   ],
   "source": [
    "# Set special tokens\n",
    "if not start_from_checkpoint:\n",
    "    tokenizer.bos_token = '<BOS>'\n",
    "    tokenizer.eos_token = '<EOS>'\n",
    "    tokenizer.pad_token = '<PAD>'\n",
    "    tokenizer.sep_token = '<SEP>'\n",
    "\n",
    "    # Add special tokens for each genre\n",
    "    genres = ['romantic drama', 'short film', 'family film',\n",
    "              'adventure', 'action/adventure', 'indie',\n",
    "              'black-and-white', 'horror', 'crime fiction',\n",
    "              'world cinema', 'action', 'thriller', \n",
    "              'romance film', 'comedy', 'drama']\n",
    "\n",
    "    print(f'Number of added genres: {len(genres)}')\n",
    "    new_special_tokens = ['<BOS>', '<EOS>', '<PAD>', '<SEP>']\n",
    "    new_special_tokens.extend([f'<{genre}>' for genre in genres])\n",
    "    special_tokens = tokenizer.additional_special_tokens\n",
    "\n",
    "    special_tokens.extend(new_special_tokens) \n",
    "    new_special_tokens_dict = {'additional_special_tokens': special_tokens}\n",
    "    num_added_toks = tokenizer.add_special_tokens(new_special_tokens_dict)\n",
    "\n",
    "    # We must resize token embeddings since new special tokens were added\n",
    "    model.resize_token_embeddings(len(tokenizer))\n",
    "\n",
    "print(model.config.vocab_size, tokenizer.vocab_size + len(tokenizer.get_added_vocab()))\n",
    "assert(model.config.vocab_size == tokenizer.vocab_size + len(tokenizer.get_added_vocab()))\n",
    "print(*tokenizer.all_special_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "853e6b5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<BOS>', 'Ġ', '<drama>', 'ĠHe', 'Ġwas']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize('<BOS> <drama> He was' )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "367d6e30",
   "metadata": {},
   "source": [
    "**Tokenize the dataset**\n",
    "\n",
    "We tokenize the dataset. The tokenized examples contain the column names 'attention_mask' which is a mask for padding tokens and 'input_ids' which is the id of each token corrsponding to a word. We drop the text as that is not needed anymore. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f618dc0",
   "metadata": {},
   "source": [
    "Note that we duplicate the inputs to add our labels. This is because the model of the 🤗 Transformers library apply the shifting to the right, so we don't need to do it manually."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "22681185",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01cb41e2352448debb1f82fbf53bc0e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=37.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fea1f36de5b14128be43cc8c19f8b04c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def tokenize_function(examples):\n",
    "    \"\"\"\n",
    "    padding='max_length' to pad to a length specified by the max_length argument \n",
    "    or the maximum length accepted by the model.\n",
    "    truncation=True to truncate each sequence to the maximum length accepted by the model\n",
    "    \"\"\"\n",
    "    result = tokenizer(examples[\"text\"], padding='max_length', truncation=True) # Max input according to model(1024)\n",
    "\n",
    "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
    "    return result\n",
    "\n",
    "tokenized_datasets = datasets.map(tokenize_function, batched=True, remove_columns=[\"text\"])\n",
    "\n",
    "#Make dataset format pytorch tensors\n",
    "tokenized_datasets.set_format(\"torch\")\n",
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bbd91dce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Finally, extract the datasets and select a subset if wanted\n",
    "train_set = tokenized_datasets['train']#.select(list(range(10)))\n",
    "valid_set = tokenized_datasets['validation']#.select(list(range(2)))\n",
    "print(train_set, valid_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fefa121",
   "metadata": {},
   "source": [
    "### Training\n",
    "First, setup training args.\n",
    "The last argument to setup everything so we can push the model to the Hub regularly during training..\n",
    "\n",
    "Then pass training args to Trainer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6e6034ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SaveTokenizer(TrainerCallback):\n",
    "    \"\"\"\n",
    "    A callback used to save the tokenizer whenever a model checkpoint is saved.\n",
    "    \"\"\"\n",
    "    def on_save(self, args, state, control, **kwargs):\n",
    "        tokenizer.save_pretrained(finetuned_model_name)\n",
    "\n",
    "        \n",
    "ce_loss = torch.nn.CrossEntropyLoss()\n",
    "        \n",
    "def compute_metrics(eval_pred):\n",
    "    \"\"\"\n",
    "    The compute function needs to receive a tuple (with logits and labels)\n",
    "    and has to return a dictionary with string keys (the name of the metric) and float values.\n",
    "    It will be called at the end of each evaluation phase on the whole arrays of predictions/labels.\n",
    "    \"\"\"\n",
    "    logits, labels = eval_pred\n",
    "    # Calculate perplexity https://huggingface.co/transformers/perplexity.html\n",
    "    # \"the exponentiation of the cross-entropy between the data and model predictions.\"\n",
    "    \n",
    "    perplexity = math.exp(ce_loss(logits, labels))\n",
    "    \n",
    "    return {'perplexity': perplexity}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0ce3eb8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()\n",
    "batch_size = 1 \n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    finetuned_model_name,\n",
    "    evaluation_strategy = \"no\",\n",
    "    num_train_epochs=1,\n",
    "    learning_rate=1e-6,\n",
    "    weight_decay=0.01,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=1,\n",
    "    save_steps=2000,\n",
    "    save_total_limit=1,\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_set,\n",
    "    eval_dataset=valid_set,\n",
    "    compute_metrics=compute_metrics,\n",
    "    callbacks=[SaveTokenizer],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cf3df559",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 36475\n",
      "  Num Epochs = 1\n",
      "  Instantaneous batch size per device = 1\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 1\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 36475\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='36475' max='36475' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [36475/36475 2:38:51, Epoch 1/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>11.512800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>1.957400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1500</td>\n",
       "      <td>1.613500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>1.503500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2500</td>\n",
       "      <td>1.467800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>1.430100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3500</td>\n",
       "      <td>1.334200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>1.445800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4500</td>\n",
       "      <td>1.453500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>1.397400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5500</td>\n",
       "      <td>1.360800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>1.324600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6500</td>\n",
       "      <td>1.321300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7000</td>\n",
       "      <td>1.309500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7500</td>\n",
       "      <td>1.371200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8000</td>\n",
       "      <td>1.364200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8500</td>\n",
       "      <td>1.290100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9000</td>\n",
       "      <td>1.280300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9500</td>\n",
       "      <td>1.318600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10000</td>\n",
       "      <td>1.346700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10500</td>\n",
       "      <td>1.387500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11000</td>\n",
       "      <td>1.391600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11500</td>\n",
       "      <td>1.342300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12000</td>\n",
       "      <td>1.339100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12500</td>\n",
       "      <td>1.325300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13000</td>\n",
       "      <td>1.329500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13500</td>\n",
       "      <td>1.333000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14000</td>\n",
       "      <td>1.232800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14500</td>\n",
       "      <td>1.334900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15000</td>\n",
       "      <td>1.323700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15500</td>\n",
       "      <td>1.323300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16000</td>\n",
       "      <td>1.339800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16500</td>\n",
       "      <td>1.294200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17000</td>\n",
       "      <td>1.279500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17500</td>\n",
       "      <td>1.254800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18000</td>\n",
       "      <td>1.307900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18500</td>\n",
       "      <td>1.404400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19000</td>\n",
       "      <td>1.355300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19500</td>\n",
       "      <td>1.308200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20000</td>\n",
       "      <td>1.245200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20500</td>\n",
       "      <td>1.292700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21000</td>\n",
       "      <td>1.378200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21500</td>\n",
       "      <td>1.340100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22000</td>\n",
       "      <td>1.302200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22500</td>\n",
       "      <td>1.349300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23000</td>\n",
       "      <td>1.326300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23500</td>\n",
       "      <td>1.305500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24000</td>\n",
       "      <td>1.292900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24500</td>\n",
       "      <td>1.359200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25000</td>\n",
       "      <td>1.399000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25500</td>\n",
       "      <td>1.347700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26000</td>\n",
       "      <td>1.161900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26500</td>\n",
       "      <td>1.333400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27000</td>\n",
       "      <td>1.254600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27500</td>\n",
       "      <td>1.312800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28000</td>\n",
       "      <td>1.304500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28500</td>\n",
       "      <td>1.318700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29000</td>\n",
       "      <td>1.305000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29500</td>\n",
       "      <td>1.227900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30000</td>\n",
       "      <td>1.314100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30500</td>\n",
       "      <td>1.280700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31000</td>\n",
       "      <td>1.273700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31500</td>\n",
       "      <td>1.379500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32000</td>\n",
       "      <td>1.290800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32500</td>\n",
       "      <td>1.318400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33000</td>\n",
       "      <td>1.373200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33500</td>\n",
       "      <td>1.287100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34000</td>\n",
       "      <td>1.361200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34500</td>\n",
       "      <td>1.327200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35000</td>\n",
       "      <td>1.241500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35500</td>\n",
       "      <td>1.281100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36000</td>\n",
       "      <td>1.301500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-2000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-2000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-2000\\pytorch_model.bin\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-4000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-4000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-4000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-2000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-6000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-6000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-6000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-4000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-8000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-8000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-8000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-6000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-10000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-10000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-10000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-8000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-12000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-12000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-12000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-10000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-14000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-14000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-14000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-12000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-16000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-16000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-16000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-14000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-18000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-18000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-18000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-16000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-20000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-20000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-20000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-18000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-22000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-22000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-22000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-20000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-24000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-24000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-24000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-22000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-26000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-26000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-26000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-24000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-28000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-28000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-28000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-26000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-30000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-30000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-30000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-28000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-32000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-32000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-32000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-30000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-34000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-34000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-34000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-32000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "Saving model checkpoint to pranavpsv/gpt2-genre-story-generator\\checkpoint-36000\n",
      "Configuration saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-36000\\config.json\n",
      "Model weights saved in pranavpsv/gpt2-genre-story-generator\\checkpoint-36000\\pytorch_model.bin\n",
      "Deleting older checkpoint [pranavpsv\\gpt2-genre-story-generator\\checkpoint-34000] due to args.save_total_limit\n",
      "tokenizer config file saved in ./pranavpsv/gpt2-genre-story-generator/tokenizer_config.json\n",
      "Special tokens file saved in ./pranavpsv/gpt2-genre-story-generator/special_tokens_map.json\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Configuration saved in movie-plot-generator\\config.json\n",
      "Model weights saved in movie-plot-generator\\pytorch_model.bin\n",
      "tokenizer config file saved in movie-plot-generator\\tokenizer_config.json\n",
      "Special tokens file saved in movie-plot-generator\\special_tokens_map.json\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('movie-plot-generator\\\\tokenizer_config.json',\n",
       " 'movie-plot-generator\\\\special_tokens_map.json',\n",
       " 'movie-plot-generator\\\\vocab.json',\n",
       " 'movie-plot-generator\\\\merges.txt',\n",
       " 'movie-plot-generator\\\\added_tokens.json',\n",
       " 'movie-plot-generator\\\\tokenizer.json')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_results=trainer.train()\n",
    "pickle.dump(train_results, open(\"train_results.pickle\", \"wb\")) #Load: train_results = pickle.load(open(\"train_results.pickle\", \"rb\"))\n",
    "\n",
    "model.save_pretrained(\"movie-plot-generator\")\n",
    "tokenizer.save_pretrained(\"movie-plot-generator\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6719995a",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_results = trainer.evaluate()\n",
    "print(f\"Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c608d27",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.push_to_hub(finetuned_model_name)\n",
    "trainer.push_to_hub(finetuned_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e89be0ae",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BOS> <romantic drama> Expecting the unexpected <SEP> Kajsa and Anton are inseparable while they meet each other in a pub.  A boy and his sister are hired to take over the family's daughter's job and is hired to take over the family's job. She is then sent to work for the family. She is then taken over by the family, and is then brought out to a girl who has a child, and she is then introduced to the girls who also have a child, and she is then brought to the main group, and she is then brought up on a tour of the family's farm as a bus driver. She is then brought back to the family to take her back to the family, and she is brought on a bus to a group that is getting the kids to be with each other. In the end, the group is brought into a group which is brought to a farm in the afternoon. The group is brought up to a group of boys, while the boys are allowed to be with the boys. The boys are given that they will be for the other boys and they are handed a few things. The group is then given what they want, and they are told that they will be given a few things. The boys are then told there are two boys, and there are two girls in three of a row, and they are told that they will be for the other boys. The boys are told that they will be there, and they are told that they will be there, so they are told that they will be in that situation. The group is given a few things, and they are told that they will be given a few things, and they are given a few things. The boys are given the truth of the boys' And the boys are told that they will be given a few things, and the boys are told that\n",
      "\n",
      "\n",
      "------------------------\n",
      " <BOS> <romantic drama> Expecting the unexpected <SEP> Kajsa and Anton are back to meet up in the middle of the night and talk about something they never made sense of. They're both the only couple in their own boat and there seems to be no one to support them, especially in the rain. They start with the idea that they're having in a small town. Kajsa is an American college student, who is the only thing on the radar that he thinks is going to find his way back in the home to try to find things out, and Anton is not even the most logical about this. Anton was the one who decided to join the party because he'd heard of an attempt to have someone replace a family member in order to steal a lost home. He's been in New York working for the Wall Street financial firm while Anton is currently in Mexico trying to find his way back.  19 September 1971 The Movie: The Lost Years by J.K. Kumpdollendell and David Poulson, a journalist from Los Angeles, is investigating the disappearance of a girl in the rain. She is found at a cabin in the middle of an arid California desert. She's lying dead at work. The next day they encounter another girl, who is already dead. The two fall in love. They go to a motel, where they meet outside. The other girl, who is not the last one to be found, tells the police she's been missing. The two meet at a Mexican motel, where they are trying to locate her. They are pursued by a cop. After the cop leaves the motel, they are pursued, and they are arrested. The police get a new woman, who does not have a boyfriend. Later that night, the house is discovered to be a dumpster belonging to a man who tried to kill them. The owner of the house,\n",
      "\n",
      "\n",
      "------------------------\n",
      " <BOS> <romantic drama> Expecting the unexpected <SEP> Kajsa and Anton are both involved in planning a plot against the Empire. However, Kajsa thinks he still has the time left. Angered by the situation, Kajsa and Anton set off. Kajsa plans to start a war on the Empire with an imperialistic outlook. But an Imperial man has come along to take control as his apprentice. The plan is to bring his father to the table and then decide whether or 10,000 D.E-541, September 25, 1975 film, \" Expectation of a War\" is directed by Chaz and starring T.J. Lawrence, Bobbie Miller, Jennifer Lawrence and Sam Neumacher. The film features a live action film from director Guillermo Domingue, which features film director David Mamet. This film is based on the comic strip \" Expectation of a War: A comic strip starring Guillermo Domingue and Sam Neumacher.\" It also features a music video production film. The film was released in 2001 and has been made available to film-makers in the United States. In its 90-minute period, the film depicts the lives and deaths of the first five billion citizens of the Empire and their families. The film also features feature film theme lyrics, and a lyric from the movie. In addition to the film story was written by the writers and producers, along with the director. It is directed by Jennifer Lawrence. It features a brief promotional video from her production of the film and is set to feature the closing credits. The film was premiered in 2001 at the Sundance Film Festival. It was distributed worldwide and had the video theme song lyrics by Lulu. It was released in 2002, but is not available in the United States. The film is based on several of the \"I Want to Do My Party\" films and an early release movie. The film is available in several English dubs by various artists, including Dwayne, whose character is a character featured in the film. He appears in the film in a large dress and playing a guitar. His character has a brief scene in which he is shown walking, and wearing an electric wheelchair in an early version of the film. There is also the music video from the film. It features music by Kevin and Paul, known as \"Kajsa and Anton\" in the film by Chaz. In the movie is an\n",
      "\n",
      "\n",
      "------------------------\n",
      " <BOS> <romantic drama> Expecting the unexpected <SEP> Kajsa and Anton are both from West Bengal. At first Kajsa is unaware of his father's affair, and the pair are not aware of these details of his history. However, Kajsa's father soon reveals that Kajsa had taken an interest in Anton and had taken a liking to his daughter at school. He finds Anton unexpectedly a part of Kajsa's plans. At first Kajsa believes Anton to be an angel but Anton's father knows about everything. However Kajsa realizes his son knows nothing about Anton's true nature; Kajsa goes on a rampage. After the massacre at Kajsa, Anton and Kajsa are left with none to judge if Anton will be allowed to live.  47 33 Dark Horse: The Story of an Indian Nunnath, Bangladesh, India, 1950-1971 | 16 In a century, a young man named Anand is a member of the town's church, and is called the \"Nunnais\". The film is shot in London. The film is a comedy film, starring John T. Leighton, an Indian-American American-American who is in a relationship with the younger John. Leighton, a young man who was the son of Jind's father, an Indian-American who is known to be a good mother to his son, and to be a better father to his wife's son. The film opens in London. The film was directed by William Shady, who was a director of the Film\n",
      "\n",
      "\n",
      "------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Inference test\n",
    "generator = pipeline(\"text-generation\", model=model, tokenizer=tokenizer, device=0)\n",
    "stories = generator(\"<BOS> <drama> Expecting the unexpected <SEP>\", max_length=512, num_return_sequences=4)\n",
    "print(*[story['generated_text'] + \"\\n\\n\\n------------------------\\n\" for story in stories])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}