Preprocess_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "16a1e617",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "058bd91d",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_data = pd.read_csv('MovieSummaries/movie.metadata.tsv', sep='\\t', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "6530dbd7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>1</th>\n",
       "      <th>Title</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>Genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>975900</td>\n",
       "      <td>/m/03vyhn</td>\n",
       "      <td>Ghosts of Mars</td>\n",
       "      <td>2001-08-24</td>\n",
       "      <td>14010832.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>{\"/m/02h40lc\": \"English Language\"}</td>\n",
       "      <td>{\"/m/09c7w0\": \"United States of America\"}</td>\n",
       "      <td>{\"/m/01jfsb\": \"Thriller\", \"/m/06n90\": \"Science...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3196793</td>\n",
       "      <td>/m/08yl5d</td>\n",
       "      <td>Getting Away with Murder: The JonBenét Ramsey ...</td>\n",
       "      <td>2000-02-16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>95.0</td>\n",
       "      <td>{\"/m/02h40lc\": \"English Language\"}</td>\n",
       "      <td>{\"/m/09c7w0\": \"United States of America\"}</td>\n",
       "      <td>{\"/m/02n4kr\": \"Mystery\", \"/m/03bxz7\": \"Biograp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28463795</td>\n",
       "      <td>/m/0crgdbh</td>\n",
       "      <td>Brun bitter</td>\n",
       "      <td>1988</td>\n",
       "      <td>NaN</td>\n",
       "      <td>83.0</td>\n",
       "      <td>{\"/m/05f_3\": \"Norwegian Language\"}</td>\n",
       "      <td>{\"/m/05b4w\": \"Norway\"}</td>\n",
       "      <td>{\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9363483</td>\n",
       "      <td>/m/0285_cd</td>\n",
       "      <td>White Of The Eye</td>\n",
       "      <td>1987</td>\n",
       "      <td>NaN</td>\n",
       "      <td>110.0</td>\n",
       "      <td>{\"/m/02h40lc\": \"English Language\"}</td>\n",
       "      <td>{\"/m/07ssc\": \"United Kingdom\"}</td>\n",
       "      <td>{\"/m/01jfsb\": \"Thriller\", \"/m/0glj9q\": \"Erotic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>261236</td>\n",
       "      <td>/m/01mrr1</td>\n",
       "      <td>A Woman in Flames</td>\n",
       "      <td>1983</td>\n",
       "      <td>NaN</td>\n",
       "      <td>106.0</td>\n",
       "      <td>{\"/m/04306rv\": \"German Language\"}</td>\n",
       "      <td>{\"/m/0345h\": \"Germany\"}</td>\n",
       "      <td>{\"/m/07s9rl0\": \"Drama\"}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Wiki_ID           1                                              Title  \\\n",
       "0    975900   /m/03vyhn                                     Ghosts of Mars   \n",
       "1   3196793   /m/08yl5d  Getting Away with Murder: The JonBenét Ramsey ...   \n",
       "2  28463795  /m/0crgdbh                                        Brun bitter   \n",
       "3   9363483  /m/0285_cd                                   White Of The Eye   \n",
       "4    261236   /m/01mrr1                                  A Woman in Flames   \n",
       "\n",
       "            3           4      5                                   6  \\\n",
       "0  2001-08-24  14010832.0   98.0  {\"/m/02h40lc\": \"English Language\"}   \n",
       "1  2000-02-16         NaN   95.0  {\"/m/02h40lc\": \"English Language\"}   \n",
       "2        1988         NaN   83.0  {\"/m/05f_3\": \"Norwegian Language\"}   \n",
       "3        1987         NaN  110.0  {\"/m/02h40lc\": \"English Language\"}   \n",
       "4        1983         NaN  106.0   {\"/m/04306rv\": \"German Language\"}   \n",
       "\n",
       "                                           7  \\\n",
       "0  {\"/m/09c7w0\": \"United States of America\"}   \n",
       "1  {\"/m/09c7w0\": \"United States of America\"}   \n",
       "2                     {\"/m/05b4w\": \"Norway\"}   \n",
       "3             {\"/m/07ssc\": \"United Kingdom\"}   \n",
       "4                    {\"/m/0345h\": \"Germany\"}   \n",
       "\n",
       "                                               Genre  \n",
       "0  {\"/m/01jfsb\": \"Thriller\", \"/m/06n90\": \"Science...  \n",
       "1  {\"/m/02n4kr\": \"Mystery\", \"/m/03bxz7\": \"Biograp...  \n",
       "2  {\"/m/0lsxr\": \"Crime Fiction\", \"/m/07s9rl0\": \"D...  \n",
       "3  {\"/m/01jfsb\": \"Thriller\", \"/m/0glj9q\": \"Erotic...  \n",
       "4                            {\"/m/07s9rl0\": \"Drama\"}  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_data.rename(columns={0:\"Wiki_ID\", 2:\"Title\", 8:\"Genre\"}, inplace=True) \n",
    "movie_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "b61c68b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_info = movie_data[[\"Wiki_ID\", \"Title\", \"Genre\"]]\n",
    "for row in range(movie_info.shape[0]):\n",
    "\n",
    "    row_genre = movie_info.loc[row, \"Genre\"]\n",
    "\n",
    "    genre = ','.join(list(ast.literal_eval(row_genre).values()))\n",
    "\n",
    "    movie_info.loc[row, \"Genre\"] = genre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "9ba72770",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Genre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>975900</td>\n",
       "      <td>Ghosts of Mars</td>\n",
       "      <td>Thriller,Science Fiction,Horror,Adventure,Supe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3196793</td>\n",
       "      <td>Getting Away with Murder: The JonBenét Ramsey ...</td>\n",
       "      <td>Mystery,Biographical film,Drama,Crime Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>28463795</td>\n",
       "      <td>Brun bitter</td>\n",
       "      <td>Crime Fiction,Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9363483</td>\n",
       "      <td>White Of The Eye</td>\n",
       "      <td>Thriller,Erotic thriller,Psychological thriller</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>261236</td>\n",
       "      <td>A Woman in Flames</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Wiki_ID                                              Title  \\\n",
       "0    975900                                     Ghosts of Mars   \n",
       "1   3196793  Getting Away with Murder: The JonBenét Ramsey ...   \n",
       "2  28463795                                        Brun bitter   \n",
       "3   9363483                                   White Of The Eye   \n",
       "4    261236                                  A Woman in Flames   \n",
       "\n",
       "                                               Genre  \n",
       "0  Thriller,Science Fiction,Horror,Adventure,Supe...  \n",
       "1        Mystery,Biographical film,Drama,Crime Drama  \n",
       "2                                Crime Fiction,Drama  \n",
       "3    Thriller,Erotic thriller,Psychological thriller  \n",
       "4                                              Drama  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_info.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "a9b8641d",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_info.to_csv(\"movie_title_genre.tsv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "a1a92cf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_plot = pd.read_csv('MovieSummaries/plot_summaries.txt', sep='\\t', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "4f7490f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_plot.rename(columns={0:\"Wiki_ID\", 1:\"Plot\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "4e98f26d",
   "metadata": {},
   "outputs": [],
   "source": [
    "movie_info_plot = movie_info.merge(movie_plot, on=\"Wiki_ID\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "1226b01d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wiki_ID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Plot</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>975900</td>\n",
       "      <td>Ghosts of Mars</td>\n",
       "      <td>Thriller,Science Fiction,Horror,Adventure,Supe...</td>\n",
       "      <td>Set in the second half of the 22nd century, th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9363483</td>\n",
       "      <td>White Of The Eye</td>\n",
       "      <td>Thriller,Erotic thriller,Psychological thriller</td>\n",
       "      <td>A series of murders of rich young women throug...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>261236</td>\n",
       "      <td>A Woman in Flames</td>\n",
       "      <td>Drama</td>\n",
       "      <td>Eva, an upper class housewife, becomes frustra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>18998739</td>\n",
       "      <td>The Sorcerer's Apprentice</td>\n",
       "      <td>Family Film,Fantasy,Adventure,World cinema</td>\n",
       "      <td>Every hundred years, the evil Morgana  returns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6631279</td>\n",
       "      <td>Little city</td>\n",
       "      <td>Romantic comedy,Ensemble Film,Comedy-drama,Dra...</td>\n",
       "      <td>Adam, a San Francisco-based artist who works a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42199</th>\n",
       "      <td>23851782</td>\n",
       "      <td>The Ghost Train</td>\n",
       "      <td>Crime Fiction,Thriller,Comedy,Supernatural</td>\n",
       "      <td>{{plot}} The film opens with a Great Western e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42200</th>\n",
       "      <td>35228177</td>\n",
       "      <td>Mermaids: The Body Found</td>\n",
       "      <td>Drama</td>\n",
       "      <td>Two former National Oceanic Atmospheric Admini...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42201</th>\n",
       "      <td>34980460</td>\n",
       "      <td>Knuckle</td>\n",
       "      <td>Biographical film,Drama,Documentary</td>\n",
       "      <td>{{No plot}} This film follows 12 years in the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42202</th>\n",
       "      <td>913762</td>\n",
       "      <td>The Super Dimension Fortress Macross II: Lover...</td>\n",
       "      <td>Science Fiction,Japanese Movies,Adventure,Anim...</td>\n",
       "      <td>The story takes place in the year 2092,The Sup...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42203</th>\n",
       "      <td>12476867</td>\n",
       "      <td>Spliced</td>\n",
       "      <td>Thriller,Horror,Slasher,Teen</td>\n",
       "      <td>The movie is about a teenage girl who loves ho...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>42204 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Wiki_ID                                              Title  \\\n",
       "0        975900                                     Ghosts of Mars   \n",
       "1       9363483                                   White Of The Eye   \n",
       "2        261236                                  A Woman in Flames   \n",
       "3      18998739                          The Sorcerer's Apprentice   \n",
       "4       6631279                                        Little city   \n",
       "...         ...                                                ...   \n",
       "42199  23851782                                    The Ghost Train   \n",
       "42200  35228177                           Mermaids: The Body Found   \n",
       "42201  34980460                                            Knuckle   \n",
       "42202    913762  The Super Dimension Fortress Macross II: Lover...   \n",
       "42203  12476867                                            Spliced   \n",
       "\n",
       "                                                   Genre  \\\n",
       "0      Thriller,Science Fiction,Horror,Adventure,Supe...   \n",
       "1        Thriller,Erotic thriller,Psychological thriller   \n",
       "2                                                  Drama   \n",
       "3             Family Film,Fantasy,Adventure,World cinema   \n",
       "4      Romantic comedy,Ensemble Film,Comedy-drama,Dra...   \n",
       "...                                                  ...   \n",
       "42199         Crime Fiction,Thriller,Comedy,Supernatural   \n",
       "42200                                              Drama   \n",
       "42201                Biographical film,Drama,Documentary   \n",
       "42202  Science Fiction,Japanese Movies,Adventure,Anim...   \n",
       "42203                       Thriller,Horror,Slasher,Teen   \n",
       "\n",
       "                                                    Plot  \n",
       "0      Set in the second half of the 22nd century, th...  \n",
       "1      A series of murders of rich young women throug...  \n",
       "2      Eva, an upper class housewife, becomes frustra...  \n",
       "3      Every hundred years, the evil Morgana  returns...  \n",
       "4      Adam, a San Francisco-based artist who works a...  \n",
       "...                                                  ...  \n",
       "42199  {{plot}} The film opens with a Great Western e...  \n",
       "42200  Two former National Oceanic Atmospheric Admini...  \n",
       "42201  {{No plot}} This film follows 12 years in the ...  \n",
       "42202  The story takes place in the year 2092,The Sup...  \n",
       "42203  The movie is about a teenage girl who loves ho...  \n",
       "\n",
       "[42204 rows x 4 columns]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_info_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "06428151",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('movie_info_plot_trainig.txt', 'w') as train_file, open('movie_info_plot_testing.txt', 'w') as test_file:\n",
    "    for row in range(movie_info_plot.shape[0]):\n",
    "        line = \"<BOS>\" + \" \"\n",
    "        for g in movie_info_plot.loc[row, \"Genre\"].split(','):\n",
    "            line += \"<\" + g+ \"> \"\n",
    "        line += \"<SEP> \" + movie_info_plot.loc[row, \"Plot\"] + \"<EOS>\\n\"\n",
    "        if row < movie_info_plot.shape[0] - 1000:\n",
    "            train_file.write(line)\n",
    "        else:\n",
    "            test_file.write(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "719c0568",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42204, 4)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie_info_plot.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0baa980",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}