Preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1M split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# UserID::Gender::Age::Occupation::Zip-code\n",
    "# MovieID::Title::Genres\n",
    "# UserID::MovieID::Rating::Timestamp (5-star scale)\n",
    "\n",
    "# Importing the dataset\n",
    "#movies = pd.read_csv('./data/ml_1m/movies.dat', sep = '::', header = None, engine = 'python', encoding = 'latin-1')\n",
    "#users = pd.read_csv('./data/ml_1m/users.dat', sep = '::', header = None, engine = 'python', encoding = 'latin-1')\n",
    "ratings = pd.read_csv('./data/ml_1m/ratings.dat', sep = '::', header = None, engine = 'python', encoding = 'latin-1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_length = len(ratings)\n",
    "ratings = ratings.sample(frac=1)\n",
    "\n",
    "len_train = int(total_length*0.85)\n",
    "len_val   = int(total_length*0.9)\n",
    "\n",
    "rating_train = ratings[:len_train]\n",
    "rating_val   = ratings[len_train:len_val]\n",
    "rating_test  = ratings[len_val:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_users  = 6040\n",
    "num_items = 3953\n",
    "rating_cnt = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, ratings in enumerate([rating_train, rating_val, rating_test]):\n",
    "    rating_mtx = torch.zeros(rating_cnt, num_users, num_items)\n",
    "    \n",
    "    for index, row in ratings.iterrows():\n",
    "        u = row[0]-1\n",
    "        v = row[1]-1\n",
    "        r = row[2]-1\n",
    "        \n",
    "        rating_mtx[r, u, v] = 1\n",
    "    torch.save(rating_mtx, './data/rating_%d.pkl'%i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_headers = ['user id', 'gender', 'age', 'occupation', 'zip code']\n",
    "users_df = pd.read_csv('./data/ml_1m/users.dat', sep = '::', header = None, names = users_headers, engine = 'python', encoding = 'latin-1')\n",
    "movie_headers = ['movie id', 'movie title', 'genre']\n",
    "movie_df = pd.read_csv('./data/ml_1m/movies.dat', sep = '::', header = None, names = movie_headers, engine = 'python', encoding = 'latin-1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "occupation = set(users_df['occupation'].values.tolist())\n",
    "age_dict = {1:0., 18:1., 25:2., 35:3., 45:4., 50:5., 56:6.}\n",
    "gender_dict = {'M': 0., 'F': 1.}\n",
    "occupation_dict = {f: i for i, f in enumerate(occupation, start=2)}\n",
    "\n",
    "num_feats = 2 + len(occupation_dict)\n",
    "\n",
    "u_features = np.zeros((num_users, num_feats), dtype=np.float32)\n",
    "for _, row in users_df.iterrows():\n",
    "    u_id = row['user id']-1\n",
    "    # age\n",
    "    u_features[u_id, 0] = age_dict[row['age']]\n",
    "    # gender\n",
    "    u_features[u_id, 1] = gender_dict[row['gender']]\n",
    "    # occupation\n",
    "    u_features[u_id, occupation_dict[row['occupation']]] = 1.\n",
    "torch.save(torch.from_numpy(u_features), './data/ml_1m/u_features.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre_dict = {'Action':0, 'Adventure':1, 'Animation':2, \"Children's\":3, 'Comedy':4,\n",
    "              'Crime':5, 'Documentary':6, 'Drama':7, 'Fantasy':8, 'Film-Noir':9, 'Horror':10,\n",
    "              'Musical':11, 'Mystery':12, 'Romance':13, 'Sci-Fi':14, 'Thriller':15,\n",
    "              'War':16, 'Western':17}\n",
    "num_genres = len(genre_dict)\n",
    "\n",
    "v_features = np.zeros((num_items, num_genres), dtype=np.float32)\n",
    "for movie_id, g_vec in zip(movie_df['movie id'].values.tolist(), movie_df['genre'].values.tolist()):\n",
    "    # check if movie_id was listed in ratings file and therefore in mapping dictionary\n",
    "    for j in [genre_dict[g] for g in g_vec.split('|')]:\n",
    "        v_features[movie_id-1][j] = 1\n",
    "\n",
    "torch.save(torch.from_numpy(v_features), './data/ml_1m/v_features.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 100K split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('./data/ml_100k/u1.base', sep = '\\t', header = None, engine = 'python', encoding = 'latin-1')\n",
    "test  = pd.read_csv('./data/ml_100k/u1.test', sep = '\\t', header = None, engine = 'python', encoding = 'latin-1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_length = len(train)\n",
    "train = train.sample(frac=1)\n",
    "\n",
    "len_train = int(train_length*0.9)\n",
    "\n",
    "rating_train = train[:len_train]\n",
    "rating_val   = train[len_train:]\n",
    "rating_test  = test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_users = 943\n",
    "num_items = 1682\n",
    "rating_cnt= 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, ratings in enumerate([rating_train, rating_val, rating_test]):\n",
    "    rating_mtx = torch.zeros(rating_cnt, num_users, num_items)\n",
    "    \n",
    "    for index, row in ratings.iterrows():\n",
    "        u = row[0]-1\n",
    "        v = row[1]-1\n",
    "        r = row[2]-1\n",
    "        \n",
    "        rating_mtx[r, u, v] = 1\n",
    "    torch.save(rating_mtx, './data/ml_100k/rating_%d.pkl'%i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_headers = ['user id', 'age', 'gender', 'occupation', 'zip code']\n",
    "users_df = pd.read_csv('./data/ml_100k/u.user', sep = '|', header = None, names = users_headers, engine = 'python', encoding = 'latin-1')\n",
    "movie_headers = ['movie id', 'movie title', 'release date', 'video release date',\n",
    "                 'IMDb URL', 'unknown', 'Action', 'Adventure', 'Animation',\n",
    "                 'Childrens', 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy',\n",
    "                 'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi',\n",
    "                 'Thriller', 'War', 'Western']\n",
    "movie_df = pd.read_csv('./data/ml_100k/u.item', sep = '|', header = None, names = movie_headers, engine = 'python', encoding = 'latin-1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "occupation = set(users_df['occupation'].values.tolist())\n",
    "age = users_df['age'].values\n",
    "age_max = age.max()\n",
    "gender_dict = {'M': 0., 'F': 1.}\n",
    "occupation_dict = {f: i for i, f in enumerate(occupation, start=2)}\n",
    "\n",
    "num_feats = 2 + len(occupation_dict)\n",
    "\n",
    "u_features = np.zeros((num_users, num_feats), dtype=np.float32)\n",
    "for _, row in users_df.iterrows():\n",
    "    u_id = row['user id']-1\n",
    "    # age\n",
    "    u_features[u_id, 0] = row['age'] / np.float(age_max)\n",
    "    # gender\n",
    "    u_features[u_id, 1] = gender_dict[row['gender']]\n",
    "    # occupation\n",
    "    u_features[u_id, occupation_dict[row['occupation']]] = 1.\n",
    "torch.save(torch.from_numpy(u_features), './data/ml_100k/u_features.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre_headers = movie_df.columns.values[6:]\n",
    "num_genres = genre_headers.shape[0]\n",
    "\n",
    "v_features = np.zeros((num_items, num_genres), dtype=np.float32)\n",
    "for movie_id, g_vec in zip(movie_df['movie id'].values.tolist(), movie_df[genre_headers].values.tolist()):\n",
    "    # check if movie_id was listed in ratings file and therefore in mapping dictionary\n",
    "    v_features[movie_id-1] = g_vec\n",
    "torch.save(torch.from_numpy(v_features), './data/ml_100k/v_features.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}