nasdaq-training-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-06T14:52:00.352138Z",
     "start_time": "2024-01-06T14:52:00.350077Z"
    }
   },
   "outputs": [],
   "source": [
    "import hopsworks\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.models import Sequential\n",
    "import os\n",
    "import numpy as np\n",
    "import joblib\n",
    "import yfinance as yf\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-06T14:52:03.066286Z",
     "start_time": "2024-01-06T14:52:01.284869Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# # You have to set the environment variable 'HOPSWORKS_API_KEY' for login to succeed\n",
    "project = hopsworks.login()\n",
    "fs = project.get_feature_store()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-06T15:13:15.073686Z",
     "start_time": "2024-01-06T15:13:11.619225Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# Get features (data) from feature store\n",
    "# nasdaq data\n",
    "nasdaq_fg = fs.get_feature_group(name=\"nasdaq\", version=1)\n",
    "print(nasdaq_fg)\n",
    "query = nasdaq_fg.select_all()\n",
    "feature_view = fs.get_or_create_feature_view(name=\"nasdaq\",\n",
    "                                  version=1,\n",
    "                                  description=\"Read from nasdaq dataset\",\n",
    "                                  labels=[\"close\"],\n",
    "                                  query=query)\n",
    "print(feature_view)\n",
    "\n",
    "df_nasdaq = nasdaq_fg.read()\n",
    "df_nasdaq = df_nasdaq.sort_values(by=\"date\")\n",
    "df_nasdaq.reset_index(drop=True,inplace=True)\n",
    "\n",
    "print(df_nasdaq)\n",
    "y_nasdaq = df_nasdaq['close'].values.reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# forecast setting\n",
    "n_forecast = 90  # length of output sequences (forecast period)\n",
    "n_lookback = 3*n_forecast  # length of input sequences (lookback period, should be 3 times longer than forecast period)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# Model Training for nasdaq\n",
    "scaler_nasdaq = MinMaxScaler(feature_range=(0, 1))\n",
    "scaler_nasdaq = scaler_nasdaq.fit(y_nasdaq)\n",
    "y_nasdaq = scaler_nasdaq.transform(y_nasdaq)\n",
    "\n",
    "X_nasdaq = []\n",
    "Y_nasdaq = []\n",
    "\n",
    "for i in range(n_lookback, len(y_nasdaq) - n_forecast + 1):\n",
    "    X_nasdaq.append(y_nasdaq[i - n_lookback: i])\n",
    "    Y_nasdaq.append(y_nasdaq[i: i + n_forecast])\n",
    "\n",
    "X_nasdaq = np.array(X_nasdaq)\n",
    "Y_nasdaq = np.array(Y_nasdaq)\n",
    "print(X_nasdaq.shape, Y_nasdaq.shape)\n",
    "\n",
    "# fit / train the model\n",
    "model_nasdaq = Sequential()\n",
    "model_nasdaq.add(LSTM(units=128, return_sequences=True, input_shape=(n_lookback, 1)))\n",
    "model_nasdaq.add(LSTM(units=64, return_sequences=True))\n",
    "model_nasdaq.add(LSTM(units=64, return_sequences=False))\n",
    "model_nasdaq.add(Dense(n_forecast))\n",
    "model_nasdaq.compile(loss='mean_squared_error', optimizer='adam')\n",
    "model_nasdaq.fit(X_nasdaq, Y_nasdaq, epochs=20, batch_size=32, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# We will now upload our model to the Hopsworks Model Registry. First get an object for the model registry.\n",
    "mr = project.get_model_registry()\n",
    "model_dir=\"nasdaq_model\"\n",
    "if os.path.isdir(model_dir) == False:\n",
    "    os.mkdir(model_dir)\n",
    "\n",
    "# Save both our model and the confusion matrix to 'model_dir', whose contents will be uploaded to the model registry\n",
    "joblib.dump(model_nasdaq, model_dir + \"/nasdaq_model.pkl\")\n",
    "# fig.savefig(model_dir + \"/wine_confusion_matrix.png\")\n",
    "\n",
    "# # Create an entry in the model registry that includes the model's name, desc, metrics\n",
    "nasdaq_model = mr.python.create_model(\n",
    "    name=\"nasdaq_model\",\n",
    "    # metrics={\"accuracy\" : metrics['accuracy']},\n",
    "    # model_schema=model_schema,\n",
    "    description=\"nasdaq Predictor\"\n",
    ")\n",
    "\n",
    "# Upload the model to the model registry, including all files in 'model_dir'\n",
    "nasdaq_model.save(model_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}