cifar10_single_layer.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kFS4HYpEbT-b"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import math\n",
        "from sklearn.metrics import accuracy_score \n",
        "from tqdm import tqdm\n",
        "from tensorflow import keras\n",
        "import tensorflow as tf\n",
        "from keras.datasets import cifar10"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "np2FtZAvbT-d"
      },
      "outputs": [],
      "source": [
        "(x_train, y_train), (x_test, y_test) = cifar10.load_data()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z3GYMltnbT-d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e2f862c3-e082-4b0a-942b-2299c95a564a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 32, 32, 3)\n",
            "(50000, 1)\n"
          ]
        }
      ],
      "source": [
        "print(x_train.shape)\n",
        "print(y_train.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DMHmX9fHbT-e"
      },
      "outputs": [],
      "source": [
        "x_train = x_train.astype('float128')\n",
        "x_train = x_train/255\n",
        "x_test = x_test.astype('float128')\n",
        "x_test = x_test/255"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MWSFAWWubT-e"
      },
      "outputs": [],
      "source": [
        "#Code to import the data, but irrelevant if using keras\n",
        "\n",
        "\"\"\"\n",
        "types = ['batches.meta', 'data_batch_1', 'data_batch_2', 'data_batch_3', 'data_batch_4', 'data_batch_5', 'test_batch']\n",
        "set_data = [0,1,2,3,4,5,6]\n",
        "\n",
        "\n",
        "# data_dict = {}\n",
        "\n",
        "# for i in range(7):\n",
        "#     data_dict[types[i]] = set_data[i]\n",
        "\n",
        "# print(type(data_dict))\n",
        "# print(data_dict['batches.meta'])\n",
        "\n",
        "#Alternate way\n",
        "batch_meta = set_data[0]\n",
        "data_batch1 = set_data[1]\n",
        "data_batch2 = set_data[2]\n",
        "data_batch3 = set_data[3]\n",
        "data_batch4 = set_data[4]\n",
        "data_batch5 = set_data[5]\n",
        "test_batch = set_data[6]\n",
        "\"\"\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8LTV-M0RbT-e"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import numpy as np\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from keras.utils import np_utils\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "y_train = np_utils.to_categorical(y_train) \n",
        "y_test = np_utils.to_categorical(y_test) \n",
        "num_classes = y_test.shape[1]\n"
      ],
      "metadata": {
        "id": "h4r0TB9g5x8W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(y_train.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JgJEL2we6SPz",
        "outputId": "a053434d-bde3-42c8-ab2c-fecbeacc1f73"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2_RgF0psbT-f"
      },
      "outputs": [],
      "source": [
        "#One hot encoding function=\n",
        "\n",
        "def one_hot(vec, vals = 10):\n",
        "    n = len(vec)\n",
        "    out = np.zeros((n,vals))\n",
        "    out[range(n), vec] = 1\n",
        "    return out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hvIODgo3bT-f"
      },
      "outputs": [],
      "source": [
        "class CifarClass():    #Creating the class for the relevant methods\n",
        "    \n",
        "    #Tried to implement early stopping upon overfitting but it didnt really work well\n",
        "    def __init__(self, learning_rate = 0.1, max_iter = 1000, minimum_step_size = 0.001, early_stopping = False, n=20):\n",
        "        self.i = 0\n",
        "        self.training_images = None\n",
        "        self.training_labels = None \n",
        "        self.testing_images = None\n",
        "        self.testing_labels = None\n",
        "        self.__learning_rate = learning_rate\n",
        "        self.__max_iter = max_iter\n",
        "        self.__minimum_step_size = minimum_step_size\n",
        "        self.__early_stopping = early_stopping\n",
        "        self.__n = n\n",
        "\n",
        "\n",
        "    def setup_of_images(self):\n",
        "        self.training_images = x_train\n",
        "        train_len = len(self.training_images)\n",
        "        self.training_labels = y_train\n",
        "\n",
        "        self.testing_images = x_test\n",
        "        test_len = len(self.testing_images)\n",
        "        self.testing_labels = y_test\n",
        "\n",
        "    #calculating the score for x for a given class k\n",
        "    def __calculate_score(self, k, x):\n",
        "        weight = self.__weights[k]\n",
        "        return np.exp(x.dot(weight))\n",
        "\n",
        "    def __calculate_softmax(self, k, x):\n",
        "        summation = 0\n",
        "        for i in range(self.__class_count):\n",
        "            summation += self.__calculate_score(i,x)\n",
        "\n",
        "         \n",
        "        return self.__calculate_score(k,x)/summation\n",
        "\n",
        "    def __cross_entropy_gradient(self, k):\n",
        "        sum = 0\n",
        "        #iterating over all the images in the set\n",
        "        for i in range (len(self.__x)):\n",
        "            \n",
        "            sum += ((self.__calculate_softmax(k,self.__x[i]) - self.__y[i][k]) * self.__x[i]) #the gradient formula\n",
        "        return sum / len(self.__x) #returning the average of the all the difference for each image\n",
        "    \n",
        "    def __new_weights(self, k): \n",
        "        #no change if class is in exclusion list\n",
        "        if k in self.__exclusion_list:\n",
        "            return self.__weights[k]\n",
        "\n",
        "        step_size = self.__cross_entropy_gradient(k)*self.__learning_rate\n",
        "\n",
        "        #code to add classes to excleusion list\n",
        "        # if np.amax(step_size) <= self.__minimum_step_size:\n",
        "        #     self.__exclusion_list.append(k)\n",
        "        \n",
        "        return self.__weights[k] - step_size\n",
        "    \n",
        "    def train(self, X, y):\n",
        "        \n",
        "        self.__class_count = 10\n",
        "        self.__weights = np.random.rand(self.__class_count, X.shape[2]) #initialize random weights\n",
        "        self.__exclusion_list = []\n",
        "        max_acc = -1\n",
        "        n=0\n",
        "        print('initial weight:', cc.__weights)\n",
        "\n",
        "        for j in tqdm(range(1)):                                            #set number of epochs\n",
        "            for i in range(3125):                                           #iterating over all the batches \n",
        "                if self.__early_stopping:                                   #ignore the if, move to the else\n",
        "                    split_index = int(len(X) * 0.1)\n",
        "                    self.__x = X[split_index:]\n",
        "                    self.__y = y[split_index:]\n",
        "                    self.__x_val = X[:split_index]\n",
        "                    self.__y_val = y[:split_index]\n",
        "                else:\n",
        "                    self.__x = X[i]                      #passes the i th batch, i is the batch index\n",
        "                    self.__y = y[i]\n",
        "                    \n",
        "\n",
        "                for j in range(self.__class_count):                #sets new weighs for every class index for each batch\n",
        "                    self.__weights[j] = self.__new_weights(j)\n",
        "        \n",
        "        print('final weight:', self.__weights)\n",
        "        \n",
        "    \n",
        "    #returns the prediccted label in one hot encoded form\n",
        "    def predict(self, X):\n",
        "        y= np.zeros((len(X), self.__class_count))\n",
        "        for i in tqdm(range(len(X))):\n",
        "            max_score_idx = 0\n",
        "            max_score = 0\n",
        "            for j in range(self.__class_count):\n",
        "                score = self.__calculate_softmax(j, X[i])\n",
        "                # print('score', j, 'is', score)\n",
        "                if score > max_score:\n",
        "                    max_score = score\n",
        "                    max_score_idx = j\n",
        "                    \n",
        "            # print('loop over')        \n",
        "            y[i][max_score_idx] = 1\n",
        "        return y\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DN_Nx004bT-g",
        "outputId": "83679ea2-e1d2-464e-bd2b-c27d6a119e44",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 32, 32, 3)\n",
            "(10000, 32, 32, 3)\n"
          ]
        }
      ],
      "source": [
        "cc = CifarClass() \n",
        "cc.setup_of_images()\n",
        "\n",
        "print (cc.training_images.shape)\n",
        "print(cc.testing_images.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jlsj8EoRbT-g",
        "outputId": "4e9930c0-9730-4915-e372-0d59761a45e8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(50000, 3072)\n",
            "final training image shape: (3125, 16, 3072)\n",
            "final training labels shape (3125, 16, 10)\n",
            "final testing images shape (10000, 3072)\n",
            "[0. 0. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
            "(10,)\n",
            "1.0\n"
          ]
        }
      ],
      "source": [
        "#flattening\n",
        "\n",
        "cc.training_images = cc.training_images.reshape(50000, 3072)\n",
        "\n",
        "cc.training_images = cc.training_images - np.full((50000, 3072), 0.5)     #biasing, but sorta random\n",
        "\n",
        "print(cc.training_images.shape)\n",
        "cc.training_images = cc.training_images.reshape(3125, 16, 3072)\n",
        "print('final training image shape:', cc.training_images.shape)\n",
        "cc.training_labels = cc.training_labels.reshape(3125, 16, 10)         \n",
        "print('final training labels shape', cc.training_labels.shape)\n",
        "cc.testing_images = cc.testing_images.reshape(10000, 3072)\n",
        "\n",
        "\n",
        "cc.testing_images = cc.testing_images - np.full((10000, 3072), 0.5)      #biasing, but sorta random\n",
        "print('final testing images shape', cc.testing_images.shape)\n",
        "\n",
        "print(cc.training_labels[23][15])\n",
        "\n",
        "#more testing\n",
        "temp = cc.training_labels[23]\n",
        "print(temp[15].shape)\n",
        "print(temp[15][2])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#plt.imshow(cc.training_images[344])"
      ],
      "metadata": {
        "id": "0je8WmnNuuWk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ccVrGvWzbT-h"
      },
      "outputs": [],
      "source": [
        "def main():\n",
        "   \n",
        "    X_train = cc.training_images \n",
        "    y_train = cc.training_labels\n",
        "\n",
        "    X_test = cc.testing_images\n",
        "    y_test = cc.testing_labels\n",
        "\n",
        "    cc.train(X_train, y_train)\n",
        "\n",
        "    predictions = cc.predict(X_test)\n",
        "\n",
        "    print(accuracy_score(y_test, predictions))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rrhJSXA_bT-h",
        "outputId": "b6303089-8b85-4241-83ab-e0eae20e193a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "initial weight: [[0.06830801 0.88681279 0.7191182  ... 0.46313683 0.30543594 0.47042983]\n",
            " [0.30481692 0.73741243 0.82097561 ... 0.9280699  0.22331612 0.74346521]\n",
            " [0.92981987 0.10301471 0.06738388 ... 0.96181257 0.09533702 0.72596458]\n",
            " ...\n",
            " [0.61403889 0.29114947 0.16441738 ... 0.08786888 0.71374871 0.86955177]\n",
            " [0.9723647  0.98188309 0.77254404 ... 0.21552337 0.6104757  0.34480281]\n",
            " [0.51453121 0.4941341  0.95692895 ... 0.52002672 0.68529032 0.04059785]]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1/1 [01:36<00:00, 96.65s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "final weight: [[0.12606849 0.97737653 0.86487803 ... 0.45526008 0.3035546  0.43451697]\n",
            " [0.24100936 0.67656363 0.85258804 ... 0.87489099 0.21958882 0.77740749]\n",
            " [0.94815486 0.18356436 0.03243772 ... 0.86497532 0.04797849 0.65915847]\n",
            " ...\n",
            " [0.70189521 0.3289739  0.27717301 ... 0.24162489 0.80927426 0.83380171]\n",
            " [0.93825494 0.91478896 0.81196238 ... 0.15095464 0.65665892 0.59053502]\n",
            " [0.46105868 0.5245161  1.05285628 ... 0.62402408 0.79807954 0.23622379]]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 10000/10000 [00:17<00:00, 585.63it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.3115\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "if __name__ == \"__main__\":\n",
        "    main()\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6VhRYkgFbT-h"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.7.13 ('tf')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.13"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "28031d7d25694f73766452af3c8f3519c0d1811094a598bb3ab04b29b8086b2e"
      }
    },
    "colab": {
      "name": "Copy of cifar-10-solve.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}