linear_regression_from_scratch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "linear-regression-from-scratch.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "tEfIOWonqpIw"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = torch.FloatTensor([[1], [2], [3]])\n",
        "y_train = torch.FloatTensor([[1], [2], [3]])"
      ],
      "metadata": {
        "id": "tMUqMXmnq650"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(x_train)\n",
        "print(x_train.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bBGHEng-rIxJ",
        "outputId": "b93bf9f4-23c3-4ac7-f7bc-4c498a8a1b7e"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[1.],\n",
            "        [2.],\n",
            "        [3.]])\n",
            "torch.Size([3, 1])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "W = torch.zeros(1, requires_grad=True)\n",
        "print(W)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_yBakF49rTsv",
        "outputId": "0060117b-895d-46e3-b6f3-705dab5d3273"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([0.], requires_grad=True)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "b = torch.zeros(1, requires_grad=True)\n",
        "print(b)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vKSKnIITrba9",
        "outputId": "ed84c5e8-583b-4df4-c9ce-14e4a22e817a"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([0.], requires_grad=True)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "hypothesis = x_train * W + b\n",
        "print(hypothesis)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KAe5HK_8rhf4",
        "outputId": "d9402273-cb84-4edd-ba05-f9e57000b8f3"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.]], grad_fn=<AddBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(hypothesis)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S8AzhYuJryrM",
        "outputId": "0951ff2c-5744-46e0-fb75-dbb0e54f9c49"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.],\n",
            "        [0.],\n",
            "        [0.]], grad_fn=<AddBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VgLDAFkJr2ZH",
        "outputId": "448b4c90-528c-41b3-a2b6-1b9d87604021"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[1.],\n",
            "        [2.],\n",
            "        [3.]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(hypothesis - y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mw2JmFY_r6XT",
        "outputId": "811cf4fb-7131-454e-d2ce-822a0ada5935"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[-1.],\n",
            "        [-2.],\n",
            "        [-3.]], grad_fn=<SubBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print((hypothesis - y_train) ** 2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QPVrWCzasEr7",
        "outputId": "0c2d6edd-e1c1-45da-9f7d-2510ec30e6cd"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[1.],\n",
            "        [4.],\n",
            "        [9.]], grad_fn=<PowBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cost = torch.mean((hypothesis - y_train) ** 2)\n",
        "print(cost)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YMTFUUBVsULF",
        "outputId": "b9162997-75a1-4a14-acca-4683003d1413"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(4.6667, grad_fn=<MeanBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer = optim.SGD([W, b], lr=0.01)"
      ],
      "metadata": {
        "id": "lpDNlr7HsrHz"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer.zero_grad()\n",
        "cost.backward()\n",
        "optimizer.step()"
      ],
      "metadata": {
        "id": "KXaoTDBqs0N7"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(W)\n",
        "print(b)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hOck5321s9XN",
        "outputId": "2d035a4c-8e37-4a51-9ef3-69cb408b841b"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([0.0933], requires_grad=True)\n",
            "tensor([0.0400], requires_grad=True)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "hypothesis = x_train * W + b\n",
        "print(hypothesis)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gi9oy6I3tCT8",
        "outputId": "54ed2f99-add8-45a4-9b17-4dd858b112a1"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.1333],\n",
            "        [0.2267],\n",
            "        [0.3200]], grad_fn=<AddBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cost = torch.mean((hypothesis - y_train) ** 2)\n",
        "print(cost)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nz83NxX8tPOQ",
        "outputId": "20a60e42-c9b4-4731-d846-12e34b76325c"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(3.6927, grad_fn=<MeanBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = torch.FloatTensor([[1], [2], [3]])\n",
        "y_tain = torch.FloatTensor([[1], [2], [3]])\n",
        "W = torch.zeros(1, requires_grad=True)\n",
        "b = torch.zeros(1, requires_grad=True)\n",
        "optimizer = optim.SGD([W, b], lr=0.01)\n",
        "\n",
        "nb_epochs = 1000\n",
        "for epoch in range(nb_epochs + 1):\n",
        "    hypothesis = x_train * W + b\n",
        "\n",
        "    cost = torch.mean((hypothesis - y_train) ** 2)\n",
        "\n",
        "    optimizer.zero_grad()\n",
        "    cost.backward()\n",
        "    optimizer.step()\n",
        "    if epoch % 100 == 0:\n",
        "        print('Epoch {:4d}/{} W: {:.3f}, b: {:.3f} Cost: {:.6f}'.format(\n",
        "            epoch, nb_epochs, W.item(), b.item(), cost.item()\n",
        "        ))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3P6wZRdXtaqi",
        "outputId": "4088f38e-e422-4fd7-89b6-6c81727608c0"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch    0/1000 W: 0.093, b: 0.040 Cost: 4.666667\n",
            "Epoch  100/1000 W: 0.873, b: 0.289 Cost: 0.012043\n",
            "Epoch  200/1000 W: 0.900, b: 0.227 Cost: 0.007442\n",
            "Epoch  300/1000 W: 0.921, b: 0.179 Cost: 0.004598\n",
            "Epoch  400/1000 W: 0.938, b: 0.140 Cost: 0.002842\n",
            "Epoch  500/1000 W: 0.951, b: 0.110 Cost: 0.001756\n",
            "Epoch  600/1000 W: 0.962, b: 0.087 Cost: 0.001085\n",
            "Epoch  700/1000 W: 0.970, b: 0.068 Cost: 0.000670\n",
            "Epoch  800/1000 W: 0.976, b: 0.054 Cost: 0.000414\n",
            "Epoch  900/1000 W: 0.981, b: 0.042 Cost: 0.000256\n",
            "Epoch 1000/1000 W: 0.985, b: 0.033 Cost: 0.000158\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_train = torch.FloatTensor([[1], [2], [3]])\n",
        "y_train = torch.FloatTensor([[1], [2], [3]])"
      ],
      "metadata": {
        "id": "SqlVo9uUvKNr"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, you just need to create a linear regression model. Basically, all models in PyTorch are created by inheriting the provided nn.Module.\n"
      ],
      "metadata": {
        "id": "v73s-LsOwDED"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class LinearRegressionModel(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.linear = nn.Linear(1, 1)\n",
        "    def forward(self, x):\n",
        "        return self.linear(x)"
      ],
      "metadata": {
        "id": "5mUXwQacvVDE"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        ""
      ],
      "metadata": {
        "id": "Tr__DnB2wWAd"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The __init__ of the model defines the layers to be used. Since we are creating a linear regression model here, we will use nn.Linear . And forward tells how the model computes the output from the input."
      ],
      "metadata": {
        "id": "hHXncoIUv9Ip"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = LinearRegressionModel()"
      ],
      "metadata": {
        "id": "MmVBGATev2yc"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let's create a model and get the predicted value H(x)"
      ],
      "metadata": {
        "id": "RZXQGKs7wXXf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "hypothesis = model(x_train)"
      ],
      "metadata": {
        "id": "rXsNyByFwYwQ"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(hypothesis)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hoLH4y-3wrkn",
        "outputId": "d2bedbe8-4ace-493a-9d4b-7e7f2d26e110"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.4784],\n",
            "        [1.2280],\n",
            "        [1.9776]], grad_fn=<AddmmBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let's find the cost with the mean squared error (MSE). MSE is also provided by default in PyTorch."
      ],
      "metadata": {
        "id": "u2JWSRV9w5DO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(hypothesis)\n",
        "print(y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ufaimibQw6bo",
        "outputId": "76507e3d-aa33-4c59-ba6a-07a6eda12c65"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.4784],\n",
            "        [1.2280],\n",
            "        [1.9776]], grad_fn=<AddmmBackward0>)\n",
            "tensor([[1.],\n",
            "        [2.],\n",
            "        [3.]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Try reducing the cost by changing W, b of H(x) using the last given cost. At this point you can use one of the optimizers from PyTorch's torch.optim ."
      ],
      "metadata": {
        "id": "Xdus1298xkx3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "cost = F.mse_loss(hypothesis, y_train)"
      ],
      "metadata": {
        "id": "xNfoAmT8xC4J"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(cost)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CneSVVavxLa5",
        "outputId": "3f42afca-493c-4bed-f414-957eef272002"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(0.6378, grad_fn=<MseLossBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer = optim.SGD(model.parameters(), lr=0.01)"
      ],
      "metadata": {
        "id": "M2rRcRCZxQqq"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "optimizer.zero_grad()\n",
        "cost.backward()\n",
        "optimizer.step()"
      ],
      "metadata": {
        "id": "oGnwFkxzxW-Z"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now that we understand the Linear Regression code, let's actually run the code and fit it."
      ],
      "metadata": {
        "id": "DJaFNkaAxyAp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# data\n",
        "x_train = torch.FloatTensor([[1], [2], [3]])\n",
        "y_tain = torch.FloatTensor([[1], [2], [3]])\n",
        "# model initialization\n",
        "model = LinearRegressionModel()\n",
        "# optimizer settings\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)\n",
        "\n",
        "nb_epochs = 1000\n",
        "for epoch in range(nb_epochs + 1):\n",
        "    # hypothesis H(x) calculate\n",
        "    prediction = model(x_train)\n",
        "    # cost calculation\n",
        "    cost = F.mse_loss(prediction, y_train)\n",
        "    # improve H(x) by cost\n",
        "    optimizer.zero_grad()\n",
        "    cost.backward()\n",
        "    optimizer.step()\n",
        "    # log output every 100 times\n",
        "    if epoch % 100 == 0:\n",
        "        params = list(model.parameters())\n",
        "        W = params[0].item()\n",
        "        b = params[1].item()\n",
        "        print('Epoch {:4d}/{} W: {:.3f}, b: {:.3f} Cost: {:.6}'.format(\n",
        "            epoch, nb_epochs, W, b, cost.item()\n",
        "        ))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xB24TePEyRV5",
        "outputId": "003a909c-7a45-46b1-b038-46c718017527"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch    0/1000 W: 0.859, b: 0.747 Cost: 0.273696\n",
            "Epoch  100/1000 W: 0.766, b: 0.533 Cost: 0.0408869\n",
            "Epoch  200/1000 W: 0.816, b: 0.419 Cost: 0.0252656\n",
            "Epoch  300/1000 W: 0.855, b: 0.329 Cost: 0.0156126\n",
            "Epoch  400/1000 W: 0.886, b: 0.259 Cost: 0.00964765\n",
            "Epoch  500/1000 W: 0.911, b: 0.203 Cost: 0.00596166\n",
            "Epoch  600/1000 W: 0.930, b: 0.160 Cost: 0.00368394\n",
            "Epoch  700/1000 W: 0.945, b: 0.126 Cost: 0.00227645\n",
            "Epoch  800/1000 W: 0.957, b: 0.099 Cost: 0.00140671\n",
            "Epoch  900/1000 W: 0.966, b: 0.078 Cost: 0.000869259\n",
            "Epoch 1000/1000 W: 0.973, b: 0.061 Cost: 0.000537152\n"
          ]
        }
      ]
    }
  ]
}