3_Normalization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "3. Normalization.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "BaQAWW8H9K4G",
        "I65xTMDV9yq-",
        "F0penUem_4bc"
      ],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMNNPrVg+55S12chw7znxvN"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BaQAWW8H9K4G",
        "colab_type": "text"
      },
      "source": [
        "# Case Folding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rQDzYrUjuaau",
        "colab_type": "text"
      },
      "source": [
        "Generalisasi kalimat menjadi huruf kecil atau besar semua"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W4TMBpdP86JG",
        "colab_type": "code",
        "outputId": "5052d0f8-e288-4775-b7ce-13a5feeb38c2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# merubah ke huruf kecil\n",
        "kalimat_1 = \"Halo, Hari INI kita belajar teXt processing menggunakan PYTHON\"\n",
        "kalimat_1 = kalimat_1.lower()\n",
        "print(kalimat_1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "halo, hari ini kita belajar text processing menggunakan python\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-HTDiyNIDhwI",
        "colab_type": "code",
        "outputId": "6b942735-adc1-4ad6-f9d8-e5d2089a3b5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# merubah ke huruf besar\n",
        "kalimat_2 = \"Halo, Hari INI kita belajar teXt processing menggunakan PYTHON\"\n",
        "kalimat_2 = kalimat_2.upper()\n",
        "print(kalimat_2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "HALO, HARI INI KITA BELAJAR TEXT PROCESSING MENGGUNAKAN PYTHON\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3xg98s1AEZu_",
        "colab_type": "code",
        "outputId": "f0a5afe4-6911-45fa-cae0-ea2db04f187a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# menggunakan fungsi pada python\n",
        "def rubahHuruf(inisial_argument):\n",
        "    kalimat_baru = inisial_argument.lower()   # merubah menjadi lowercase    \n",
        "    return kalimat_baru\n",
        "\n",
        "kalimat_3 = \"Halo, Hari INI kita belajar teXt processing menggunakan PYTHON\"\n",
        "rubahHuruf(kalimat_3)   # memanggil fungsi \"rubahHuruf\" dengan mengirimkan parameter variabel \"kalimat\""
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'halo, hari ini kita belajar text processing menggunakan python'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I65xTMDV9yq-",
        "colab_type": "text"
      },
      "source": [
        "# Remove Punctuation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yov42Yfnumbo",
        "colab_type": "text"
      },
      "source": [
        "Menghapus simbol dan karakter pada kalimat"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HO3HXkGT9u8l",
        "colab_type": "code",
        "outputId": "7432c7a1-a558-4457-85a7-85cd39bdc9e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# menggunakan string\n",
        "import string\n",
        "\n",
        "kalimat_4 =  \"kit@ coba menghapu$ , . & (simbol karakter)\"\n",
        "\n",
        "hasil_1 = kalimat_4.translate(str.maketrans({key: None for key in string.punctuation}))\n",
        "print(hasil_1)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "kit coba menghapu    simbol karakter\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "trfPOCgV_UUQ",
        "colab_type": "code",
        "outputId": "32208ee4-fa88-46ec-a353-a7482de3cd4d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# menggunakan regex\n",
        "import re\n",
        "\n",
        "hasil_2 = re.sub(r'[^\\w\\s]','', kalimat_4)\n",
        "print(hasil_2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "kit coba menghapu    simbol karakter\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pr3WejXHI83b",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "Contoh lengkap lihat disini:\n",
        "\n",
        "https://colab.research.google.com/drive/10DeWW4yOpopYXpfYnPEfz-Epdezz7NFM"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F0penUem_4bc",
        "colab_type": "text"
      },
      "source": [
        "# Remove White Space"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yI257GgWuyEs",
        "colab_type": "text"
      },
      "source": [
        "Menghapus spasi atau tab pada kalimat"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aNXjt9le_7So",
        "colab_type": "code",
        "outputId": "c07cc90f-8aa2-4f68-d552-a37c83277da4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "kalimat_5 = \"masih     ingin terus           belajar\"\n",
        "print(\"Before:\", kalimat_5)\n",
        "\n",
        "kalimat_6 = re.sub('[\\s]+', ' ', kalimat_5)   # hilangkan spasi/tab\n",
        "print(\"After :\" ,kalimat_6)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Before: masih     ingin terus           belajar\n",
            "After : masih ingin terus belajar\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YNp1qFhzBqcD",
        "colab_type": "code",
        "outputId": "674ae03e-2ca0-4889-92df-55dd756cb045",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "kalimat_7 = 'Hi! @Mukidi, apa kabar? #sapa_Pagi.'\n",
        "kalimat_7 = re.sub(r'[^\\w]',' ',kalimat_7)   # hilangkan special char (!) (@) (,) (?) (#) (.)\n",
        "print(\"Before :\" ,kalimat_7)\n",
        "\n",
        "kalimat_8 = re.sub(r'\\s\\s+',' ',kalimat_7)  # kurangi multi whitespace\n",
        "print(\"After  :\" ,kalimat_8)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Before : Hi   Mukidi  apa kabar   sapa_Pagi \n",
            "After  : Hi Mukidi apa kabar sapa_Pagi \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Sq8BFZuDCHa",
        "colab_type": "code",
        "outputId": "268dd609-b1c0-41fa-8634-66ed7f385206",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# contoh lain\n",
        "kalimat_9 = \"\\t contoh whitespace berikutnya\\t\"\n",
        "print(\"Before :\" ,kalimat_9)\n",
        "\n",
        "kalimat_10 = kalimat_9.strip()\n",
        "print(\"After  :\" ,kalimat_10)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Before : \t contoh whitespace berikutnya\t\n",
            "After  : contoh whitespace berikutnya\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}