Demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.5"
    },
    "colab": {
      "name": "DSL - GSC Clustering Demo.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "303c2910322c4e1791a80a7ca73b3035": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fd5bbb5e4dcb4316ad3e46d80864c418",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8daace4c478e4690aa8841026356fd2d",
              "IPY_MODEL_d0c8fd4bcd554e5da1ebb3d72cedcaf4"
            ]
          }
        },
        "fd5bbb5e4dcb4316ad3e46d80864c418": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8daace4c478e4690aa8841026356fd2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bdbe08e3d97e4fd68ccd1c0ce960ebaf",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 596,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 596,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_539a17cabd25455ab4fd42a7fc250acf"
          }
        },
        "d0c8fd4bcd554e5da1ebb3d72cedcaf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6bc3d3525a1e4afbbdffab0a56efaef6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 596/596 [00:00&lt;00:00, 17.3kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_338f1b39d8414660902323b04a316a29"
          }
        },
        "bdbe08e3d97e4fd68ccd1c0ce960ebaf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "539a17cabd25455ab4fd42a7fc250acf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6bc3d3525a1e4afbbdffab0a56efaef6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "338f1b39d8414660902323b04a316a29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d5bd80f520034d23a2f784e08cff0389": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ea39bcc6107e485eb08536124eab2faa",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e491ebb4e62841b99ae870261e68e248",
              "IPY_MODEL_f0978f9138984fee8ea02e3786ec1886"
            ]
          }
        },
        "ea39bcc6107e485eb08536124eab2faa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e491ebb4e62841b99ae870261e68e248": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c79c3e83545b4ae4b36e5a04835ee8e1",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 445032417,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 445032417,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d21b3afad66942ef97730567606120f0"
          }
        },
        "f0978f9138984fee8ea02e3786ec1886": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3bf7a4f5d68047b2825fd0c2a0e1bdea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 445M/445M [00:11&lt;00:00, 37.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9816d07fd02a4ce1a31a7b49fe0b1915"
          }
        },
        "c79c3e83545b4ae4b36e5a04835ee8e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d21b3afad66942ef97730567606120f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3bf7a4f5d68047b2825fd0c2a0e1bdea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9816d07fd02a4ce1a31a7b49fe0b1915": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a107ab7e69f24d899e3dd3c307e74c20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_91bc7ac737b140e2923ea29f42747bf4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_555e7bd2851340a7bead02aeb37d9152",
              "IPY_MODEL_d27b51b360fa426dade096dbe5f785d1"
            ]
          }
        },
        "91bc7ac737b140e2923ea29f42747bf4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "555e7bd2851340a7bead02aeb37d9152": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b8d247fb311646618b6ef04ff4261b11",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 810912,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 810912,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_48713f95c8f245ce865539a86e052fd8"
          }
        },
        "d27b51b360fa426dade096dbe5f785d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_da225c952b104f8a81199ae5920e487e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 811k/811k [00:00&lt;00:00, 2.56MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b34ad090026741bd841270725d6eaab8"
          }
        },
        "b8d247fb311646618b6ef04ff4261b11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "48713f95c8f245ce865539a86e052fd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da225c952b104f8a81199ae5920e487e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b34ad090026741bd841270725d6eaab8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bv7R5x0XwNIo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Licensed under the Apache License, Version 2.0 (the \"License\")\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "\n",
        " https://www.apache.org/licenses/LICENSE-2.0\n",
        "\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BeQquQKYb5Ds",
        "colab_type": "text"
      },
      "source": [
        "# GSC Clustering\n",
        "\n",
        "Based on the work of JR Oakes : https://github.com/jroakes/querycat\n",
        "\n",
        "Before starting, you need to use the best model from huggingface.io according to your language :\n",
        "\n",
        "https://huggingface.co/models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hbyUc23XFk1B",
        "colab_type": "code",
        "outputId": "9c000776-2bdf-460c-e1cf-e5104e5586e1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#@title Choose your model and your language\n",
        "\n",
        "# import spacy libs\n",
        "!python -m spacy download fr_core_news_sm\n",
        "!python -m spacy download de_core_news_sm\n",
        "!python -m spacy download it_core_news_sm\n",
        "!python -m spacy download es_core_news_sm\n",
        "!python -m spacy download nl_core_news_sm\n",
        "!python -m spacy download pt_core_news_sm\n",
        "\n",
        "from IPython.display import clear_output\n",
        "!git clone https://github.com/voltek62/querycat.git\n",
        "!pip install -r querycat/requirements_colab.txt\n",
        "\n",
        "TRANSFORMER_MODEL = \"camembert-base\" #@param {type:\"string\"}\n",
        "RANDOM_SEED = 42\n",
        "LANGUAGE = 'french' #@param {type:\"string\"} [\"english\", \"german\", \"french\", \"spanish\", \"portuguese\", \"italian\", \"dutch\"]\n",
        "\n",
        "import querycat\n",
        "querycat.init_lib(LANGUAGE)\n",
        "\n",
        "clear_output()\n",
        "print('All loaded.')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All loaded.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "azyvUd1RV5Kj",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "#### Search Console Data\n",
        "\n",
        "##### Getting a Google API Credential.\n",
        "\n",
        "1) Sign up for a new project in the Google APIs console at https://code.google.com/apis/console.\n",
        "\n",
        "2) Choose \"Credentials\" on the left-hand menu.  \n",
        "\n",
        "3) Choose \"Create Credentials\"\n",
        "\n",
        "4) Generate an API key for your application.\n",
        "\n",
        "5) Copy your client ID, client secret, and redirect URL into the next cell\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FNF7pFIxv04G",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "#@title Configure your access\n",
        "import httplib2\n",
        "\n",
        "from apiclient import errors\n",
        "from apiclient.discovery import build\n",
        "from oauth2client.client import OAuth2WebServerFlow\n",
        "\n",
        "# Copy your credentials from the console\n",
        "CLIENT_ID = 'XXXXXXXXX-5s8ll088odmhee5lusa14jhse6p4l5a4.apps.googleusercontent.com' #@param {type:\"string\"}\n",
        "CLIENT_SECRET = 'XXXXXXXX-Oihmqnq9mPd' #@param {type:\"string\"}\n",
        "\n",
        "# Check https://developers.google.com/webmaster-tools/search-console-api-original/v3/ for all available scopes\n",
        "OAUTH_SCOPE = 'https://www.googleapis.com/auth/webmasters.readonly'\n",
        "\n",
        "# Redirect URI for installed apps\n",
        "REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'\n",
        "\n",
        "# Run through the OAuth flow and retrieve credentials\n",
        "flow = OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, OAUTH_SCOPE, REDIRECT_URI)\n",
        "authorize_url = flow.step1_get_authorize_url()\n",
        "print('Go to the following link in your browser: ' + authorize_url)\n",
        "code = input('Enter verification code: ').strip()\n",
        "credentials = flow.step2_exchange(code)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ifRkQVGhv7RK",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "#@title Choose your projects\n",
        "\n",
        "# Create an httplib2.Http object and authorize it with our credentials\n",
        "http = httplib2.Http()\n",
        "http = credentials.authorize(http)\n",
        "\n",
        "webmasters_service = build('webmasters', 'v3', http=http)\n",
        "\n",
        "# Retrieve list of properties in account\n",
        "site_list = webmasters_service.sites().list().execute()\n",
        "\n",
        "# Filter for verified websites\n",
        "verified_sites_urls = [s['siteUrl'] for s in site_list['siteEntry']\n",
        "                       if s['permissionLevel'] != 'siteUnverifiedUser'\n",
        "                          and s['siteUrl'][:4] == 'http']\n",
        "\n",
        "# Print the URLs of all websites you are verified for.\n",
        "projects = []\n",
        "for site_url in verified_sites_urls:\n",
        "  #print(site_url)\n",
        "  projects.append(('{}'.format(site_url), site_url))\n",
        "\n",
        "# Create Selector\n",
        "import ipywidgets as widgets\n",
        "output = widgets.Output()\n",
        "\n",
        "dropdown_purpose = widgets.Dropdown(options = projects, description=\"Site: \")\n",
        "\n",
        "def dropdown_project_eventhandler(change):\n",
        "    output.clear_output()    \n",
        "    with output:\n",
        "        display(projects)\n",
        "\n",
        "dropdown_purpose.observe(dropdown_project_eventhandler, names='value')\n",
        "\n",
        "display(dropdown_purpose)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hwBqLKrfwTNw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Get your data\n",
        "#@markdown The script cut the period into two intervals for comparing the current against the previous interval\n",
        "import pandas as pd\n",
        "from dateutil import rrule\n",
        "from datetime import datetime, timedelta\n",
        "\n",
        "PERIOD = 30 #@param {type:\"integer\"}\n",
        "\n",
        "url = \"\"\n",
        "if(len(dropdown_purpose.value)==0):\n",
        "  print(\"no Site detected\")\n",
        "else:\n",
        "  url = dropdown_purpose.value\n",
        "\n",
        "##################\n",
        "#TODO : add pagination\n",
        "\n",
        "now = datetime.now()\n",
        "before = now - timedelta(days=PERIOD)\n",
        "before_date = before.strftime('%Y-%m-%d')\n",
        "\n",
        "end_date = now - timedelta(days=1)\n",
        "\n",
        "# init\n",
        "DF = pd.DataFrame()\n",
        "\n",
        "for dt in rrule.rrule(rrule.DAILY, dtstart=before, \n",
        "                      until=end_date):\n",
        "\n",
        "  #if (before_date!=dt.strftime('%Y-%m-%d')):\n",
        "  print(before_date+' to '+dt.strftime('%Y-%m-%d'))\n",
        "\n",
        "  request = {\n",
        "    'startDate': before_date,    \n",
        "    'endDate': dt.strftime('%Y-%m-%d'),    \n",
        "    'dimensions': [\"date\",\"query\"], # ,\"page\"\n",
        "    'searchType': \"web\",      \n",
        "    'rowLimit': 25000     \n",
        "  }\n",
        "\n",
        "  response =  webmasters_service.searchanalytics().query(siteUrl=url, body=request).execute()\n",
        "\n",
        "  dfGSC = pd.DataFrame.from_dict(response['rows'], orient='columns')\n",
        "\n",
        "  dfGSC[[\"date\",\"query\"]] = pd.DataFrame(dfGSC[\"keys\"].values.tolist()) # ,\"page\" \n",
        "\n",
        "  # drop Key column\n",
        "  #print(dfGSC[\"keys\"])\n",
        "  dfGSC =  dfGSC.drop(columns=['keys'])\n",
        "  DF_new = dfGSC[['date','query','clicks']]\n",
        "  print(len(DF_new))\n",
        "\n",
        "  DF = DF.append(DF_new, ignore_index=True, verify_integrity=True )\n",
        "  #print(DF)\n",
        "\n",
        "  before_date=dt.strftime('%Y-%m-%d')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AuXClxd1qtzS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Prepare Comparative data\n",
        "import numpy as np\n",
        "\n",
        "DT = DF.copy()\n",
        "\n",
        "DT['date'] = pd.to_datetime(DT['date'])\n",
        "\n",
        "# cut dataset into 2 parts\n",
        "interval = str(round(PERIOD/2))+\"D\"\n",
        "\n",
        "# group by interval\n",
        "DT = DT.groupby(['query', pd.Grouper(key='date',freq=interval)])['clicks'].sum().reset_index()\n",
        "\n",
        "# sum by date\n",
        "DT = DT.groupby(['query','date'])['clicks'].sum()\n",
        "DT.columns = ['query', 'date', 'clicks']\n",
        "DT = DT.reset_index()\n",
        "\n",
        "DT = pd.pivot_table(DT, values= 'clicks', index=['query'],columns=['date'])\n",
        "\n",
        "# replace NaN by 0\n",
        "DT[np.isnan(DT)] = 0\n",
        "DT = DT.reset_index()\t      \n",
        "DT.columns = ['query', 'previous', 'last']\n",
        "\n",
        "# Sort by the `diff` columnn\n",
        "DT.sort_values(by='last', ascending=False, inplace=True)\n",
        "\n",
        "DT"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "olh1wXFaUkCG",
        "colab_type": "text"
      },
      "source": [
        "##  Categorize the queries\n",
        "\n",
        "The `querycat.Categorize` function, takes the following arguments.\n",
        "\n",
        "* **Position 1**: A Pandas Dataframe that includes at least a column of search queries.\n",
        "* **col**: The name of the query column from the DataFrame\n",
        "* **alg**: Either `apriori` (Apriori Algorithm), or `fpgrowth` (FP Growth Algorithm)\n",
        "* **min_support**: The number of times the subset terms should be found in the dataset to be included.\n",
        "* **min_probablity**: FP Growth only. Finds patterns of associated terms above this minimum probability 0-1 (float)\n",
        "* **min_lift**: Apriori only. Finds patterns of associated terms. <1 less probable, >1 more probable  (float)\n",
        "\n",
        "After it runs, it will print the number of categories found, as well as the frequencies of queries in each category."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JsBok9aBUkCJ",
        "colab_type": "code",
        "outputId": "5edfef57-4c94-439d-a764-0f11ac8b0e26",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 774
        }
      },
      "source": [
        "MINSUPPORT =  3#@param {type:\"integer\"}\n",
        "\n",
        "catz = querycat.Categorize(DT, 'query', min_support=MINSUPPORT,  alg='apriori')\n",
        "catz.counts.head(20)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Converting to transactions.\n",
            "Normalizing the keyword corpus.\n",
            "Total queries: 354\n",
            "Total unique queries: 295\n",
            "Total transactions: 294\n",
            "Running Apriori\n",
            "Making Categories\n",
            "Total Categories: 133\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>seo</th>\n",
              "      <td>75</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>##other##</th>\n",
              "      <td>34</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>nlp</th>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>training seo</th>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>data science</th>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>learning machine</th>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>course seo</th>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>formation data scientist</th>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>formation learning machine</th>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>data seo science</th>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>formation data ligne en</th>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>seo python</th>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>data lab</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>data lab seo</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>lab seo</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>dataiku certification</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>niveau</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>learning seo machine</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>learning deep</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>in seo</th>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                            category\n",
              "seo                               75\n",
              "##other##                         34\n",
              "nlp                               13\n",
              "training seo                      11\n",
              "data science                      10\n",
              "learning machine                   8\n",
              "course seo                         7\n",
              "formation data scientist           7\n",
              "formation learning machine         6\n",
              "data seo science                   6\n",
              "formation data ligne en            6\n",
              "seo python                         6\n",
              "data lab                           5\n",
              "data lab seo                       5\n",
              "lab seo                            5\n",
              "dataiku certification              5\n",
              "niveau                             5\n",
              "learning seo machine               5\n",
              "learning deep                      5\n",
              "in seo                             4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1cLOFkcQUkCS",
        "colab_type": "text"
      },
      "source": [
        "## Show original data\n",
        "Below we print the original DataFrame with categories added.\n",
        "\n",
        "This can be saved to CSV using `catz.df.to_csv('filename.csv', index=False)` to save and use in Excel."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tUpKqduCUkCV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "catz.df.head(10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p4O4lkCcUkCc",
        "colab_type": "text"
      },
      "source": [
        "## Make use of original data\n",
        "\n",
        "This cell will aggregate data from your original CSV to produce an updated DataFrame that has current and previous period data consolidated by category.  We also include a `diff` column that shows the period change."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sRddqx2wUkCg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = catz.df\n",
        "df.head()\n",
        "\n",
        "# Goup by category and aggregate the other metrics\n",
        "df = df.groupby(['category'], as_index=False).agg({'query':'count', 'last':'sum', 'previous': 'sum' })\n",
        "\n",
        "# Create a diff column\n",
        "df['diff'] = df['last'] - df['previous']\n",
        "\n",
        "# Rename the columns\n",
        "df.columns = ['category', 'count of queries', 'current', 'prior', 'diff']\n",
        "\n",
        "# Sort by the `diff` columnn\n",
        "df.sort_values(by='diff', ascending=False, inplace=True)\n",
        "\n",
        "df.head(10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eWpHdAWXUkCo",
        "colab_type": "text"
      },
      "source": [
        "## BERT\n",
        "\n",
        "The BERTSim class has a few functions that are useful:\n",
        "\n",
        "* **read_df**: Reads a dataframe where the `term_col` and `diff_col` are specified as named parameters.\n",
        "* **get_similar_df**: After running `read_df`, you can provide a term as a parameter to get the top matching categories, by cosine similarity.\n",
        "* **diff_plot**: After running `read_df`, plots the output in 2d vector space, using either `umap`, `pca`, or `tsne` as vector reduction strategies. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-6RdxTXwUkCq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163,
          "referenced_widgets": [
            "303c2910322c4e1791a80a7ca73b3035",
            "fd5bbb5e4dcb4316ad3e46d80864c418",
            "8daace4c478e4690aa8841026356fd2d",
            "d0c8fd4bcd554e5da1ebb3d72cedcaf4",
            "bdbe08e3d97e4fd68ccd1c0ce960ebaf",
            "539a17cabd25455ab4fd42a7fc250acf",
            "6bc3d3525a1e4afbbdffab0a56efaef6",
            "338f1b39d8414660902323b04a316a29",
            "d5bd80f520034d23a2f784e08cff0389",
            "ea39bcc6107e485eb08536124eab2faa",
            "e491ebb4e62841b99ae870261e68e248",
            "f0978f9138984fee8ea02e3786ec1886",
            "c79c3e83545b4ae4b36e5a04835ee8e1",
            "d21b3afad66942ef97730567606120f0",
            "3bf7a4f5d68047b2825fd0c2a0e1bdea",
            "9816d07fd02a4ce1a31a7b49fe0b1915",
            "a107ab7e69f24d899e3dd3c307e74c20",
            "91bc7ac737b140e2923ea29f42747bf4",
            "555e7bd2851340a7bead02aeb37d9152",
            "d27b51b360fa426dade096dbe5f785d1",
            "b8d247fb311646618b6ef04ff4261b11",
            "48713f95c8f245ce865539a86e052fd8",
            "da225c952b104f8a81199ae5920e487e",
            "b34ad090026741bd841270725d6eaab8"
          ]
        },
        "outputId": "22753bd8-7628-4ce1-eaed-c1570e46eed0"
      },
      "source": [
        "bsim = querycat.BERTSim(TRANSFORMER_MODEL, RANDOM_SEED)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "303c2910322c4e1791a80a7ca73b3035",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=596, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d5bd80f520034d23a2f784e08cff0389",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=445032417, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a107ab7e69f24d899e3dd3c307e74c20",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=810912, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sr0Z6XdEUkCy",
        "colab_type": "text"
      },
      "source": [
        "### Read DataFrame into Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MouR3jiNUkCz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bsim.read_df(df, term_col = 'category', diff_col = 'diff')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0bPfh-YUkC6",
        "colab_type": "text"
      },
      "source": [
        "### Print similar categories to a given term."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_fDu2TXGUkC8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_sim = bsim.get_similar_df('train')\n",
        "df_sim.sort_values(by='sim', ascending=False, inplace=True)\n",
        "df_sim.head(30)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JiER4oqJUkDH",
        "colab_type": "text"
      },
      "source": [
        "### Plot Categories\n",
        "\n",
        "This cell plots the categories in 2D space, with reduction from BERT summed 768 dimension vectors to 2 dimension, based on the given reduction strategy.  Green is increase PoP, red is descrease.  Size is the magnitude of the change.\n",
        "\n",
        "Reduction strategies available are `umap`, `tsne`, and `pca`\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hDRU7DpqUkDJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bsim.diff_plot(reduction='tsne')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aJCLTWy4ykTu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "bsim.diff_plot(reduction='pca')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}