DL_Project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DL_Project.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "CguZ0u-Mr_e3",
        "Dql3oFh31xmW",
        "h2_ttxft169l",
        "IfUsXDd7n-SF",
        "Zk_cp8qNoUdy",
        "pPe3ICOSg1oi",
        "0B_oXyh0Se7X",
        "iyorKZxw08kB",
        "FqpSYcEDxEgQ",
        "Rf5CVZFbg5uw",
        "spBjl0Ot-BXS",
        "aiF3C08q6FMS",
        "8faPuKnOrS8w"
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#Library Imports"
      ],
      "metadata": {
        "id": "FW_eO2axyj0K"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IzZ5lFpMC5h3"
      },
      "source": [
        "# Installing additional libraries\n",
        "%%capture\n",
        "!pip install torch torchvision torchtext torchaudio pytorch-metric-learning torch-lr-finder"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import zipfile, os.path\n",
        "import shutil\n",
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from torch.utils.data import Dataset, Subset, DataLoader, WeightedRandomSampler\n",
        "from PIL import Image\n",
        "from torchvision import transforms\n",
        "import random\n",
        "import torch.nn.functional as F\n",
        "from typing import Dict, Set, List\n",
        "from pytorch_metric_learning import distances, losses, miners, reducers\n",
        "import torch.utils.checkpoint as cp\n",
        "import math\n",
        "from torchvision import models\n",
        "from torch_lr_finder import TrainDataLoaderIter, ValDataLoaderIter, LRFinder\n",
        "from tqdm import tqdm\n",
        "from torch.utils.tensorboard import SummaryWriter\n",
        "import gc\n",
        "from google.colab import files"
      ],
      "metadata": {
        "id": "cnVxWLTbytwE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CguZ0u-Mr_e3"
      },
      "source": [
        "# Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dql3oFh31xmW"
      },
      "source": [
        "## Importing the Drive archive\n",
        "Following passages import and extract the image dataset in the Colab files, if not already present"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ra-n_XgHpYr_"
      },
      "source": [
        "if(not (os.path.isdir('test') and os.path.isdir('train') and os.path.isdir('queries') and os.path.isfile('annotations_train.csv'))):\n",
        "  if(not os.path.isfile('dataset.zip')):\n",
        "    # Download the file directly from Matteo's Drive (The file is public)\n",
        "    !gdown --id 13tm5l5uwj4zUMhm2YMRx0IKXySaRdxEE\n",
        "  # Dataset extraction\n",
        "  with zipfile.ZipFile('dataset.zip', 'r') as zip_ref:\n",
        "      zip_ref.extractall()\n",
        "  # Remove the zip file\n",
        "  os.remove('dataset.zip')\n",
        "  shutil.rmtree('sample_data')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h2_ttxft169l"
      },
      "source": [
        "## Dataset and Sample\n",
        "A Dataset class for training and testing samples. Samples have the following structure:\n",
        "\n",
        "| Key         | Value |\n",
        "| ----------- | ----------- |\n",
        "| image       | A Tensor containing the image. |\n",
        "| img_file    | Name of the image file. Used on the result reporting. |\n",
        "| person_id    | Id of the person in the image. `None` if not available. |\n",
        "| annotations | Annotations array for the person in the image. `None` if not available. |\n",
        "\n",
        "If the `csv_file` constructor parameter of `MarketDataset` is not provided, then the ID and the annotations will be not available\n",
        "\n",
        "[Pytorch tutorial for custom Datasets, Dataloaders and Transforms](https://pytorch.org/tutorials/beginner/data_loading_tutorial.html)\n",
        "\n",
        "Image size: (3, 128, 64)\n",
        "\n",
        "Output size: (56)\n",
        "\n",
        "Output size (vector of probabilities): 56"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LwTia13m65di"
      },
      "source": [
        "# Class for the Dataset\n",
        "class MarketDataset(Dataset):\n",
        "  \"\"\"Market-1501 (project version) dataset.\"\"\"\n",
        " \n",
        "  def __init__(self, root, csv_file=None, transform=None, target_transform=None):\n",
        "      \"\"\"\n",
        "      Args:\n",
        "          csv_file (string): Path to the csv file with annotations. If None, no\n",
        "              labeling will be added to samples (test samples).\n",
        "          root (string): Directory with all the images.\n",
        "          transform (callable, optional): Optional transform to be applied\n",
        "              on a sample.\n",
        "      \"\"\"\n",
        "      self.annotations_frame = pd.read_csv(csv_file).set_index('id') if csv_file is not None else None\n",
        "      self.root = root\n",
        "      self.img_files = os.listdir(root)\n",
        "      self.transform = transform\n",
        "      self.target_transform = target_transform\n",
        " \n",
        "  def __len__(self):\n",
        "      return len(self.img_files)\n",
        " \n",
        "  def __getitem__(self, idx):\n",
        "      if torch.is_tensor(idx):\n",
        "          idx = idx.tolist()\n",
        " \n",
        "      # Loading the image from its folder\n",
        "      img_path = os.path.join(self.root, self.img_files[idx])\n",
        "      # The image must be in PIL format in order to apply the transformations\n",
        "      image = Image.open(img_path)\n",
        " \n",
        "      # Creating the sample\n",
        "      sample = {'image': image, 'img_file': self.img_files[idx]}\n",
        " \n",
        "      if self.annotations_frame is not None:\n",
        "        # Extracting its PersonID\n",
        "        person_id = int(self.img_files[idx].partition('_')[0])\n",
        "        # Extracting corresponding annotations\n",
        "        annotations = self.annotations_frame.loc[person_id]\n",
        "        annotations = torch.tensor(annotations).int()\n",
        "        # Adding to the sample\n",
        "        sample['person_id'] = person_id\n",
        "        sample['annotations'] = annotations\n",
        "      \n",
        "      # Apply the transformations\n",
        "      if self.transform:\n",
        "        sample['image'] = self.transform(sample['image'])\n",
        "      if self.target_transform:\n",
        "        sample['annotations'] = self.target_transform(sample['annotations'])\n",
        "      \n",
        "      return sample"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oTl3mTwbiknO"
      },
      "source": [
        "## Transformations\n",
        "Typically, a composition of transformations is passed to the Dataset, so the samples are modified when loaded"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IfUsXDd7n-SF"
      },
      "source": [
        "### Image transformations (data augmentation)\n",
        "This composition of transformations is applied to the images when loaded by the dataset\n",
        "\n",
        "Further ideas about transformations: https://www.programmersought.com/article/19232071306/\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xgATkqL6jAtU"
      },
      "source": [
        "# Define transformations for the data\n",
        "data_transform = transforms.Compose([\n",
        "                          transforms.RandomRotation(degrees=(0,5), center =(62,30)),\n",
        "                          transforms.RandomAffine(2, translate=(0.05,0.05), scale=None, shear=None),\n",
        "                          transforms.RandomPerspective(distortion_scale=0.02, fill=(0,0,0)),\n",
        "                          transforms.ColorJitter(brightness=0.1),\n",
        "                          transforms.ColorJitter(contrast=0.1),\n",
        "                          transforms.RandomAutocontrast(p=1),\n",
        "                          transforms.ColorJitter(saturation=0.1),\n",
        "                          transforms.RandomAdjustSharpness(sharpness_factor=2, p=0.3),\n",
        "                          transforms.RandomHorizontalFlip(p=0.5),\n",
        "                          transforms.ToTensor(),\n",
        "                          transforms.RandomErasing(scale=(0.01,0.05), value =(1,1,1)),\n",
        "])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zk_cp8qNoUdy"
      },
      "source": [
        "### Label transformations\n",
        "These transformations can be applied to annotations to switch from label-based annotations to probability-based annotations. The probability vector is a sequence of 0-1 values, where each group of values encode the label with a 1 at the corresponding (local) index. Colors fields could be all ones: in this case, the correct classification is `multicolor`\n",
        "\n",
        "**Annotation form:**\n",
        "\n",
        "| Description | Attribute | Labels |\n",
        "| :----------- | ----------- | ----------- | \n",
        "| age | age | young(1), teenager(2), adult(3), old(4) \n",
        "| carrying backpack| backpack | no(1), yes(2) | \n",
        "| carrying bag | bag | no(1), yes(2) | \n",
        "| carrying handbag | handbag | no(1), yes(2) | \n",
        "| type of lower-body clothing| clothes | dress(1), pants(2) | \n",
        "| length of lower-body clothing | down | long(1), short(2) | \n",
        "| sleeve length | up | long(1), short(2) | \n",
        "| hair length | hair | short(1), long(2) | \n",
        "| wearing hat| hat | no(1), yes(2) | \n",
        "| gender | gender | male(1), female(2) | \n",
        "| Black upper-body clothing | upblack | no(1), yes(2) | \n",
        "| White upper-body clothing | upwhite | no(1), yes(2) | \n",
        "| Red upper-body clothing | upred | no(1), yes(2) | \n",
        "| Purple upper-body clothing | uppurple | no(1), yes(2) | \n",
        "| Yellow upper-body clothing | upyellow | no(1), yes(2) | \n",
        "| Gray upper-body clothing | upgray | no(1), yes(2) | \n",
        "| Blue upper-body clothing | upblue | no(1), yes(2) | \n",
        "| Green upper-body clothing | upgreen | no(1), yes(2) | \n",
        "| Black lower-body clothing | downblack | no(1), yes(2) | \n",
        "| White lower-body clothing | downwhite | no(1), yes(2) | \n",
        "| Pink lower-body clothing | downpink | no(1), yes(2) | \n",
        "| Purple lower-body clothing | downpurple | no(1), yes(2) | \n",
        "| Yellow lower-body clothing | downyellow | no(1), yes(2) | \n",
        "| Gray lower-body clothing | downgray | no(1), yes(2) | \n",
        "| Blue lower-body clothing | downblue | no(1), yes(2) | \n",
        "| Green lower-body clothing | downgreen | no(1), yes(2) | \n",
        "| Brown lower-body clothing | downbrown | no(1), yes(2) |\n",
        "\n",
        "**Target form:**\n",
        "\n",
        "| Task | Classes |\n",
        "| :--- | ------- |\n",
        "| age | [young, teenager, adult, old] |\n",
        "| backpack | [no, yes] |\n",
        "| bag | [no, yes] |\n",
        "| handbag | [no, yes] |\n",
        "| clothes | [dress, pants]\n",
        "| down | [long, short]\n",
        "| up | [long, short]\n",
        "| hair | [short, long]\n",
        "| hat | [no, yes]\n",
        "| gender | [male, female]\n",
        "| upcolor | [black, white, red, purple, yellow, gray, blue, green, multicolor]\n",
        "| downcolor | [black, white, pink, purple, yellow, gray, blue, green, brown, multicolor]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9gOobL-VtzzW"
      },
      "source": [
        "def annotation_to_target(annotation):\n",
        "  \"\"\"\n",
        "    Transforms the labels in the annotation form (as they are encoded in the\n",
        "    dataset, with the color information spread across several values), into the \n",
        "    target form (a single value for each task representing the correct class), \n",
        "    which is used in the training for the loss function.\n",
        "    :param annotation (torch.Tensor) labels in the annotation form.\n",
        "    :return (torch.Tensor) labels in the target form.\n",
        "  \"\"\"\n",
        "  # Convert the annotation into integers starting from 0\n",
        "  not_colors = annotation[..., :10] - 1\n",
        "  upcolors = annotation[..., 10:18] - 1\n",
        "  downcolors = annotation[..., 18:] - 1\n",
        "  \n",
        "  # Handle the multicolors\n",
        "  upmulticolor = 1 - torch.unsqueeze(torch.sum(upcolors, axis=-1), 0).T\n",
        "  upcolor = torch.argmax(torch.hstack((upcolors, upmulticolor)), axis=-1)\n",
        "  upcolor = torch.unsqueeze(upcolor, 0).T\n",
        "\n",
        "  downmulticolor = 1 - torch.unsqueeze(torch.sum(downcolors, axis=-1), 0).T\n",
        "  downcolor = torch.argmax(torch.hstack((downcolors, downmulticolor)), axis=-1)\n",
        "  downcolor = torch.unsqueeze(downcolor, 0).T\n",
        "\n",
        "  return torch.hstack((not_colors, upcolor, downcolor))\n",
        "\n",
        "def prediction_to_target(prediction):\n",
        "  \"\"\"\n",
        "    Transform the labels in the prediction form (the one-hot encoding form that\n",
        "    the network should output) into the target form (a single value for each task \n",
        "    representing the correct class) by taking the most probable value for each task.\n",
        "    :param prediction (torch.Tensor) labels in the prediction form.\n",
        "    :return (torch.Tensor) labels in the target form.\n",
        "  \"\"\"\n",
        "  tasks_splits = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10]\n",
        "  target = torch.vstack([torch.argmax(task, axis=-1) for task in torch.split(prediction, tasks_splits, dim=-1)]).T\n",
        "  return target\n",
        "\n",
        "def target_to_annotation(target):\n",
        "  \"\"\"\n",
        "    Transforms the labels of multiple samples in the target form (a single value \n",
        "    for each task representing the correct class), into the annotation form, \n",
        "    (as they are encoded in the dataset, with the color information spread \n",
        "    across several values)\n",
        "    :param target (torch.Tensor) labels in the target form.\n",
        "    :return (torch.Tensor) labels in the annotation form.\n",
        "  \"\"\"  \n",
        "  # Handle the multicolors\n",
        "  upcolor = target[..., -2]\n",
        "  upcolors = F.one_hot(upcolor, num_classes=9)[..., :-1]\n",
        "\n",
        "  downcolor = target[..., -1]\n",
        "  downcolors = F.one_hot(downcolor, num_classes=10)[..., :-1]\n",
        "  \n",
        "  # First part, without colors\n",
        "  not_colors = target[..., :10]\n",
        "  \n",
        "  # Go back to 1-starting values\n",
        "  return torch.hstack((not_colors, upcolors, downcolors)).int() + 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pPe3ICOSg1oi"
      },
      "source": [
        "### Id transformations\n",
        "These transformations can be used to normalize the person IDs within a dataset split.\n",
        "\n",
        "Since the IDs on the dataset are not uniform, and will be randomly divided into different splits, these transformations will normalize them in the interval [0, num_of_ids]."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kyzZN0yElH_n"
      },
      "source": [
        "def normalize_pids(pids: set):\n",
        "  \"\"\"\n",
        "  The functions creates a lambda to convert a person ID into its normalized\n",
        "  version, which is its index in the pids function parameter.\n",
        "\n",
        "  :param pids(set) The list of all the person IDs in the considered split\n",
        "  :return A lambda that normalize a pid from the pids list\n",
        "  \"\"\"\n",
        "  pids = list(pids)\n",
        "  translator = {pid: idx for idx, pid in enumerate(pids)}\n",
        "\n",
        "  return lambda pid: translator[pid]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0B_oXyh0Se7X"
      },
      "source": [
        "## Dataset splitting\n",
        "As stated in the project assignment:\n",
        "> Be careful when producing your train and validation splits. All images of the same person should be either in the train or in the validation dataset, otherwise, your validation performance will be higher than test performance.\n",
        "\n",
        "So, we need a splitter function to handle this.\n",
        "\n",
        "Moreover, a custom Subset is needed to apply the transformations directly to it and not on the main dataset."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pE-9ttb8DLBv"
      },
      "source": [
        "class TransformSubset(Subset):\n",
        "  \"\"\"\n",
        "  Subset of a dataset at specified indices, with optional transformations\n",
        "  applicable to data or target.\n",
        "  \n",
        "  Args:\n",
        "    dataset (Dataset): The whole Dataset\n",
        "    indices (sequence): Indices in the whole set selected for subset\n",
        "    pid_transform: a function that transforms the person_id attribute\n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self, dataset: Dataset, indices, transform=None, target_transform=None, pid_transform=None) -> None:\n",
        "    super().__init__(dataset, indices)\n",
        "    self.transform = transform\n",
        "    self.target_transform = target_transform\n",
        "    self.pid_transform = pid_transform\n",
        "\n",
        "  def __getitem__(self, idx):\n",
        "    sample = self.dataset[self.indices[idx]]\n",
        "    if self.transform:\n",
        "      sample['image'] = self.transform(sample['image'])\n",
        "    if self.target_transform:\n",
        "      sample['annotations'] = self.target_transform(sample['annotations'])\n",
        "    if self.pid_transform:\n",
        "      sample['person_id'] = self.pid_transform(sample['person_id'])\n",
        "    return sample"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XVJgvJ4_WSJe"
      },
      "source": [
        "def rand_balanced_split(dataset, proportions):\n",
        "  \"\"\"\n",
        "    Method to randomly split the MarketDataset, grouping by the personID\n",
        "    :param dataset (MarketDataset) holds market Sample.\n",
        "    :param proportions (List[float]) proportions to divide the dataset; should sum to 1.\n",
        "    :return (Tuple[Dataset]) splitted datasets.\n",
        "  \"\"\"\n",
        "  # Grouping by personID\n",
        "  indices_of_id = {}\n",
        "  for sample_idx, img_file in enumerate(dataset.img_files):\n",
        "    person_id = int(img_file.partition('_')[0])\n",
        "    if person_id in indices_of_id:\n",
        "      indices_of_id[person_id].append(sample_idx)\n",
        "    else:\n",
        "      indices_of_id[person_id] = [sample_idx]\n",
        "  # Get the annotations for each personID\n",
        "  target_of_id = {}\n",
        "  for person_id in indices_of_id:\n",
        "    annotation = dataset.annotations_frame.loc[person_id]\n",
        "    target = annotation_to_target(torch.tensor(annotation).int())\n",
        "    target_of_id[person_id] = target\n",
        "  # Creating the lists of indices for splitting:\n",
        "  # The target quantity of samples for each split\n",
        "  target_counts = np.array(proportions) * len(dataset)\n",
        "  # Lists of how should the indices be divided among the splits\n",
        "  samples_splits = [[] for _ in proportions]\n",
        "  # A collection of the still incomplete splits\n",
        "  incomplete_splits = [*range(len(proportions))]\n",
        "  # List, for each split, of a set for each task to check the added classes\n",
        "  tasks_splits = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10]\n",
        "  unfilled_classes = [[set(range(split)) for split in tasks_splits] for _ in proportions]\n",
        "  # Randomly iterate over all person_ids:\n",
        "  for person_id in random.sample(indices_of_id.keys(), len(indices_of_id)):\n",
        "    indices = indices_of_id[person_id]\n",
        "    target = target_of_id[person_id]\n",
        "    # Compute which split is better completed by the target of the random person_id\n",
        "    best_split_idx = None\n",
        "    max_score = -1\n",
        "    # Randomly choose among the max-scoring incomplete splits\n",
        "    for split_idx in random.sample(incomplete_splits, len(incomplete_splits)):\n",
        "      split_unfilled_classes = unfilled_classes[split_idx]\n",
        "      # Score: how many new classes can the new sample fill in the split\n",
        "      score = 0\n",
        "      for task_class, task_unfilled_classes in zip(target, split_unfilled_classes):\n",
        "        # Increase the score if the task_class is still in the set of the unfilled classes\n",
        "        score += task_class in task_unfilled_classes\n",
        "      if score > max_score:\n",
        "        max_score = score\n",
        "        best_split_idx = split_idx\n",
        "    # Add the indices of the selected person_id to the best index\n",
        "    samples_splits[best_split_idx].extend(indices)\n",
        "    # Remove classes added just now from the unfilled classes\n",
        "    for task_class, task_unfilled_classes in zip(target, unfilled_classes[best_split_idx]):\n",
        "      if task_class in task_unfilled_classes:\n",
        "        task_unfilled_classes.remove(task_class)\n",
        "    # Reset the unfilled counter when completely emptied\n",
        "    reset = True\n",
        "    for split_unfilled_classes in unfilled_classes:\n",
        "      for task_unfilled_classes in split_unfilled_classes:\n",
        "        if not len(task_unfilled_classes) == 0:\n",
        "          reset = False\n",
        "    if reset:\n",
        "      unfilled_classes = [[set(range(split)) for split in tasks_splits] for _ in proportions]\n",
        "\n",
        "    # Remove the split from the incomplete ones when it reaches its capacity\n",
        "    if len(samples_splits[best_split_idx]) > target_counts[best_split_idx] and len(incomplete_splits) > 1:\n",
        "      incomplete_splits.remove(best_split_idx)\n",
        "  \n",
        "  return tuple(TransformSubset(dataset, indices) for indices in samples_splits)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1gUfA3oBlBRb"
      },
      "source": [
        "## Loading the data\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iyorKZxw08kB"
      },
      "source": [
        "### Class balancing (weight)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zKRryyZCSS2I"
      },
      "source": [
        "def get_dataset_targets(dataset):\n",
        "  \"\"\"\n",
        "    Method to recover the targets from the dataset's annotations, without having\n",
        "    to load the images.\n",
        "    :param dataset (MarketDataset) the dataset containing the targets.\n",
        "  \"\"\"\n",
        "  if isinstance(dataset, torch.utils.data.Subset):\n",
        "    # If the dataset is a subset, only some indices are used from the internal original dataset\n",
        "    indices = dataset.indices\n",
        "    dataset = dataset.dataset # Replace the subset with the whole dataset\n",
        "  else:\n",
        "    # Otherwise, all the indices are used\n",
        "    indices = range(len(dataset.img_files))\n",
        "  targets = []\n",
        "  for idx in indices:\n",
        "    # Retrieve the target of the considered image without loading it in memory\n",
        "    img_file = dataset.img_files[idx]\n",
        "    person_id = int(img_file.partition('_')[0])\n",
        "    annotation = dataset.annotations_frame.loc[person_id]\n",
        "    annotation = torch.tensor(annotation).int()\n",
        "    targets.append(annotation_to_target(annotation))\n",
        "  return torch.vstack(targets)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5mWa6VVBp7OU"
      },
      "source": [
        "def get_dataset_unique_pids(dataset):\n",
        "  \"\"\"\n",
        "    Method to recover the unique person IDs from the dataset's annotations, \n",
        "    without having to load the images.\n",
        "    :param dataset (MarketDataset) The dataset containing the person IDs.\n",
        "    :return pids (set) The set of all unique person IDs in the dataset\n",
        "  \"\"\"\n",
        "  if isinstance(dataset, torch.utils.data.Subset):\n",
        "    # If the dataset is a subset, only some indices are used from the internal original dataset\n",
        "    indices = dataset.indices\n",
        "    dataset = dataset.dataset # Replace the subset with the whole dataset\n",
        "  else:\n",
        "    # Otherwise, all the indices are used\n",
        "    indices = range(len(dataset.img_files))\n",
        "  pids = set()\n",
        "  for idx in indices:\n",
        "    # Retrieve the target of the considered image without loading it in memory\n",
        "    img_file = dataset.img_files[idx]\n",
        "    person_id = int(img_file.partition('_')[0])\n",
        "    pids.add(person_id)\n",
        "  return pids"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-gAZm4bo1CCT"
      },
      "source": [
        "def get_class_weight(targets, weights=None):\n",
        "  \"\"\"\n",
        "    Method to get the class weights, used to balance the samples importance in\n",
        "    the loss, from the frequencies of the various task-classes in the dataset. \n",
        "    The frequencies can be pre-weighted, in the case that the samples are not\n",
        "    uniformely sampled but extracted with different probabilities.\n",
        "    :param dataset (Dataset)\n",
        "    :param weights (torch.Tensor): The sampling weights of the samples\n",
        "  \"\"\"\n",
        "  # Count the frequencies for each task\n",
        "  tasks_splits = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10]\n",
        "  freq = [torch.bincount(task.reshape(-1), weights, minlength) for task, minlength in zip(torch.split(targets, 1, dim=1), tasks_splits)]\n",
        "  # Compute the weights as the reciprocal of the normalized frequencies\n",
        "  # The normalization permit to compare tasks with a different num of classes\n",
        "  return torch.cat([torch.reciprocal(task / torch.sum(task) * len(task)) for task in freq])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FqpSYcEDxEgQ"
      },
      "source": [
        "### Dataloaders\n",
        "\n",
        "The DataLoader objects helps to create batches of samples from the Dataset. The following `get_data` function is used to obtain all needed dataloaders.\n",
        "\n",
        "- The dataset is split into balanced validation and training\n",
        "- Data transformations are applied to training data\n",
        "- Person IDs are normalized separately in both splits\n",
        "- Balanced class weights are recomputed"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AhsOxB9ylBqQ"
      },
      "source": [
        "# Define the variables with the data from the dataset to support the get_data() function\n",
        "full_data = MarketDataset(csv_file='annotations_train.csv', root='train', target_transform=annotation_to_target)\n",
        "test_data = MarketDataset(root='test', transform=transforms.ToTensor())\n",
        "queries_data = MarketDataset(root='queries', transform=transforms.ToTensor())\n",
        "# Train and validation split\n",
        "while True:\n",
        "  train_data, val_data = rand_balanced_split(full_data, [.9, .1])\n",
        "  train_targets = get_dataset_targets(train_data)\n",
        "  train_class_weights = get_class_weight(train_targets)\n",
        "  # Check that all classes are present at least once in the training set\n",
        "  if torch.all(torch.isfinite(train_class_weights)):\n",
        "    break\n",
        "# Apply the data transformation ONLY on the training set, not to the validation\n",
        "train_data.transform = data_transform\n",
        "val_data.transform = transforms.ToTensor()\n",
        "# Apply the person IDs transformation SEPARATELY on training and validation sets\n",
        "train_pids = get_dataset_unique_pids(train_data)\n",
        "train_data.pid_transform = normalize_pids(train_pids)\n",
        "val_pids = get_dataset_unique_pids(val_data)\n",
        "val_data.pid_transform = normalize_pids(val_pids)\n",
        "\n",
        "# Compute the samples' weights as the mean of the its target classes' weights\n",
        "tasks_splits = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10]\n",
        "# Splitting the class weights into a list with the different tasks\n",
        "class_weights = list(torch.split(train_class_weights, tasks_splits))\n",
        "samples_weights = torch.zeros(train_targets.shape[0])\n",
        "for sample, target in enumerate(train_targets):\n",
        "  for task, task_class in enumerate(target):\n",
        "    samples_weights[sample] += class_weights[task][task_class]\n",
        "samples_weights /= len(tasks_splits)\n",
        "# Create a sampler that extract more frequently samples presenting low-frequence classes \n",
        "sampler = WeightedRandomSampler(samples_weights, int(len(samples_weights) * 1.25), replacement=True)\n",
        "# Recalculate the class weights taking into account the sampling weights\n",
        "balanced_class_weights = get_class_weight(train_targets, weights=samples_weights)\n",
        "\n",
        "# Get a DataLoader object for all the sets\n",
        "def get_data(batch_size, test_batch_size=128):\n",
        "  full_loader = DataLoader(full_data, batch_size, shuffle=True, num_workers=2, pin_memory=True)\n",
        "  train_loader = DataLoader(train_data, batch_size, sampler=sampler, num_workers=2, pin_memory=True)\n",
        "  val_loader = DataLoader(val_data, test_batch_size, shuffle=False, num_workers=2, pin_memory=True)\n",
        "  test_loader = DataLoader(test_data, test_batch_size, shuffle=False, num_workers=2, pin_memory=True)\n",
        "  queries_loader = DataLoader(queries_data, test_batch_size, shuffle=False, num_workers=2, pin_memory=True)\n",
        "  return {\n",
        "    'full': full_loader,\n",
        "    'train': train_loader,\n",
        "    'val': val_loader,\n",
        "    'test': test_loader,\n",
        "    'queries': queries_loader,\n",
        "    'class_weight': balanced_class_weights,\n",
        "    'train_pids_count': len(train_pids),\n",
        "    'val_pids_count': len(val_pids),\n",
        "  }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "klL8Vc3e2NGI"
      },
      "source": [
        "# Training Utilities"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QypJdpb-nyfS"
      },
      "source": [
        "## Metrics & Logging"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BXBw6W-kxyoF"
      },
      "source": [
        "# Class that is used to compute the metrics:\n",
        "# + Accuracy for each sub-task of the attribute recognition\n",
        "# + Average global accuracy\n",
        "# + Average mAcc\n",
        "# + Average mAP\n",
        "class Metrics():\n",
        "  def __init__(self, tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10], device='cuda'):\n",
        "    self.M = [torch.zeros(class_count, class_count, device=device) for class_count in tasks_splits]\n",
        "    self.device = device\n",
        "    self.baselines = 1 / torch.tensor(tasks_splits, device=device)\n",
        "    self.reid_prediction = None\n",
        "    self.reid_target = None\n",
        "    self.suggested_threshold = None\n",
        "  \n",
        "  def clear(self):\n",
        "    for C in self.M:\n",
        "      C.zero_()\n",
        "    self.reid_prediction = None\n",
        "    self.reid_target = None\n",
        "    self.suggested_threshold = None\n",
        "  \n",
        "  def add(self, prediction, target):\n",
        "    # Split the prediction and the target columns of the same task\n",
        "    prediction = prediction.to(self.device).split(1, dim=-1)\n",
        "    target = target.to(self.device).split(1, dim=-1)\n",
        "    with torch.no_grad(): # We require no computation graph\n",
        "      for C, yp, yt in zip(self.M, prediction, target):\n",
        "        C += (yt.reshape(-1) * C.shape[1] + yp.reshape(-1)).bincount(minlength=C.numel()).view(C.shape).float()\n",
        "  \n",
        "  def add_reid(self, predictions: Dict[str, List], ground_truth: Dict[str, Set], suggested_threshold: int):\n",
        "    \"\"\"\n",
        "    :param predictions: dictionary from query filename to list of test image \n",
        "      filenames associated with the query ordered from the most to the least \n",
        "      confident prediction. Represents the predictions to be evaluated.\n",
        "    :param ground_truth: dictionary from query filename to set of test image \n",
        "      filenames associated with the query. Represents the ground truth on which \n",
        "      to evaluate predictions.\n",
        "    :param suggested_threshold: a value between 0 and 1 corresponding to the \n",
        "      threshold used to compute the predictions.\n",
        "    \"\"\"\n",
        "    self.reid_prediction = predictions\n",
        "    self.reid_target = ground_truth\n",
        "\n",
        "  # Computes the global accuracy\n",
        "  def acc(self):\n",
        "    return torch.stack([C.diag().sum() / C.sum() for C in self.M])\n",
        "\n",
        "  # Computes the class-averaged accuracy\n",
        "  def mAcc(self):\n",
        "    averages = [C.diag() / C.sum(-1) for C in self.M]\n",
        "    return torch.stack([average[~torch.isnan(average)].mean() for average in averages])\n",
        "\n",
        "  # Computes the normalized mAcc, that is set to zero if below the baseline mAcc (1 / # of Classes)\n",
        "  def mAcc_norm(self):\n",
        "    mAcc = self.mAcc()\n",
        "    # Rescaling and set to zero negative values\n",
        "    mAcc_norm = torch.max(((mAcc - self.baselines) / (1 - self.baselines)),  torch.tensor(0))\n",
        "    return mAcc_norm\n",
        "\n",
        "  # Returns the confusion matrix\n",
        "  # Rows: target classes | Columns: predicted classes\n",
        "  def confusion_matrices(self):\n",
        "    return self.M    \n",
        "\n",
        "  # Willi's evaluator for mAP (mAcc)\n",
        "  def reid_mAP(self):\n",
        "    \"\"\"\n",
        "    Computes the mAP (https://jonathan-hui.medium.com/map-mean-average-precision-for-object-detection-45c121a31173) \n",
        "    of the predictions with respect to the given ground truth. In person reidentification \n",
        "    mAP refers to the mean of the AP over all queries. The AP for a query is the \n",
        "    area under the precision-recall curve obtained from the list of predictions \n",
        "    considering the ground truth elements as positives and the other ones as negatives.\n",
        "    \"\"\"\n",
        "    m_ap = 0.0\n",
        " \n",
        "    for current_ground_truth_query, current_ground_truth_query_set in self.reid_target.items():\n",
        "      # No predictions were performed for the current query, AP = 0\n",
        "      if not current_ground_truth_query in self.reid_prediction:\n",
        "        continue\n",
        "\n",
        "      current_ap = 0.0  # The area under the curve for the current sample\n",
        "      current_predictions_list = self.reid_prediction[current_ground_truth_query]\n",
        "\n",
        "      # Recall increments of this quantity each time a new correct prediction is encountered in the prediction list\n",
        "      delta_recall = 1.0 / len(current_ground_truth_query_set)\n",
        "\n",
        "      # Goes through the list of predictions\n",
        "      encountered_positives = 0\n",
        "      for idx, current_prediction in enumerate(current_predictions_list):\n",
        "        # Each time a positive is encountered, compute the current precition and the area under the curve\n",
        "        # since the last positive\n",
        "        if current_prediction in current_ground_truth_query_set:\n",
        "          encountered_positives += 1\n",
        "          current_precision = encountered_positives / (idx + 1)\n",
        "          current_ap += current_precision * delta_recall\n",
        "\n",
        "      m_ap += current_ap\n",
        "    # Compute mean over all queries\n",
        "    m_ap /= len(self.reid_target)\n",
        "    return m_ap"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CR6DjmZj0SF7"
      },
      "source": [
        "# Logging functions for the various prints at each epoch\n",
        "def log_tensorboard(prefix, writer, step, loss, metric):\n",
        "  writer.add_scalar(f\"{prefix}/loss\", loss, step)\n",
        "  writer.add_scalar(f\"{prefix}/accuracy\", metric.acc().mean().cpu().numpy(), step)\n",
        "  writer.add_scalar(f\"{prefix}/mAccuracy\", metric.mAcc().mean().cpu().numpy(), step)\n",
        "\n",
        "def log(tr_loss, val_loss, tr_metric, val_metric):\n",
        "  columns = [\"Age\", \"Backpack\", \"Bag\", \"Handbag\", \"Cloth\", \"Down\", \"Up\", \"Hair\", \"Hat\", \"Gender\", \"UpColor\", \"DownColor\"]\n",
        "  print(\"==== ATTRIBUTES: Global Accuracy % ====\")\n",
        "  df = pd.DataFrame.from_dict({\"Train\": tr_metric.acc().cpu().numpy() * 100, \"Val\": val_metric.acc().cpu().numpy() * 100}, columns=columns, orient='index')\n",
        "  pd.set_option('display.max_columns', None)\n",
        "  pd.set_option(\"display.precision\", 1)\n",
        "  print(df)\n",
        "  print(\"Train average Acc:\", tr_metric.acc().mean().cpu().numpy())\n",
        "  print(\"Val average Acc:\", val_metric.acc().mean().cpu().numpy())\n",
        "  print(\"==== ATTRIBUTES: Class Averaged Accuracy % ====\")\n",
        "  df = pd.DataFrame.from_dict({\"Train\": tr_metric.mAcc().cpu().numpy() * 100, \"Val\": val_metric.mAcc().cpu().numpy() * 100}, columns=columns, orient='index')\n",
        "  pd.set_option('display.max_columns', None)\n",
        "  pd.set_option(\"display.precision\", 1)\n",
        "  print(df)\n",
        "  print(\"Train average mAcc:\", tr_metric.mAcc().mean().cpu().numpy())\n",
        "  print(\"Val average mAcc:\", val_metric.mAcc().mean().cpu().numpy())\n",
        "  print(\"==== RE-IDENTIFICATION: mean Average Precision % ====\")\n",
        "  print(\"Train mAP:\", tr_metric.reid_mAP())\n",
        "  print(\"Val mAP:\", val_metric.reid_mAP())\n",
        "  print(\"==== Loss ====\")\n",
        "  print(f\"Train: {tr_loss}\")\n",
        "  print(f\"Validation: {val_loss}\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dVV2wtWOLzOE"
      },
      "source": [
        "## Cost functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Yn1KejsaTUI"
      },
      "source": [
        "### Focal loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XLM_yMXEZho5"
      },
      "source": [
        "class FocalLoss(torch.nn.Module):\n",
        "  \"\"\"\n",
        "  Implementation of the Focal Loss introduced in \tarXiv:1708.02002v2\n",
        "  \"\"\"\n",
        "  def __init__(self, gamma=2):\n",
        "    super().__init__()\n",
        "    self.gamma = gamma\n",
        "  def forward(self, input: torch.Tensor, target: torch.Tensor, weight=None) -> torch.Tensor:\n",
        "    '''weight is equivalent to the alpha parameter of Focal loss. It controls the\n",
        "    importance of the sample, taking into consideration the class imbalance.\n",
        "    (1-p)**gamma controls instead the difficulty of the sample, reducing the\n",
        "    importance of easy examples'''\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    cE = torch.nn.CrossEntropyLoss()\n",
        "    ce_loss = cE(input,target.clone().detach())\n",
        "    # Get back the probability confidence of the network on the correct samples\n",
        "    p = torch.exp(-ce_loss)\n",
        "    # The more common is the class, the less the sample matters (alpha)\n",
        "    if weight is None:\n",
        "      alphas = torch.tensor(1)\n",
        "    else:\n",
        "      # Convert the targets in the corresponding class weights\n",
        "      alphas = weight.gather(0, target)\n",
        "    # The more confident is the network, the less the sample matters (gamma)\n",
        "    focal_losses = alphas * (1-p)**self.gamma * ce_loss\n",
        "    # Reduction\n",
        "    focal_loss = focal_losses.sum() / alphas.sum()\n",
        "    return focal_loss.to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1rgERlPyf5tp"
      },
      "source": [
        "### Multitask aggregators\n",
        "These losses aggregators use different loss weighting to combine together the losses computed on multiple tasks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yYzU7jHtcTTb"
      },
      "source": [
        "class MultitaskAggregator(torch.nn.Module):\n",
        "  \"\"\"\n",
        "    A generic class for multitask aggregators, defines a common taskwise_losses\n",
        "    function that compute all the losses in the several tasks defined by tasks_splits.\n",
        "    Should be subclassed by an aggregator that defines how to aggregate the losses.\n",
        "\n",
        "    :param loss_fn (function) The loss function used to compute the task-wise losses\n",
        "    :param tasks_splits (Tensor[int]) A tensor of size (T) where T = number of tasks,\n",
        "      which contains the number of classes of each task.\n",
        "    :param weight (Tensor[int]) A tensor of size (C*T) where C = n. of \n",
        "      classes of each task, T = number of tasks. It should contain the weights of\n",
        "      the classes for each task, normalized across single tasks.\n",
        "  \"\"\"\n",
        "  def __init__(self, loss_fn,\n",
        "               weight: torch.Tensor = None,\n",
        "               tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10]):\n",
        "    super().__init__()\n",
        "    self.tasks_count = len(tasks_splits)\n",
        "    self.tasks_splits = tasks_splits\n",
        "    self.loss_fn = loss_fn\n",
        "    self.weight = weight\n",
        "  \n",
        "  def aggregate(self, losses: torch.Tensor):\n",
        "    return losses\n",
        "\n",
        "  def forward(self, input: torch.Tensor, target: torch.Tensor):\n",
        "    \"\"\"\n",
        "      :param input (Tensor) A tensor of size (N, C*T) where N = size of the \n",
        "        minibatch, C = n. of classes of each task, T = number of tasks. It is a\n",
        "        one-hot encoding of the prediction.\n",
        "      :param target (Tensor[int]) A tensor of size (N, T) where N = size of the \n",
        "        minibatch, T = number of tasks. It contains the correct class for each task.\n",
        "    \"\"\"\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    # Divide the input in T columns of width C\n",
        "    task_inputs = torch.split(input, [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10], dim=-1)\n",
        "    # Divide the target into T columns of width 1\n",
        "    task_targets = torch.split(target, 1, dim=-1)\n",
        "    # If there aren't weights, a None for each task. Otherwise, divide in chunks\n",
        "    if self.weight is None:\n",
        "      task_weight = [None for _ in self.tasks_splits]\n",
        "    else: \n",
        "      task_weight = torch.split(self.weight, self.tasks_splits)\n",
        "\n",
        "    losses = torch.hstack([\n",
        "      self.loss_fn(t_input, t_target.flatten().clone().detach())\n",
        "      for t_input, t_target in zip(task_inputs, task_targets)\n",
        "    ])\n",
        "    return self.aggregate(losses)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OPw4XokSerO2"
      },
      "source": [
        "#### UniformMultitask"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MEqqVl2MfQxE"
      },
      "source": [
        "class UniformMultitask(MultitaskAggregator):\n",
        "  \"\"\"\n",
        "    A simple aggregators that uniformely sums all the losses together\n",
        "  \"\"\"\n",
        "  def __init__(self, loss_fn,\n",
        "               weight: torch.Tensor = None,\n",
        "               tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10]):\n",
        "    super().__init__(loss_fn, weight, tasks_splits)\n",
        "\n",
        "  def aggregate(self, losses: torch.Tensor):\n",
        "    return losses.sum()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HjG5oiBfIfGo"
      },
      "source": [
        "#### UncertMultitask"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1PWnzhi8j3k8"
      },
      "source": [
        "class UncertMultitask(MultitaskAggregator):\n",
        "  \"\"\"\n",
        "    [Task Uncertanty Weighting]\n",
        "    Originally, a CrossEntropy loss function that can be used for Multi Task Learning.\n",
        "    Can be used with different losses.\n",
        "    It employs an uncertainty-based weighting approach to combine the losses of\n",
        "    the various tasks, as described in the paper\n",
        "    (Auxiliary Tasks in Multi-task Learning)[https://arxiv.org/pdf/1805.06334.pdf].\n",
        "  \"\"\"\n",
        "  def __init__(self, loss_fn,\n",
        "               weight: torch.Tensor = None,\n",
        "               tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10]):\n",
        "    super().__init__(loss_fn, weight, tasks_splits)\n",
        "\n",
        "    #Sum to 1, based on the number of classes per each task\n",
        "    # num_class*12/41\n",
        "    self.params = torch.nn.Parameter(torch.ones(self.tasks_count))\n",
        "\n",
        "  def aggregate(self, losses: torch.Tensor):\n",
        "    return (0.5 / (self.params ** 2) * losses + torch.log(1 + self.params ** 2)).sum()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P0opeEXeIiyS"
      },
      "source": [
        "#### LBTWMultitask"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J9ept7SdoH4M"
      },
      "source": [
        "class LBTWMultitask(MultitaskAggregator):\n",
        "  \"\"\"\n",
        "    [Loss-Balanced Task Weighting]\n",
        "    A method to balance several loss functions for a Multi Task Learning scenario. \n",
        "    It employs an learning speed-based weighting approach to combine the losses of\n",
        "    the various tasks, as described in the paper:\n",
        "    (Loss-Balanced Task Weighting to Reduce Negative Transfer in Multi-Task Learning)[https://doi.org/10.1609/aaai.v33i01.33019977].\n",
        "    \n",
        "    The aim is to reduce the negative transfer by limiting the influence of the\n",
        "    tasks dominating the training process.\n",
        "    Poorly trained tasks have ratios closer to 1, contribuiting more to the final loss.\n",
        "    It's important to call epoch_reset() after each epoch to reset the baseline losses values.\n",
        "\n",
        "    :param alpha(float) An hyperparameter to control the influence of the weights\n",
        "  \"\"\"\n",
        "  def __init__(self, loss_fn,\n",
        "               weight: torch.Tensor = None, \n",
        "               tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10], \n",
        "               alpha=.5):\n",
        "    super().__init__(loss_fn, weight, tasks_splits)\n",
        "    self.alpha = alpha\n",
        "    self.eps = 1e-10\n",
        "    self.baseline_losses = None\n",
        "  \n",
        "  def epoch_reset(self):\n",
        "    \"\"\"Has to be called at the end of every epoch, to reset the baseline values\"\"\"\n",
        "    self.baseline_losses = None\n",
        "\n",
        "  def aggregate(self, losses: torch.Tensor):\n",
        "    # If the computed loss is on the first batch of the current epoch, save it: l(0,i)\n",
        "    if self.baseline_losses is None:\n",
        "      self.baseline_losses = losses.detach()\n",
        "    # Compute the losses task weights\n",
        "    weights = (losses.detach() / (self.baseline_losses + self.eps)) ** self.alpha\n",
        "    return (losses * weights).sum()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mHXZDYmMcSZJ"
      },
      "source": [
        "####DTPMultitask"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rbWYc1zRcjfx"
      },
      "source": [
        "class DTPMultitask(MultitaskAggregator):\n",
        "  \"\"\"\n",
        "    [Dynamic Task Prioritization]\n",
        "    This loss functions balance strategy is inspired by the same strategy used by\n",
        "    the focal loss. For each task, a meaningful metric called Key Performance Indicator \n",
        "    (KPI) is chosen, which is used to determine the dynamic evolution of the difficulty\n",
        "    of the tasks as the training progresses. The loss of the most difficult tasks\n",
        "    is increased, while the loss of the better learned tasks is downweighted.\n",
        "\n",
        "    (Dynamic Task Prioritization for Multitask Learning)[https://openaccess.thecvf.com/content_ECCV_2018/papers/Michelle_Guo_Focus_on_the_ECCV_2018_paper.pdf].\n",
        "\n",
        "    :param gamma(float) An hyperparameter to control the reweighting factor\n",
        "    :param kpi_metric('mAcc'|'mAcc_norm') Which metric should be used as KPI value\n",
        "    :param df(float) The discount factor for the KPI update\n",
        "  \"\"\"\n",
        "  def __init__(self, loss_fn,\n",
        "               weight: torch.Tensor = None, \n",
        "               tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10], \n",
        "               gamma=1, df=.7, kpi_metric='mAcc_norm', kpi_norm=False, device='cuda'):\n",
        "    super().__init__(loss_fn, weight, tasks_splits)\n",
        "    self.gamma = gamma\n",
        "    self.df = df\n",
        "    self.metrics = Metrics(device=device)\n",
        "    self.kpi_metric = kpi_metric\n",
        "    self.kpi_norm = kpi_norm\n",
        "    self.eps = 1e-10\n",
        "    self.kpi = None  # Uninitialized\n",
        "  \n",
        "  def forward(self, input: torch.Tensor, target: torch.Tensor):\n",
        "    \"\"\"Needs to be overridden in order to compute the metrics\"\"\"\n",
        "    predicted = prediction_to_target(input)\n",
        "    self.metrics.add(predicted, target)\n",
        "    return super().forward(input, target)\n",
        "\n",
        "  def aggregate(self, losses: torch.Tensor):\n",
        "    # Compute the new KPI on arbitrary metric\n",
        "    if self.kpi_metric == 'mAcc_norm':\n",
        "      new_kpi = self.metrics.mAcc_norm()\n",
        "    elif self.kpi_metric == 'mAcc':\n",
        "      new_kpi = self.metrics.mAcc()\n",
        "    else:\n",
        "      raise TypeError(\"Invalid 'kpi_metric' value\")\n",
        "    # Compute the next KPI values as an exponential moving average from new kpi computed from mAcc and the previous values\n",
        "    if self.kpi is None:\n",
        "      self.kpi = new_kpi\n",
        "    else:\n",
        "      self.kpi = self.df * new_kpi + (1 - self.df) * self.kpi\n",
        "    # Compute the difficulty using the Focal Loss strategy\n",
        "    d = (1 - self.kpi)**self.gamma * -torch.log(self.kpi + self.eps)\n",
        "    if self.kpi_norm:\n",
        "      # Applies a normalization on the difficulties such that only the relative scale is taken into account (our contribution)\n",
        "      d = d / d.mean()\n",
        "    # Reset the new kpi computation after every training iteration \n",
        "    self.metrics.clear()\n",
        "\n",
        "    return (losses * d).sum()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rf5CVZFbg5uw"
      },
      "source": [
        "### Aggregated losses"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "These classes are in order to initialize the various possible losses for the attribute prediction task"
      ],
      "metadata": {
        "id": "C0UF9bMb6sQc"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sfFj4Vjch-_j"
      },
      "source": [
        "class UncertCrossEntropy(UncertMultitask):\n",
        "  def __init__(self, tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10],\n",
        "               weight: torch.Tensor = None, learnable=True):\n",
        "    super().__init__(F.cross_entropy, tasks_splits=tasks_splits, weight=weight)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o6TOXaN0m1LL"
      },
      "source": [
        "class LBTWCrossEntropy(LBTWMultitask):\n",
        "  def __init__(self, tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10],\n",
        "               weight: torch.Tensor = None, alpha=.5):\n",
        "    super().__init__(F.cross_entropy, tasks_splits=tasks_splits, weight=weight, alpha=alpha)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ofvK_l2njrQz"
      },
      "source": [
        "class UncertFocal(UncertMultitask):\n",
        "  def __init__(self, gamma=2, \n",
        "               tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10],\n",
        "               weight: torch.Tensor = None, learnable=True):\n",
        "    super().__init__(FocalLoss(gamma), tasks_splits=tasks_splits, weight=weight)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F7ZjyT1FmSL-"
      },
      "source": [
        "class LBTWFocal(LBTWMultitask):\n",
        "  def __init__(self, gamma=2, \n",
        "               tasks_splits: list = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10],\n",
        "               weight: torch.Tensor = None, alpha=.5):\n",
        "    super().__init__(FocalLoss(gamma), tasks_splits=tasks_splits, weight=weight, alpha=alpha)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XwuRBAVb9Ops"
      },
      "source": [
        "## Cost functions (Re-ID)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dTBLSgJo9svC"
      },
      "source": [
        "### Center loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BBHC8sDgnjC-"
      },
      "source": [
        "class CenterLoss(torch.nn.Module):\n",
        "  \"\"\"\"\n",
        "  Center Loss (https://doi.org/10.1007/978-3-319-46478-7_31)\n",
        "  An auxiliary loss that learns a center for each class (PIDs in our case); \n",
        "  embeddings that are far away from their respective class center are penalized, \n",
        "  so the similar samples tend to assume a spherical shape in the features space.\n",
        "  It has to be paired to an actual loss, and it should be downweighted w.r.t it\n",
        "  by adopting an appropriate hyperparameter.\n",
        "\n",
        "  IMPORTANT: Needs to be learnt, so its parameters need to be passed to an \n",
        "    optimizer. A possible learning rate is .5\n",
        "\n",
        "  :num_classes(int) Number of classes of the classification problem\n",
        "  :dim_embedding(int) Dimension of the embedded images\n",
        "  \"\"\"\n",
        "  def __init__(self, num_classes, dim_embedding):\n",
        "    super().__init__()\n",
        "    self.num_classes = num_classes\n",
        "    self.dim_embedding = dim_embedding\n",
        "    self.centers = torch.nn.Parameter(torch.randn(self.num_classes, self.dim_embedding))\n",
        "\n",
        "  def forward(self, embeddings, labels):\n",
        "    # Compute the squared 2-norm between the embeddings and the center corresponding to their label\n",
        "    dist = .5 * (embeddings - self.centers[labels]).pow(2).sum(dim=-1)\n",
        "    # Restrict the loss for numerical stability\n",
        "    loss = torch.clamp(dist, min=1e-12, max=1e+12).mean(dim=-1)\n",
        "    return loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ly9slEkK4GJv"
      },
      "source": [
        "#### Island Loss\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I9KdZexBdcpS"
      },
      "source": [
        "class IslandLoss(CenterLoss):\n",
        "  \"\"\"\"\n",
        "  Island Loss (https://arxiv.org/pdf/1710.03144v3.pdf)\n",
        "  An auxiliary loss that complements the Center Loss with an additional term\n",
        "  that penalizes normalized centers that are close to each other. So only their\n",
        "  degree is taken into account, not the magnitude. The resulting clusters tend\n",
        "  to be more distributed in the feature space\n",
        "\n",
        "  IMPORTANT: Needs to be learnt, see CenterLoss for more details\n",
        "\n",
        "  :num_classes(int) Number of classes of the classification problem\n",
        "  :dim_embedding(int) Dimension of the embedded images\n",
        "  :omega_island(float) The hyperparameter that controls the influence of the \n",
        "    island term with respect to the Center Loss\n",
        "  \"\"\"\n",
        "  def __init__(self, num_classes, dim_embedding, omega_island=1.5):\n",
        "    super().__init__(num_classes, dim_embedding)\n",
        "    self.omega_island = omega_island\n",
        "\n",
        "  def forward(self, embeddings, labels):\n",
        "    # Compute the Center Loss from the superclass\n",
        "    center_loss = super().forward(embeddings, labels)\n",
        "    # ISLAND LOSS\n",
        "    # Compute the normalized centers\n",
        "    norm_centers = F.normalize(self.centers)\n",
        "    # Compute the sum of pairwise distances (+1) between the normalized centers\n",
        "    dists = norm_centers.mm(norm_centers.t()) + 1\n",
        "    dists = dists.triu(diagonal=1)\n",
        "    island_term = dists.sum()\n",
        "    return center_loss + self.omega_island * island_term"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "spBjl0Ot-BXS"
      },
      "source": [
        "### Centroid Triplet Loss"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This loss function was used only during training"
      ],
      "metadata": {
        "id": "S9EByXnu7kBL"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RpPF0jyo73v3"
      },
      "source": [
        "class CentroidTripletLoss(torch.nn.Module):\n",
        "  \"\"\"\n",
        "  Centroid Triplet Loss (https://arxiv.org/pdf/2104.13643v1.pdf)\n",
        "  \"\"\"\n",
        "  def __init__(self, margin=1):\n",
        "    super().__init__()\n",
        "    self.margin = margin\n",
        "  \n",
        "  def forward(self, embeddings, labels):\n",
        "    # Normalize the labels into a continuous range. This permits to simplify computations\n",
        "    u_labels = torch.unique(labels)\n",
        "    translate = {label: i for i, label in enumerate(u_labels.tolist())}\n",
        "    labels = torch.tensor([translate[label] for label in labels.tolist()])\n",
        "    num_classes = u_labels.numel()\n",
        "    # Compute the centroids for each class through matrix multiplication\n",
        "    M = torch.zeros(num_classes, len(embeddings), device=embeddings.device)\n",
        "    ## Columns are the one-hot encodings of the labels of the samples\n",
        "    M[labels, torch.arange(len(embeddings))] = 1\n",
        "    ## Normalize across the rows, to have the correct weights for the mean centroid\n",
        "    M = F.normalize(M, p=1)\n",
        "    ## Centroids for each class. Only classes in the batch are present\n",
        "    centroids = M.mm(embeddings)\n",
        "    # Positives are the same-class centroids\n",
        "    positive = centroids[labels]\n",
        "    # Negatives are other-class nearest (to the same-class centroid) centroids\n",
        "    dists = torch.cdist(positive, centroids)\n",
        "    idx = torch.topk(dists, 2, largest=False).indices[:, 1]\n",
        "    negative = centroids[idx]\n",
        "    # Compute the triplet loss w.r.t the centroids\n",
        "    loss = F.triplet_margin_loss(embeddings, positive, negative, margin=self.margin)\n",
        "\n",
        "    return loss"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aiF3C08q6FMS"
      },
      "source": [
        "### JointLoss\n",
        "An aggregator of all the losses for the multitask scenario. Assumes that the outputs are in the form returned by the JointNet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T3mxL8qh6U_M"
      },
      "source": [
        "# Loss function combination designed to work with the outputs of the JointNet\n",
        "class JointLoss(torch.nn.Module):\n",
        "  def __init__(self,dim_embedding, attr_loss, \n",
        "               triplet_lambda = 0.7, pids_lambda = 1, margin=0.4):\n",
        "    super().__init__()\n",
        "    # This method of computing the triplet loss, was taken from the pytorch-metric-learning\n",
        "    # documentation. Uses a miner to extract the tuples of indices, and then uses a 3-step\n",
        "    # procedure to compute the loss using a regularizer and a reducer (see https://kevinmusgrave.github.io/pytorch-metric-learning/)\n",
        "    # for more information.\n",
        "    distance_metric = distances.CosineSimilarity()\n",
        "    reducer = reducers.ThresholdReducer(low=0)\n",
        "    self.triplet_loss = losses.TripletMarginLoss(margin=margin, distance=distance_metric, reducer=reducer)\n",
        "    self.miner = miners.TripletMarginMiner(margin=margin, distance=distance_metric, type_of_triplets=\"semihard\")\n",
        "    # Initialize the CrossEntropy Loss for pids classification\n",
        "    self.pids_loss = torch.nn.CrossEntropyLoss()\n",
        "    self.attr_loss = attr_loss\n",
        "    # Define regularization terms for the losses\n",
        "    self.triplet_lambda = triplet_lambda\n",
        "    self.pids_lambda = pids_lambda\n",
        "    # Define learnable parameters for the Loss\n",
        "    self.params = torch.nn.Parameter(torch.ones(3))\n",
        "\n",
        "  def forward(self, outputs, attr_targets, pid_labels):\n",
        "    # Mine for the tuple indices\n",
        "    tuple_indices = self.miner(outputs['rich_embedding'], pid_labels)\n",
        "    # Computation fo the triplet loss\n",
        "    triplet_loss = self.triplet_lambda*self.triplet_loss(outputs['rich_embedding'], pid_labels, tuple_indices)#/2\n",
        "    # Computation fo the attribute loss\n",
        "    attr_loss = self.attr_loss(outputs['attributes'], attr_targets)\n",
        "    # Consider the loss for the person IDs only when they are predicted (training)\n",
        "    if outputs['pid'] is not None:\n",
        "      pid_loss =  (self.pids_loss(outputs['pid'], pid_labels))*self.pids_lambda#/20\n",
        "      losses = torch.hstack([triplet_loss, attr_loss, pid_loss\n",
        "      ])\n",
        "      return (0.5 / (self.params ** 2) * losses + torch.log(1 + self.params ** 2)).sum()\n",
        "    else:\n",
        "      losses = torch.hstack([triplet_loss, attr_loss])\n",
        "      return (0.5 / (self.params[:2] ** 2) * losses + torch.log(1 + self.params[:2] ** 2)).sum()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8faPuKnOrS8w"
      },
      "source": [
        "## Initialization of the weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZDt0tOmcrWkC"
      },
      "source": [
        "def initialize_weights(model, nonlinearity='leaky_relu'):\n",
        "  \"\"\"\n",
        "  When calling MyNeuralNetwork.apply(initialize_weights), this function is\n",
        "  recursively applied to every module of the network, initializing correctly\n",
        "  the weights according to the type of model\n",
        "  \"\"\"\n",
        "  if isinstance(model, torch.nn.Conv2d):\n",
        "      torch.nn.init.kaiming_uniform_(model.weight.data, nonlinearity=nonlinearity)\n",
        "      if model.bias is not None:\n",
        "          torch.nn.init.constant_(model.bias.data, 0)\n",
        "  elif isinstance(model, torch.nn.BatchNorm2d):\n",
        "      torch.nn.init.constant_(model.weight.data, 1)\n",
        "      torch.nn.init.constant_(model.bias.data, 0)\n",
        "  elif isinstance(model, torch.nn.Linear):\n",
        "      torch.nn.init.kaiming_uniform_(model.weight.data, nonlinearity=nonlinearity)\n",
        "      if model.bias is not None:\n",
        "        torch.nn.init.constant_(model.bias.data, 0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rDwgvNrKLGDP"
      },
      "source": [
        "# Neural Networks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CirLtOUxxsQI"
      },
      "source": [
        "Multi-head network: a network that given a common backbone, splits the input with other nets and returns the concatenated outputs. All heads need to be added to a ModuleList, so they can be correctly registered.\n",
        "\n",
        "Inspired by: https://stackoverflow.com/questions/59763775/how-to-use-pytorch-to-construct-multi-task-dnn-e-g-for-more-than-100-tasks"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###JointNet"
      ],
      "metadata": {
        "id": "WJXZKgsy2riG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class JointNet(torch.nn.Module):\n",
        "  \"\"\"\n",
        "  \n",
        "  :params backbone (nn.Module) The initial part of the network, that extracts\n",
        "    the features from the image, creating the embedding\n",
        "  :params attr_classifiers (list[nn.Module]) The list of classifiers used to \n",
        "    predict the various attributes from the embeddings\n",
        "  :params pid_classifier (nn.Module) The classifier used to predict \n",
        "  \"\"\"\n",
        "  def __init__(self, backbone, attr_classifiers, pid_classifier):\n",
        "    super().__init__()\n",
        "    self.task_splits = torch.tensor([4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10])\n",
        "    self.attr_pred_size = self.task_splits.sum()\n",
        "\n",
        "    self.attr_reweight = AttrReweight(self.attr_pred_size)\n",
        "    \n",
        "    self.backbone = backbone\n",
        "    self.multihead = MultiHead(torch.nn.Identity(), attr_classifiers)\n",
        "    self.pid_classifier = pid_classifier\n",
        "\n",
        "    \n",
        "  def forward(self, x):\n",
        "    embedding = self.backbone(x)\n",
        "    attributes = self.multihead(embedding)\n",
        "    reweight_attributes = self.attr_reweight(attributes)\n",
        "    rich_embedding = torch.hstack([reweight_attributes, embedding])\n",
        "    # The person IDs are computed only when in training mode\n",
        "    if self.training:\n",
        "      pid = self.pid_classifier(rich_embedding)\n",
        "    else:\n",
        "      pid = None\n",
        "    return {\n",
        "      'embedding': embedding,\n",
        "      'attributes': attributes,\n",
        "      'reweight_attributes': reweight_attributes,\n",
        "      'rich_embedding': rich_embedding,\n",
        "      'pid': pid\n",
        "    }"
      ],
      "metadata": {
        "id": "0lGmSWTO2nzv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##JointConvNet"
      ],
      "metadata": {
        "id": "-pf40Mvh3HAg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class ConvBlock(torch.nn.Module):\n",
        "  def __init__(self, in_spatial_shape, in_features, out_features_conv, out_features_linear, kernel_size):\n",
        "    super(ConvBlock, self).__init__()\n",
        "    self.conv = torch.nn.Conv2d(in_features, out_features_conv, kernel_size=kernel_size, stride=(2, 2), padding=(1, 1), bias=False)\n",
        "    self.bn = torch.nn.BatchNorm2d(out_features_conv)\n",
        "    #If initial operations\n",
        "    self.fc1 = torch.nn.Linear(in_features = 1024, out_features = 128)\n",
        "    self.fc2 = torch.nn.Linear(in_features = 128, out_features = out_features_linear)\n",
        "\n",
        "  def forward(self, x):\n",
        "    y = self.conv(x)\n",
        "    x = F.relu(self.bn(y))\n",
        "    x = x.view(x.size(0), -1)  \n",
        "    dr = torch.nn.Dropout(.6)\n",
        "    x = dr(x)\n",
        "    x = self.fc1(x)\n",
        "    return self.fc2(x)"
      ],
      "metadata": {
        "id": "kiMUlSEQYs-w"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class JointConvNet(torch.nn.Module):\n",
        "  \"\"\"\n",
        "  \n",
        "  :params backbone (nn.Module) The initial part of the network, that extracts\n",
        "    the features from the image, creating the embedding\n",
        "  :params attr_classifiers (list[nn.Module]) The list of classifiers used to \n",
        "    predict the various attributes from the embeddings\n",
        "  :params pid_classifier (nn.Module) The classifier used to predict \n",
        "  \"\"\"\n",
        "  def __init__(self, backbone, attr_classifiers, pid_classifier):\n",
        "    super().__init__()\n",
        "    self.task_splits = torch.tensor([4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10])\n",
        "    attr_pred_size = self.task_splits.sum()\n",
        "    self.attr_reweight = AttrReweight(attr_pred_size)\n",
        "    self.backbone = backbone\n",
        "    self.multihead = MultiHead(torch.nn.Identity(), attr_classifiers)\n",
        "    self.pid_classifier = pid_classifier\n",
        "    self.first_time = True\n",
        "    self.conv_activation = None\n",
        "  \n",
        "  \n",
        "  def conv_act(self, model, input, output): \n",
        "    self.conv_activation = output\n",
        "    \n",
        "  def forward(self, x):\n",
        "    if self.first_time :\n",
        "          self.first_time=False\n",
        "          self.backbone.layer3[-1].relu.register_forward_hook(self.conv_act)\n",
        "    embedding = self.backbone(x)\n",
        "    attributes = self.multihead(self.conv_activation)\n",
        "    \n",
        "    reweight_attributes = self.attr_reweight(attributes)\n",
        "    rich_embedding = torch.hstack([reweight_attributes, embedding])\n",
        "    \n",
        "    #The person IDs are computed only when in training mode\n",
        "    if self.training:\n",
        "       pid = self.pid_classifier(rich_embedding)\n",
        "    else:\n",
        "       pid = None\n",
        "\n",
        "    return {\n",
        "      'embedding': embedding,\n",
        "      'attributes': attributes,\n",
        "      'reweight_attributes': reweight_attributes,\n",
        "      'rich_embedding': rich_embedding,\n",
        "      'pid': pid\n",
        "    }\n",
        "  "
      ],
      "metadata": {
        "id": "ei2MVGKh3EUh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Aniro_tjk_ZP"
      },
      "source": [
        "### MultiHead architecture"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HANhB1tBlWZo"
      },
      "source": [
        "class MultiHead(torch.nn.Module):\n",
        "  def __init__(self, backbone, heads):\n",
        "    super().__init__()\n",
        "    self.backbone = backbone\n",
        "    # Initializing all the heads as part of a ModuleList\n",
        "    self.heads = torch.nn.ModuleList(heads)\n",
        "\n",
        "  def forward(self, x):\n",
        "    common_features = self.backbone(x)  # compute the shared features\n",
        "    outputs = [head(common_features) for head in self.heads]\n",
        "    outputs = torch.cat(outputs, dim=1)\n",
        "    return outputs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Backbone"
      ],
      "metadata": {
        "id": "jfKTJX-23iaa"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "####Trained-from-scratch backbone"
      ],
      "metadata": {
        "id": "u_LKEUdh1gYN"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aXMoF8DBk0cs"
      },
      "source": [
        "##### SE-DenseNet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TkZIr2QHFo8X"
      },
      "source": [
        "class BasicLayer(torch.nn.Sequential):\n",
        "  def __init__(self, in_channels: int, out_channels: int, kernel_size: int, stride=1, droprate=0., extra_modules=[]):\n",
        "    super().__init__(\n",
        "      torch.nn.BatchNorm2d(in_channels),\n",
        "      torch.nn.LeakyReLU(),\n",
        "      torch.nn.Conv2d(in_channels, out_channels, kernel_size,\n",
        "                      stride=stride, padding=kernel_size // 2, bias=False),\n",
        "      torch.nn.Dropout(droprate),\n",
        "      *extra_modules\n",
        "    )\n",
        "\n",
        "class BottleneckLayer(BasicLayer):\n",
        "  def __init__(self, in_channels: int, out_channels: int, droprate=0.):\n",
        "    super().__init__(in_channels, out_channels, 1, droprate=droprate)\n",
        "\n",
        "class CompositionLayer(BasicLayer):\n",
        "  def __init__(self, in_channels: int, out_channels: int, droprate=0.):\n",
        "    super().__init__(in_channels, out_channels, 3, droprate=droprate)\n",
        "\n",
        "class TransitionBlock(BasicLayer):\n",
        "  def __init__(self, in_channels: int, compression, droprate=0.):\n",
        "    out_channels = math.floor(in_channels * compression)\n",
        "    super().__init__(in_channels, out_channels, 1, droprate=droprate,\n",
        "      extra_modules=[torch.nn.AvgPool2d(kernel_size=2)])\n",
        "\n",
        "class DenseLayer(torch.nn.Module):\n",
        "  def __init__(self, in_channels: int, growth_rate: int, bn_size: int, memory_efficient: bool, droprate=0.):\n",
        "    super().__init__()\n",
        "    self.memory_efficient = memory_efficient\n",
        "    self.bottleneck = BottleneckLayer(in_channels, bn_size * growth_rate, droprate=droprate)\n",
        "    self.composition = CompositionLayer(bn_size * growth_rate, growth_rate, droprate=droprate)\n",
        "  def forward(self, x):\n",
        "    if self.memory_efficient and x.requires_grad:\n",
        "      x = cp.checkpoint((lambda x: self.bottleneck(x)), x)\n",
        "    else:\n",
        "      x = self.bottleneck(x)\n",
        "    x = self.composition(x)\n",
        "    return x\n",
        "\n",
        "class DenseBlock(torch.nn.Module):\n",
        "  def __init__(self, layers_count: int, in_channels: int, growth_rate: int, bn_size: int, memory_efficient: bool, use_SE: bool, droprate=0.):\n",
        "    super().__init__()\n",
        "    self.layers = torch.nn.ModuleList([\n",
        "      DenseLayer(in_channels + i * growth_rate, growth_rate, bn_size, memory_efficient, droprate=droprate) \n",
        "      for i in range(layers_count)\n",
        "    ])\n",
        "    self.use_SE = use_SE\n",
        "    if self.use_SE:\n",
        "      out_channels = in_channels + layers_count * growth_rate\n",
        "      self.se = SEBlock(out_channels)\n",
        "\n",
        "  def forward(self, x):\n",
        "    for layer in self.layers:\n",
        "      new_features = layer(x)\n",
        "      x = torch.cat((x, new_features), 1)\n",
        "    if self.use_SE:\n",
        "      x = self.se(x)\n",
        "    return x\n",
        "\n",
        "class ClassificationLayer(torch.nn.Sequential):\n",
        "  def __init__(self, in_channels, classes_count):\n",
        "    super().__init__(\n",
        "      torch.nn.ReLU(),\n",
        "      torch.nn.AdaptiveAvgPool2d(1),\n",
        "      torch.nn.Flatten(),\n",
        "      torch.nn.Linear(in_channels, classes_count)\n",
        "    )\n",
        "\n",
        "class MTDenseNet(MultiHead):\n",
        "  def __init__(self, starting_channels: int, backbone_config: tuple, head_configs: list,\n",
        "               tasks_splits = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10],\n",
        "               growth_rate=32, bn_size=4, compression=.5, droprate=0., memory_efficient=False, use_SE=False):\n",
        "    # The backbone starts with an initial convolution and average pooling\n",
        "    backbone = [\n",
        "      BasicLayer(3, starting_channels, 7, stride=2),\n",
        "      torch.nn.AvgPool2d(kernel_size=3, stride=2, padding=1)\n",
        "    ]\n",
        "    # And continues with a series of dense blocks with the number of layers specified by backbone_config\n",
        "    channels = starting_channels\n",
        "    for layers_count in backbone_config:\n",
        "      backbone.append(DenseBlock(layers_count, channels, growth_rate, bn_size, memory_efficient, droprate=droprate, use_SE=use_SE))\n",
        "      channels += layers_count * growth_rate\n",
        "      backbone.append(TransitionBlock(channels, compression, droprate=droprate))\n",
        "      channels = math.floor(channels * compression)\n",
        "    # heads is the list holding every task-specific part of the network\n",
        "    heads = []\n",
        "    final_backbone_channels = channels\n",
        "    for head_config, classes_count in zip(head_configs, tasks_splits):\n",
        "      channels = final_backbone_channels\n",
        "      head = []\n",
        "      # Add the dense blocks to the head as specified by this head's head_config\n",
        "      for i, layers_count in enumerate(head_config):\n",
        "        head.append(DenseBlock(layers_count, channels, growth_rate, bn_size, memory_efficient, droprate=droprate, use_SE=use_SE))\n",
        "        channels += layers_count * growth_rate\n",
        "        # Do not append the transition block to the last dense block\n",
        "        if i < len(head_config) - 1:\n",
        "          head.append(TransitionBlock(channels, compression, droprate=droprate))\n",
        "          channels = math.floor(channels * compression)\n",
        "      # Append the final classification layer\n",
        "      head.append(ClassificationLayer(channels, classes_count))\n",
        "      # Add the current head to the heads\n",
        "      heads.append(torch.nn.Sequential(*head))\n",
        "    \n",
        "    super().__init__(torch.nn.Sequential(*backbone), heads)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fk4zTiZJlEnL"
      },
      "source": [
        "###### SEBlock\n",
        "Squeeze-and-Excitation block"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LwaKgV2ulRBf"
      },
      "source": [
        "class SEBlock(torch.nn.Module):\n",
        "  def __init__(self, in_channels: int, ratio=16):\n",
        "    super().__init__()\n",
        "    self.squeeze = torch.nn.AdaptiveAvgPool2d(1)\n",
        "    mid_channels = in_channels // ratio if in_channels // ratio > 0 else 1\n",
        "    self.excitation = torch.nn.Sequential(\n",
        "      torch.nn.Linear(in_channels, mid_channels, bias=False),\n",
        "      torch.nn.ReLU(),\n",
        "      torch.nn.Linear(mid_channels, in_channels, bias=False),\n",
        "      torch.nn.Sigmoid()\n",
        "    )\n",
        "  def forward(self, x):\n",
        "    s = self.squeeze(x).squeeze()\n",
        "    e = self.excitation(s)[..., None, None]\n",
        "    return x * e"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eknbYCPB8zVF"
      },
      "source": [
        "#### Pre-trained backbones"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EL1clRa58zCx"
      },
      "source": [
        "def pretrained_backbone(architecture, hidden_size):\n",
        "  if architecture == 'resnet':\n",
        "    net = models.resnet18(pretrained=True)\n",
        "  elif architecture == 'wide':\n",
        "    net = models.wide_resnet50_2(pretrained=True)\n",
        "  elif architecture == 'res_next':\n",
        "    net = models.resnext50_32x4d(pretrained=True)\n",
        "  elif architecture == 'shufflenet':\n",
        "    net = models.shufflenet_v2_x1_0(pretrained=True)\n",
        "  elif architecture == 'inception':\n",
        "    net = models.inception_v3(pretrained=True)\n",
        "  elif architecture == 'dense':\n",
        "    net = models.densenet161(pretrained=True)\n",
        "  \n",
        "  if architecture == 'dense':\n",
        "    # Different final layer name for the densenet\n",
        "    embedding_size = net.classifier.in_features\n",
        "    net.classifier = torch.nn.Identity()\n",
        "  elif architecture == 'vgg':\n",
        "    embedding_size = net.classifier.features\n",
        "    net.classifier = torch.nn.Identity()\n",
        "  else:\n",
        "    embedding_size = net.fc.in_features\n",
        "    net.fc =  torch.nn.Identity()\n",
        "  \n",
        "  return net, embedding_size"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IkOyLm2bwzfT"
      },
      "source": [
        "##### Attribute Re-weighting module\n",
        "\n",
        "Recalibrate predicted attributes based on a score called confidence score.\n",
        "\n",
        "The confidence score encodes the relationship between the attributes itself.\n",
        "\n",
        "It is computed by means of:\n",
        "\n",
        "1.  a linear layer mapping 41 features into 41 features\n",
        "2.  a sigmoid layer to map the output of the previous layer in the range [0,1]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WHPwD4vzw3nN"
      },
      "source": [
        "class AttrReweight(torch.nn.Module):\n",
        "  \"\"\"\n",
        "  Attribute Re-weighting Module (https://arxiv.org/pdf/1703.07220.pdf)\n",
        "  This module aims to recalibrate the predicted attributes by taking into \n",
        "  account the relationship with the other attributes\n",
        "\n",
        "  :params attr_size (int) \n",
        "          The size of the final attributes prediction vector\n",
        "  :params softmax_flag(Bool) \n",
        "          Convert into probabilities (True) or not (False) the output passed in the forward step\n",
        "  :params task_split(list)\n",
        "          Allow the softmax to convert output into probabilities\n",
        "  \"\"\"\n",
        "  def __init__(self, attr_size, softmax_flag=False, task_split = [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10]):\n",
        "    super().__init__()\n",
        "\n",
        "    self.softmax_flag = softmax_flag\n",
        "    self.task_split = task_split\n",
        "    self.confidence_net = torch.nn.Sequential (\n",
        "              torch.nn.Linear(attr_size, attr_size),\n",
        "              torch.nn.Sigmoid()\n",
        "              )\n",
        "\n",
        "  def forward(self, output):\n",
        "\n",
        "    if self.softmax_flag == True:\n",
        "      #Convert the task_output, a tensor having size: [batch_size, 41]\n",
        "      #into a tuple of 12 tensors \n",
        "      #where the i-th tensor has size: [batch_size, task_split[i]]\n",
        "      task_output = torch.split(output, self.task_split, dim=-1)\n",
        "      #Convert tuple into list, to modify its values\n",
        "      task_output = list(task_output)\n",
        "\n",
        "      softmax = torch.nn.Softmax(dim=1)\n",
        "      for idx,out in enumerate(task_output):\n",
        "        task_output[idx] = softmax(out)\n",
        "      #Convert the list of 12 tensors\n",
        "      #into a single tensor having size: [batch_size, 41]\n",
        "      softmax_output = torch.hstack(task_output)\n",
        "\n",
        "      #Get confidence score from softmax output\n",
        "      confidence = self.confidence_net(softmax_output)\n",
        "\n",
        "    else:\n",
        "      #Get confidence score from output\n",
        "      confidence = self.confidence_net(output)\n",
        "    \n",
        "    #Output recalibration based on confidence\n",
        "    return output * confidence"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ERnPu0xLTdn"
      },
      "source": [
        "# Routines"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wYTHNii8nLj1"
      },
      "source": [
        "## Learning rate finder\n",
        "\n",
        "Existing tools can be used to find ideal values for the learning rate or lr boundaries for the 1Cycle scheduler\n",
        "\n",
        "https://github.com/davidtvs/pytorch-lr-finder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lu8n8ddurlMY"
      },
      "source": [
        "class TrainLRFinderLoader(TrainDataLoaderIter):\n",
        "  def inputs_labels_from_batch(self, batch_data):\n",
        "    return batch_data[\"image\"], batch_data[\"annotations\"]\n",
        "\n",
        "class ValLRFinderLoader(ValDataLoaderIter):\n",
        "  def inputs_labels_from_batch(self, batch_data):\n",
        "    return batch_data[\"image\"], batch_data[\"annotations\"]\n",
        "    \n",
        "def tune_lr(model, data, loss, optimizer, end_lr, num_iter, precise=False):\n",
        "  # torch.backends.cudnn.benchmark = True\n",
        "  lr_finder = LRFinder(model, optimizer, loss, device=\"cuda\")\n",
        "  train_loader = TrainLRFinderLoader(data['train'])\n",
        "  val_loader = ValLRFinderLoader(data['val'])\n",
        "  if precise:\n",
        "    lr_finder.range_test(train_loader, val_loader=val_loader, end_lr=end_lr, num_iter=num_iter, step_mode=\"linear\")\n",
        "    lr_finder.plot(log_lr=False)\n",
        "  else:\n",
        "    lr_finder.range_test(train_loader, end_lr=end_lr, num_iter=num_iter)\n",
        "    lr_finder.plot() # to inspect the loss-learning rate graph\n",
        "  lr_finder.reset() # to reset the model and optimizer to their initial state"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8kXS8NVEoD3f"
      },
      "source": [
        "## Learning\n",
        "Set of functions to learn the weights of the network and compute the metrics on the validation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n1PDGynrzLmo"
      },
      "source": [
        "### Re-identification\n",
        "Re-identification code can be used during the training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kcu_cTIcM5fK"
      },
      "source": [
        "#### Extract queries\n",
        "Function to get a random list of possible queries and the corresponding ground truth reference from a dataset of embeddings. Used to get re-identification scores for training and validation sets.\n",
        "\n",
        "Moreover, the function returns a suggested threshold value: the mean of the cosine distances between each query and the further embedding in the same class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bOn8gb707Zj0"
      },
      "source": [
        "def extract_queries(samples_emb, samples_pid, samples_name, utility, static_threshold = True):\n",
        "  \"\"\"\n",
        "  From a group of embeddings with their names and pids, select randomly the queries\n",
        "  such that there is a single query for each pid.\n",
        "  Additionally, compute a suggested threshold given the current state of the network.\n",
        "\n",
        "  :params samples_emb (torch.Tensor(dtype=float)) Embeddings of available samples\n",
        "  :params samples_pid (torch.Tensor(dtype=int)) Continuous person IDs of available samples\n",
        "  :params samples_name (list[String]) Names of available samples\n",
        "\n",
        "  :return ground_truth, queries_emb, queries_name, suggested_threshold\n",
        "  \"\"\"\n",
        "  pids_count = samples_pid.max() + 1\n",
        "  # List of tensors containing indices of embeddings with the same pid\n",
        "  groups = [(samples_pid == pid).nonzero().squeeze() for pid in range(pids_count)]\n",
        "  # Trim groups that are too small\n",
        "  groups = [group for group in groups if group.numel() > 2]\n",
        "  # Select a random embedding index for every group of same-pid embedding idices\n",
        "  queries = torch.tensor([group[torch.randint(group.numel(), ())] for group in groups])\n",
        "  # Use queries (indexes) to get file names and the embeddings\n",
        "  queries_name = [samples_name[i] for i in queries]\n",
        "  queries_emb = samples_emb[queries]\n",
        "  # Get the sets of file names, excluding the query, from the groups\n",
        "  groups_names = [set(samples_name[i] for i in group if i != query) for query, group in zip(queries, groups)]\n",
        "  # Create the ground truth dictionary\n",
        "  ground_truth = {query_name: group_names for query_name, group_names in zip(queries_name, groups_names)}\n",
        "\n",
        "  if static_threshold == False:\n",
        "    # Compute a suggested threshold: average of maximum cosine distance across for each query\n",
        "    min_dists = torch.tensor([F.cosine_similarity(query_emb[None, ...], samples_emb[group]).min() for query_emb, group in zip(queries_emb, groups)])\n",
        "    suggested_threshold = min_dists.mean()\n",
        "  else:\n",
        "    if utility == 'train':\n",
        "      suggested_threshold = 0.75\n",
        "    else: #utility == 'val'\n",
        "      suggested_threshold=.82\n",
        "\n",
        "  return ground_truth, queries_emb, queries_name, suggested_threshold"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "91j-U7SNy-fk"
      },
      "source": [
        "#### Re-identification"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xHhYMK0OWTKq"
      },
      "source": [
        "def reidentification(queries_emb, queries_name, samples_emb, samples_name, threshold):\n",
        "  \"\"\"\n",
        "  Use the queries embeddings and the test embeddings to create a dictionary where\n",
        "  each query image name is associated to the list of test images names having a\n",
        "  similarity above the defined threshold.\n",
        "  \"\"\"\n",
        "  predicted_pids = {}\n",
        "  for query_name, query_emb in zip(queries_name, queries_emb):\n",
        "    # Compute the cosine similarities between the current query and all embeddings\n",
        "    similarity = F.cosine_similarity(query_emb[None, ...], samples_emb)\n",
        "    # Pick the indices of the embeddings with similartiy >= threshold\n",
        "    matching_indices = (similarity >= threshold).nonzero().squeeze(-1)\n",
        "    matching_indices = matching_indices.tolist()\n",
        "\n",
        "    # Use the indices to select the corresponding file names. Remove the query, if present.\n",
        "    matching_names = [samples_name[i] for i in matching_indices if samples_name[i] != query_name]\n",
        "    # Add to the pids dictionary\n",
        "    predicted_pids[query_name] = matching_names\n",
        "  return predicted_pids"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "clcYJ5ucn6f1"
      },
      "source": [
        "### Train"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_loss(outputs, targets):\n",
        "  task_inputs = torch.split(outputs, [4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10], dim=-1)\n",
        "  # Divide the target into T columns of width 1\n",
        "  task_targets = torch.split(targets, 1, dim=-1)\n",
        "  loss = torch.nn.CrossEntropyLoss()\n",
        "  l = 0\n",
        "  for inpt, trg in zip(task_inputs, task_targets):\n",
        "    l += loss(inpt, trg.clone().detach().flatten())\n",
        "  return l"
      ],
      "metadata": {
        "id": "n6Xf638szr-H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D6BMzlhDOkQ_"
      },
      "source": [
        "def train(net, data_loader, loss, optimizer, scheduler, scaler, device='cuda'):\n",
        "  batch_count = 0\n",
        "  cumulative_loss = 0.\n",
        "  metrics = Metrics(device=device)\n",
        "\n",
        "  samples_emb = []\n",
        "  samples_pid = []\n",
        "  samples_name = []\n",
        "\n",
        "  net.train()\n",
        "  for batch_idx, sample in enumerate(tqdm(data_loader, desc='Training', leave=False)):   \n",
        "    # Load data into GPU\n",
        "    inputs = sample['image'].to(device)\n",
        "    attr_targets = sample['annotations'].to(device)\n",
        "    pid_labels = sample['person_id'].to(device)\n",
        "    # Reset the gradients\n",
        "    optimizer.zero_grad()\n",
        "    # Forward pass with autocasting\n",
        "    with torch.cuda.amp.autocast():\n",
        "      outputs = net(inputs)\n",
        "      loss_result = loss(outputs, attr_targets, pid_labels)\n",
        "    # Loss scaling and backward pass\n",
        "    # loss_result.backward()\n",
        "    scaler.scale(loss_result).backward()\n",
        "    # Optimizer step under scaling\n",
        "    # optimizer.step()\n",
        "    scaler.step(optimizer)\n",
        "    scaler.update()\n",
        "    # Updates the scheduler for next iteration\n",
        "    if scheduler is not None:\n",
        "      scheduler.step()\n",
        "    # Storing execution information\n",
        "    batch_count += 1\n",
        "    cumulative_loss += loss_result.item()\n",
        "    # Attributes evaluation\n",
        "    attr_predict = prediction_to_target(outputs['attributes'])\n",
        "    metrics.add(attr_predict, attr_targets)\n",
        "    # Re-id accumulation\n",
        "    samples_emb.append(outputs['embedding'])\n",
        "    samples_pid.append(pid_labels)\n",
        "    samples_name.extend(sample['img_file'])\n",
        "  # Pack re-id components\n",
        "  samples_emb = torch.vstack(samples_emb)\n",
        "  samples_pid = torch.cat(samples_pid)\n",
        "  # Re-identification\n",
        "  ground_truth, queries_emb, queries_name, threshold = extract_queries(samples_emb, samples_pid, samples_name, static_threshold = True, utility='train')\n",
        "  predicted_pids = reidentification(queries_emb, queries_name, samples_emb, samples_name, threshold)\n",
        "  metrics.add_reid(predicted_pids, ground_truth, threshold)\n",
        "\n",
        "  if isinstance(loss, LBTWFocal):\n",
        "    loss.epoch_reset()\n",
        "\n",
        "  return cumulative_loss/batch_count, metrics"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F5YfRUGMn3QU"
      },
      "source": [
        "### Validate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IVwKyl3QOeZV"
      },
      "source": [
        "def validate(net, data_loader, loss, device='cuda'):\n",
        "  batch_count = 0\n",
        "  cumulative_loss = 0.\n",
        "  metrics = Metrics(device=device)\n",
        "\n",
        "  samples_emb = []\n",
        "  samples_pid = []\n",
        "  samples_name = []\n",
        "\n",
        "  net.eval() # Strictly needed if network contains layers which has different behaviours between train and test\n",
        "  with torch.no_grad():\n",
        "    for batch_idx, sample in enumerate(tqdm(data_loader, desc='Validation', leave=False)):\n",
        "      # Load data into GPU\n",
        "      inputs = sample['image'].to(device)\n",
        "      attr_targets = sample['annotations'].to(device)\n",
        "      pid_labels = sample['person_id'].to(device)\n",
        "      # Forward pass\n",
        "      outputs = net(inputs)\n",
        "      # Apply the loss\n",
        "      loss_result = loss(outputs, attr_targets, pid_labels)\n",
        "\n",
        "      # Storing execution information\n",
        "      batch_count += 1\n",
        "      cumulative_loss += loss_result.item()\n",
        "      # Attributes evaluation\n",
        "      attr_predict = prediction_to_target(outputs['attributes'])\n",
        "      metrics.add(attr_predict, attr_targets)\n",
        "      \n",
        "      samples_emb.append(outputs['embedding'])\n",
        "      samples_pid.append(pid_labels)\n",
        "      samples_name.extend(sample['img_file'])\n",
        "      \n",
        "  samples_emb = torch.vstack(samples_emb)\n",
        "  samples_pid = torch.cat(samples_pid)\n",
        "  # Re-identification\n",
        "  ground_truth, queries_emb, queries_name, threshold = extract_queries(samples_emb, samples_pid, samples_name, static_threshold = True, utility='val')\n",
        "\n",
        "  predicted_pids = reidentification(queries_emb, queries_name, samples_emb, samples_name, threshold)\n",
        "  metrics.add_reid(predicted_pids, ground_truth, threshold)\n",
        "  ##############################################################################################\n",
        "\n",
        "  \n",
        "  if isinstance(loss, LBTWFocal):\n",
        "    loss.epoch_reset()\n",
        "  \n",
        "  return cumulative_loss/batch_count, metrics"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "47ZNJWAS5YQO"
      },
      "source": [
        "### Fit"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class EarlyStopping():\n",
        "  \"\"\"\n",
        "  :params metric ('score'|'loss') Which type of metric is analayzed. If it's a score,\n",
        "  it should be maximized, if it's a loss, it should be minimized\n",
        "  :params patience_epochs (int) Number of epochs to wait before stopping\n",
        "  \"\"\"\n",
        "  #metric can be either 'score' or 'loss'\n",
        "  def __init__(self, metric = 'score', patience_epochs=10):\n",
        "    self.patience_epochs = patience_epochs\n",
        "    # Metric can be either a score (to be maximized) or a loss (to be minimized)\n",
        "    if metric != 'score' and metric != 'loss':\n",
        "      raise ValueError('An invalid \"metric\" paramater has been specified')\n",
        "    self.metric = metric\n",
        "    self.epochs_count = 0\n",
        "    self.wasted_epochs = 0\n",
        "    self.best_metric = None\n",
        "    self.best_epoch = 0\n",
        "\n",
        "  def add_metric(self, new_metric):\n",
        "    self.epochs_count += 1\n",
        "    improved = False\n",
        "    if self.best_metric == None:\n",
        "      # Always improves at the start\n",
        "      improved = True\n",
        "    elif self.metric == 'score' and new_metric > self.best_metric:\n",
        "      # If the score has increased, there is no overfitting\n",
        "      improved = True\n",
        "    elif self.metric == 'loss' and new_metric < self.best_metric:\n",
        "      # If the loss has decreased, there is no overfitting\n",
        "      improved = True\n",
        "    \n",
        "    if improved:\n",
        "      self.wasted_epochs = 0\n",
        "      self.best_epoch = self.epochs_count\n",
        "      self.best_metric = new_metric\n",
        "    else:\n",
        "      self.wasted_epochs += 1\n",
        "    \n",
        "    return self.check_best_model()\n",
        "\n",
        "  def check_best_model(self):\n",
        "    \"\"\" Returns True if the current model is the best model \"\"\"\n",
        "    # If the wasted epochs are 0, the model is currently the best model\n",
        "    return self.wasted_epochs == 0\n",
        "\n",
        "  def check_overfitting(self):\n",
        "    \"\"\" Returns True if it's overfitting \"\"\"\n",
        "    return self.wasted_epochs > self.patience_epochs"
      ],
      "metadata": {
        "id": "N17yJApUDgCx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ojCqWA5hOuen"
      },
      "source": [
        "def fit(model, epochs, max_lr, early_stopping_patience=50):\n",
        "  # Automatically set the best available device\n",
        "  device = torch.device(\"cuda:0\" if torch.cuda.is_available() or False else \"cpu\")\n",
        "  # Create the Early Stopping mechanism\n",
        "  early_stopping = EarlyStopping(metric='score', patience_epochs = early_stopping_patience)\n",
        "\n",
        "  # Creates a logger for the experiment\n",
        "  writer = SummaryWriter(log_dir=\"runs/exp1\")\n",
        "  \n",
        "  # Instantiate a LR scheduler\n",
        "  scheduler = None\n",
        "\n",
        "  # Autoscaler instantiation: downgrades certain values' precision to reduce computations\n",
        "  scaler = torch.cuda.amp.GradScaler()\n",
        "\n",
        "  # For each epoch, train the network and then compute evaluation results\n",
        "  min_val_error = math.inf\n",
        "  wasted_epochs = 0\n",
        "  # Autoscaler instantiation: downgrades certain values' precision to reduce computations\n",
        "  #scaler = torch.cuda.amp.GradScaler()\n",
        "  # Train the epochs\n",
        "  # Save all the validation accuracies for early stopping\n",
        "  accuracies = []\n",
        "  # Train the epochs\n",
        "  for e in range(epochs):\n",
        "    print(f\"---Epoch {e+1}:---------------------------------\")\n",
        "    tr_error, tr_metric = train(model.net, model.train_loader, model.multitask_loss, model.optimizer, scheduler, scaler, device=device)\n",
        "    val_error, val_metric = validate(model.net, model.val_loader, model.multitask_loss, device=device)\n",
        "    log(tr_error, val_error, tr_metric, val_metric)\n",
        "    log_tensorboard(\"Train\", writer, e, tr_error, tr_metric)\n",
        "    log_tensorboard(\"Val\", writer, e, val_error, val_metric)\n",
        "    # Early stopping control\n",
        "    score = val_metric.mAcc().mean() + val_metric.reid_mAP()\n",
        "    early_stopping.add_metric(score)\n",
        "    # Save the model if it's currently the best\n",
        "    if early_stopping.check_best_model():\n",
        "      torch.save(model.net.state_dict(), 'model_weights.pth')\n",
        "    # Stop the training if it's overfitting\n",
        "    if early_stopping.check_overfitting():\n",
        "      print(f\"Training stopped because {early_stopping_patience} epochs passed without improvements\")\n",
        "      break\n",
        "  # Reload the weights of the best model\n",
        "  print(f\"The best weights have been saved as 'model_weights.pth'\")\n",
        "  print(f\"The best epoch was Epoch {early_stopping.best_epoch}\")\n",
        "  model.net.load_state_dict(torch.load('model_weights.pth'))\n",
        "  # Closes the logger\n",
        "  writer.close()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Ksz4FcD3lEV"
      },
      "source": [
        "## Predict"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_fkaE66o3knr"
      },
      "source": [
        "def predict(net, data, threshold=.82):\n",
        "  device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "  net = net.to(device)\n",
        "  test_loader = data[\"test\"]\n",
        "  queries_loader = data[\"queries\"]\n",
        "\n",
        "  samples_attr = []\n",
        "  samples_emb = []\n",
        "  samples_name = []\n",
        "  queries_emb = []\n",
        "  queries_name = []\n",
        "\n",
        "  net.eval()\n",
        "  with torch.no_grad():\n",
        "    for batch in tqdm(test_loader, desc='Test', leave=False):\n",
        "      # Load data into GPU\n",
        "      inputs = batch['image'].to(device)\n",
        "      # Forward pass\n",
        "      outputs = net(inputs)\n",
        "      samples_attr.append(outputs['attributes'])\n",
        "      samples_emb.append(outputs['embedding'])\n",
        "      samples_name.extend(batch['img_file'])\n",
        "    for batch in tqdm(queries_loader, desc='Queries', leave=False):\n",
        "      # Load data into GPU\n",
        "      inputs = batch['image'].to(device)\n",
        "      # Forward pass\n",
        "      outputs = net(inputs)\n",
        "      queries_emb.append(outputs['embedding'])\n",
        "      queries_name.extend(batch['img_file'])\n",
        "  # Pack components\n",
        "  samples_attr = torch.vstack(samples_attr)\n",
        "  samples_emb = torch.vstack(samples_emb)\n",
        "  queries_emb = torch.vstack(queries_emb)\n",
        "  # Re-identification: search the queries in the test set\n",
        "  predicted_pids = reidentification(queries_emb, queries_name, samples_emb, samples_name, threshold)\n",
        "  # Save the classification annotations as a dataframe file\n",
        "  annotations = target_to_annotation(prediction_to_target(samples_attr))\n",
        "  save_attributes(annotations, samples_name)\n",
        "  # Save the re-IDs as a text file\n",
        "  save_reids(predicted_pids)\n",
        "  \n",
        "  return annotations"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F2MS0mvCbCvZ"
      },
      "source": [
        "def save_attributes(annotations, file_names, save_file='classification_test'):\n",
        "  columns = ['age', 'backpack', 'bag', 'handbag', 'clothes', 'down', 'up', 'hair', 'hat', 'gender', 'upblack', 'upwhite', 'upred', 'uppurple', 'upyellow', 'upgray', 'upblue', 'upgreen', 'downblack', 'downwhite', 'downpink', 'downpurple', 'downyellow', 'downgray', 'downblue', 'downgreen', 'downbrown']\n",
        "  index = pd.Index(file_names, name='file')\n",
        "  df = pd.DataFrame(annotations, index=index, columns=columns).astype(int)\n",
        "  df.to_csv(f\"{save_file}.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I7_myQcmn0l-"
      },
      "source": [
        "def save_reids(predicted_pids, save_file='reid_test'):\n",
        "  with open(f\"{save_file}.txt\", mode='w') as f:\n",
        "    for query in predicted_pids:\n",
        "      f.write(f\"{query}: \")\n",
        "      f.write(', '.join(predicted_pids[query]))\n",
        "      f.write('\\n')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o5MvNeWBO-4l"
      },
      "source": [
        "# Execution"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2xN-Iod-PH8c"
      },
      "source": [
        "%load_ext tensorboard\n",
        "%tensorboard --logdir=runs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UCfJ2TKBL1Nf"
      },
      "source": [
        "## Training instances"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3UvOf5wEaACz"
      },
      "source": [
        "Fetch the DataLoaders (only the batch size it set, the splits are already created)\n",
        "- 'full': the Dataloader of the entire dataset\n",
        "- 'train': the Dataloader of the training set\n",
        "- 'val': the Dataloader of the validation set\n",
        "- 'test': the Dataloader of the test set\n",
        "- 'class_weight': the class weights for every task, computed on the training set\n",
        "- 'train_pids_count': number of normalized IDs present in the training set [0, train_pids_count)\n",
        "- 'val_pids_count': number of normalized IDs present in the validation set [0, val_pids_count)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Model Conv + Linear"
      ],
      "metadata": {
        "id": "TJIV3eVtYu5x"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U_FqZIeYdWPq"
      },
      "source": [
        "Optimizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e6lFWslaL05e"
      },
      "source": [
        "def get_optimizer(net, cost_function=None, learning_rate=0.01, betas=(0.9, 0.999), weight_decay=0.01):\n",
        "\n",
        "    return torch.optim.AdamW(\n",
        "      [{'params': net.parameters(), 'lr': learning_rate},\n",
        "       {'params': cost_function.parameters(), 'lr':learning_rate},\n",
        "       ], betas=betas, weight_decay=weight_decay)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lmbr4KF4jtO6"
      },
      "source": [
        "CUDA memory cleaning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ah7IzX-wxrhL"
      },
      "source": [
        "# Clear the CUDA memory\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Model():\n",
        "\n",
        "  def __init__(self, task_splits, batch_size = 128, hidden_size = 256, learning_rate=1e-3, weight_decay=1e-1, expand_images=True, fc_dim=16, shape=(16,16)):\n",
        "    self.task_splits = task_splits\n",
        "    self.hidden_size = hidden_size\n",
        "    self.device = torch.device(\"cuda:0\" if torch.cuda.is_available() or False else \"cpu\")\n",
        "    self.data = get_data(batch_size=batch_size)\n",
        "    # Gets DataLoaders and class weights (computed wrt the sampler probabilities)\n",
        "    self.train_loader = self.data[\"train\"]\n",
        "    self.val_loader = self.data[\"val\"]\n",
        "    self.class_weight = self.data[\"class_weight\"].to(self.device)\n",
        "  \n",
        "    backbone, EMB_SIZE = pretrained_backbone('resnet', hidden_size = hidden_size)\n",
        "    self.RICH_EMB_SIZE = EMB_SIZE + task_splits.sum()\n",
        " \n",
        "    if expand_images == True:\n",
        "      backbone.conv1.stride = (2, 1)\n",
        "      backbone.maxpool.stride = (1, 1)\n",
        "   \n",
        "    # fc_dim = 16\n",
        "    # shape = (16, 16)\n",
        "    # Attribute recognition\n",
        "    attr_classifiers = [torch.nn.Sequential(\n",
        "           ConvBlock(in_spatial_shape=shape, in_features=hidden_size, out_features_conv=fc_dim, out_features_linear=num_classes, kernel_size=(3,3)))\n",
        "           for num_classes in task_splits]\n",
        "    \n",
        "    #Pid classifier instantiation\n",
        "    pid_classifier = torch.nn.Sequential(torch.nn.Dropout(.6), torch.nn.Linear(self.RICH_EMB_SIZE, self.data['train_pids_count']))\n",
        "\n",
        "    self.net = JointConvNet(backbone, attr_classifiers, pid_classifier).to(self.device)\n",
        "\n",
        "    # Initialize the network weights\n",
        "    self.net.apply(initialize_weights)\n",
        "\n",
        "    #Loss functions\n",
        "    self.attr_rec_loss = UncertFocal(gamma=2, weight= self.data['class_weight']).to(self.device)\n",
        "    self.multitask_loss = JointLoss(self.RICH_EMB_SIZE, self.attr_rec_loss).to(self.device)\n",
        "\n",
        "    #self.learning_rate = learning_rate\n",
        "    #self.weight_decay = weight_decay\n",
        "    #Optimizer instatiation\n",
        "    self.optimizer = get_optimizer(self.net, cost_function=self.multitask_loss, learning_rate=learning_rate, weight_decay=weight_decay)"
      ],
      "metadata": {
        "id": "q1na0XBNmWkT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qFtIKl81aE9y"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kCGyBjrnPDSn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c8bd858e-f0fa-46e4-9469-a448b8b409a9"
      },
      "source": [
        "task_splits = torch.tensor([4, 2, 2, 2, 2, 2, 2, 2, 2, 2, 9, 10])\n",
        "model = Model(task_splits = task_splits)\n",
        "\n",
        "# Training the defined network\n",
        "! rm -rf runs\n",
        "fit(model, epochs=40, max_lr=1e-3, early_stopping_patience=10)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "embedding size:  512\n",
            "---Epoch 1:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  75.2      74.7  71.1     83.8   79.1  70.1  88.4  61.5  91.9    61.7   \n",
            "Val    79.6      74.4  72.6     94.1   94.6  82.9  91.2  75.6  98.1    65.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     42.3       39.8  \n",
            "Val       54.3       29.7  \n",
            "Train average Acc: 0.69978094\n",
            "Val average Acc: 0.760225\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  25.8      50.6  50.6     50.9   53.5  67.7  50.2  59.1  50.9    61.7   \n",
            "Val    33.8      51.2  48.8     50.6   51.1  83.6  49.6  58.8  49.9    63.5   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     31.9       18.4  \n",
            "Val       45.5       19.0  \n",
            "Train average mAcc: 0.47607756\n",
            "Val average mAcc: 0.5045742\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.005578813936367555\n",
            "Val mAP: 0.08279580546645364\n",
            "==== Loss ====\n",
            "Train: 9.763106511986773\n",
            "Validation: 5.246827038851651\n",
            "---Epoch 2:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  77.7      75.7  72.9     86.0   81.4  79.6  89.9  69.5  92.4    71.6   \n",
            "Val    78.6      72.2  74.1     93.0   90.0  85.7  91.9  74.1  97.8    67.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     60.8       51.2  \n",
            "Val       63.7       56.9  \n",
            "Train average Acc: 0.7571945\n",
            "Val average Acc: 0.78815186\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  27.4      52.4  50.9     50.9   58.7  78.4  50.6  67.8  50.7    71.7   \n",
            "Val    34.6      51.9  49.7     49.5   56.4  84.1  50.0  64.7  49.7    65.7   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     53.8       31.7  \n",
            "Val       56.8       45.1  \n",
            "Train average mAcc: 0.53752875\n",
            "Val average mAcc: 0.5485917\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.015635306413772972\n",
            "Val mAP: 0.14289892012945432\n",
            "==== Loss ====\n",
            "Train: 7.507966443766718\n",
            "Validation: 4.404538458043879\n",
            "---Epoch 3:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  78.2      74.8  73.3     85.3   82.5  81.6  88.1  72.4  92.4    74.4   \n",
            "Val    78.7      74.5  74.3     93.9   90.4  88.3  92.0  73.5  98.2    73.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     64.5       57.6  \n",
            "Val       62.0       64.7  \n",
            "Train average Acc: 0.77080196\n",
            "Val average Acc: 0.8037449\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  31.7      54.0  51.3     51.3   63.9  80.9  50.4  70.4  52.2    74.4   \n",
            "Val    34.5      52.3  50.2     50.0   62.6  88.8  50.5  68.2  50.0    70.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     59.2       42.1  \n",
            "Val       58.0       50.1  \n",
            "Train average mAcc: 0.5684337\n",
            "Val average mAcc: 0.5711769\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.024672001357929556\n",
            "Val mAP: 0.20771477074943792\n",
            "==== Loss ====\n",
            "Train: 6.303661035454792\n",
            "Validation: 3.716443885456432\n",
            "---Epoch 4:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  79.3      76.4  73.8     85.9   84.7  83.5  90.2  75.2  93.1    77.5   \n",
            "Val    76.8      73.0  73.8     93.4   89.0  87.3  91.8  66.7  98.0    63.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     68.2       60.6  \n",
            "Val       59.7       61.0  \n",
            "Train average Acc: 0.79029167\n",
            "Val average Acc: 0.7782464\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  38.0      57.1  52.6     51.4   68.1  82.7  50.7  73.8  56.7    77.5   \n",
            "Val    37.0      55.7  49.7     50.3   63.6  84.5  49.9  72.6  49.8    67.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     63.8       50.9  \n",
            "Val       54.9       50.4  \n",
            "Train average mAcc: 0.6027934\n",
            "Val average mAcc: 0.5718929\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.036243111444448604\n",
            "Val mAP: 0.19937932832233468\n",
            "==== Loss ====\n",
            "Train: 5.456656294283659\n",
            "Validation: 3.722295045852661\n",
            "---Epoch 5:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  81.1      78.2  74.2     85.7   85.5  85.2  89.7  76.3  93.8    78.2   \n",
            "Val    71.5      74.3  73.4     92.7   94.5  89.0  92.5  77.4  97.2    74.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     69.4       62.7  \n",
            "Val       62.8       64.1  \n",
            "Train average Acc: 0.8000309\n",
            "Val average Acc: 0.8034253\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  47.5      59.9  53.7     52.2   70.8  84.4  52.4  75.2  61.7    78.3   \n",
            "Val    39.4      55.6  50.6     49.9   60.4  88.4  58.1  69.9  53.9    70.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     65.3       54.1  \n",
            "Val       62.3       45.3  \n",
            "Train average mAcc: 0.629486\n",
            "Val average mAcc: 0.58725536\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.03987404377809174\n",
            "Val mAP: 0.2486551642098085\n",
            "==== Loss ====\n",
            "Train: 4.975229342087456\n",
            "Validation: 3.3621134757995605\n",
            "---Epoch 6:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  79.6      79.3  74.9     85.4   86.6  86.3  90.3  78.1  94.1    80.2   \n",
            "Val    74.5      74.3  74.2     93.5   95.0  89.2  92.9  83.7  95.7    76.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     70.7       65.2  \n",
            "Val       41.5       60.9  \n",
            "Train average Acc: 0.8087487\n",
            "Val average Acc: 0.7935838\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  46.5      63.4  55.1     52.3   73.5  85.6  57.4  76.9  67.1    80.2   \n",
            "Val    33.9      53.0  50.1     50.9   64.1  89.4  57.0  71.6  55.4    72.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     66.3       57.6  \n",
            "Val       55.2       45.4  \n",
            "Train average mAcc: 0.6515516\n",
            "Val average mAcc: 0.5821583\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.054398073875348174\n",
            "Val mAP: 0.2249498363569767\n",
            "==== Loss ====\n",
            "Train: 4.639583699599556\n",
            "Validation: 3.376641641963612\n",
            "---Epoch 7:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  81.2      80.7  75.2     85.8   87.7  87.0  91.1  79.8  94.4    81.5   \n",
            "Val    77.4      71.8  72.7     93.5   93.9  90.9  91.8  82.1  96.9    77.0   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     72.3       66.5  \n",
            "Val       66.0       67.9  \n",
            "Train average Acc: 0.81923527\n",
            "Val average Acc: 0.81818765\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  51.8      65.3  56.7     53.4   76.7  86.4  61.3  78.7  67.7    81.5   \n",
            "Val    35.3      61.8  53.6     50.3   72.1  90.3  52.5  79.3  58.2    77.7   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     68.5       60.5  \n",
            "Val       59.4       52.6  \n",
            "Train average mAcc: 0.6737613\n",
            "Val average mAcc: 0.61934984\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.06121381088863044\n",
            "Val mAP: 0.30993300973083704\n",
            "==== Loss ====\n",
            "Train: 4.370747371341871\n",
            "Validation: 3.3488028916445645\n",
            "---Epoch 8:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  82.6      81.2  76.0     86.8   88.4  88.5  91.7  80.9  94.9    83.5   \n",
            "Val    78.1      73.2  72.3     92.7   87.0  91.3  90.2  74.0  95.6    72.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     73.0       67.7  \n",
            "Val       67.3       68.7  \n",
            "Train average Acc: 0.8293452\n",
            "Val average Acc: 0.8020195\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  57.4      67.1  58.8     55.7   78.3  88.1  65.4  79.9  72.2    83.6   \n",
            "Val    34.8      65.1  54.9     50.5   74.4  90.5  56.4  79.7  57.5    76.0   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     69.4       63.5  \n",
            "Val       61.5       51.9  \n",
            "Train average mAcc: 0.6993898\n",
            "Val average mAcc: 0.6276599\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.07188750097006369\n",
            "Val mAP: 0.33520209096748266\n",
            "==== Loss ====\n",
            "Train: 4.189914114578911\n",
            "Validation: 3.551039132204923\n",
            "---Epoch 9:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  82.7      81.8  76.4     86.7   89.1  88.3  91.9  81.6  95.5    83.7   \n",
            "Val    74.4      75.0  74.5     91.8   95.4  90.3  89.3  84.0  97.4    78.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     74.6       68.7  \n",
            "Val       62.0       63.4  \n",
            "Train average Acc: 0.83426905\n",
            "Val average Acc: 0.8135864\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  58.6      68.8  59.9     59.0   79.7  87.9  68.4  80.7  74.5    83.8   \n",
            "Val    38.0      59.3  50.2     53.6   61.8  91.4  59.9  77.4  49.5    77.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     70.9       65.3  \n",
            "Val       61.7       42.9  \n",
            "Train average mAcc: 0.71455824\n",
            "Val average mAcc: 0.60294414\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.08129635053694194\n",
            "Val mAP: 0.27759849404610004\n",
            "==== Loss ====\n",
            "Train: 4.0311102867126465\n",
            "Validation: 3.529865763404153\n",
            "---Epoch 10:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  83.5      83.5  77.5     86.7   90.4  88.7  92.6  82.9  95.8    85.0   \n",
            "Val    67.1      74.8  71.5     92.6   94.2  89.1  91.3  81.4  97.5    79.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     75.0       70.1  \n",
            "Val       42.9       55.1  \n",
            "Train average Acc: 0.84311813\n",
            "Val average Acc: 0.78125\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  60.5      71.9  62.1     57.6   81.8  88.3  70.4  82.0  76.8    85.0   \n",
            "Val    37.6      57.1  54.9     53.5   64.5  90.4  60.1  78.1  56.3    76.7   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     71.7       67.8  \n",
            "Val       50.7       41.7  \n",
            "Train average mAcc: 0.72973365\n",
            "Val average mAcc: 0.60132825\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.09551927041670012\n",
            "Val mAP: 0.305266187706797\n",
            "==== Loss ====\n",
            "Train: 3.9159207530643627\n",
            "Validation: 4.255492643876509\n",
            "---Epoch 11:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  83.8      83.9  77.0     86.9   90.6  89.7  93.3  83.9  95.7    86.0   \n",
            "Val    68.7      76.2  72.4     93.6   94.5  80.9  92.5  83.6  95.6    79.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     75.2       69.8  \n",
            "Val       62.3       62.2  \n",
            "Train average Acc: 0.84652424\n",
            "Val average Acc: 0.8019556\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  63.7      71.8  61.9     60.3   82.3  89.3  74.9  82.9  75.6    86.1   \n",
            "Val    37.1      63.7  53.0     55.2   67.3  84.1  58.1  79.2  57.5    74.3   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     71.5       66.9  \n",
            "Val       60.4       45.5  \n",
            "Train average mAcc: 0.73927486\n",
            "Val average mAcc: 0.612874\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.11054803254459912\n",
            "Val mAP: 0.30453895691424704\n",
            "==== Loss ====\n",
            "Train: 3.8373231991477637\n",
            "Validation: 3.7030892155387183\n",
            "---Epoch 12:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  84.7      84.1  78.5     87.9   90.8  88.3  93.7  83.8  96.4    83.9   \n",
            "Val    77.4      78.1  72.3     92.7   88.7  89.0  91.6  84.3  96.8    80.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     76.1       71.6  \n",
            "Val       63.4       65.8  \n",
            "Train average Acc: 0.84985054\n",
            "Val average Acc: 0.81742084\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  62.8      73.6  65.0     61.1   83.0  87.6  76.2  83.0  80.7    84.0   \n",
            "Val    37.3      64.7  52.5     50.5   68.5  86.7  55.9  82.9  58.2    79.6   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     73.1       69.8  \n",
            "Val       58.8       50.4  \n",
            "Train average mAcc: 0.74993765\n",
            "Val average mAcc: 0.62178373\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.11268150356469418\n",
            "Val mAP: 0.3760439795280483\n",
            "==== Loss ====\n",
            "Train: 3.761821402674136\n",
            "Validation: 4.108234643936157\n",
            "---Epoch 13:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  84.8      84.4  79.1     88.1   91.8  89.4  93.8  84.6  95.9    87.5   \n",
            "Val    77.7      74.5  67.9     90.3   90.2  91.3  91.7  74.1  94.8    75.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     77.2       72.6  \n",
            "Val       63.9       63.6  \n",
            "Train average Acc: 0.8576441\n",
            "Val average Acc: 0.7958845\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  67.6      74.4  66.0     63.0   85.2  88.9  76.5  83.9  78.6    87.5   \n",
            "Val    35.7      63.7  52.7     51.0   80.4  91.7  59.9  78.6  54.9    78.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     74.5       70.7  \n",
            "Val       62.2       53.5  \n",
            "Train average mAcc: 0.7640022\n",
            "Val average mAcc: 0.6356118\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.12070688825301817\n",
            "Val mAP: 0.40679299034684596\n",
            "==== Loss ====\n",
            "Train: 3.638517962331357\n",
            "Validation: 4.164867639541626\n",
            "---Epoch 14:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  86.0      85.3  80.0     87.8   92.0  91.0  94.4  85.8  96.6    88.3   \n",
            "Val    78.4      78.1  72.5     91.6   92.1  90.9  91.9  86.9  96.2    84.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     78.0       74.1  \n",
            "Val       55.2       65.8  \n",
            "Train average Acc: 0.866111\n",
            "Val average Acc: 0.8202965\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  69.1      75.3  67.7     63.0   85.8  90.6  79.0  85.0  83.4    88.3   \n",
            "Val    38.5      63.4  53.0     53.5   70.3  89.9  54.3  80.9  55.6    81.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     75.2       72.9  \n",
            "Val       53.9       46.6  \n",
            "Train average mAcc: 0.7792926\n",
            "Val average mAcc: 0.61763304\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.11952286011147974\n",
            "Val mAP: 0.38074750275781366\n",
            "==== Loss ====\n",
            "Train: 3.533297240215799\n",
            "Validation: 4.712145545265892\n",
            "---Epoch 15:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  85.0      84.6  79.7     87.8   92.2  89.9  92.8  84.7  96.2    87.0   \n",
            "Val    70.4      78.7  72.9     90.6   91.0  86.9  91.9  81.7  94.9    79.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     77.8       73.0  \n",
            "Val       46.9       61.6  \n",
            "Train average Acc: 0.85902476\n",
            "Val average Acc: 0.7889187\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  68.9      74.6  66.7     62.8   85.4  89.4  70.5  84.0  81.2    87.1   \n",
            "Val    36.9      60.3  50.6     66.8   62.9  88.2  57.8  80.0  52.8    79.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     75.0       71.0  \n",
            "Val       55.8       48.6  \n",
            "Train average mAcc: 0.7638453\n",
            "Val average mAcc: 0.6164412\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.12584362123465653\n",
            "Val mAP: 0.35930521766640466\n",
            "==== Loss ====\n",
            "Train: 3.579318546212238\n",
            "Validation: 4.721405831250277\n",
            "---Epoch 16:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  86.2      85.6  80.3     86.8   92.3  90.4  94.5  86.3  96.8    89.1   \n",
            "Val    76.7      77.1  70.2     89.2   94.6  90.4  90.6  84.8  94.2    80.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     78.2       74.3  \n",
            "Val       63.3       64.3  \n",
            "Train average Acc: 0.8673319\n",
            "Val average Acc: 0.8135225\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  70.4      75.7  68.3     59.8   85.5  89.9  76.4  85.6  83.0    89.1   \n",
            "Val    39.2      62.4  48.9     54.6   69.0  91.6  59.2  81.0  50.2    79.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     75.9       72.9  \n",
            "Val       62.0       45.8  \n",
            "Train average mAcc: 0.777143\n",
            "Val average mAcc: 0.6197438\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.13279549498595228\n",
            "Val mAP: 0.4222429414841985\n",
            "==== Loss ====\n",
            "Train: 3.441379176015439\n",
            "Validation: 4.0505736524408515\n",
            "---Epoch 17:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  87.2      86.1  81.3     88.3   92.9  91.2  94.8  87.5  97.0    89.6   \n",
            "Val    75.5      80.3  73.1     92.7   92.2  92.1  93.1  78.2  93.9    77.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     79.2       76.4  \n",
            "Val       62.9       63.0  \n",
            "Train average Acc: 0.87615824\n",
            "Val average Acc: 0.8119249\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  74.5      76.4  71.5     65.0   86.6  90.8  79.8  86.8  84.4    89.6   \n",
            "Val    37.2      71.8  52.8     52.9   67.8  91.4  63.2  83.8  52.2    79.5   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     76.2       75.1  \n",
            "Val       61.6       48.6  \n",
            "Train average mAcc: 0.7972498\n",
            "Val average mAcc: 0.63562286\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.1396231686575687\n",
            "Val mAP: 0.41382023046795713\n",
            "==== Loss ====\n",
            "Train: 3.3179120478422743\n",
            "Validation: 4.6421004858883945\n",
            "---Epoch 18:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  87.2      86.6  81.9     89.4   93.7  91.5  95.3  87.9  97.1    90.5   \n",
            "Val    77.9      77.8  70.6     91.6   93.9  91.7  92.9  86.4  97.6    84.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     80.3       76.6  \n",
            "Val       67.6       64.6  \n",
            "Train average Acc: 0.88153267\n",
            "Val average Acc: 0.8307132\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  74.2      78.5  71.9     67.1   88.6  91.1  82.8  87.3  85.6    90.5   \n",
            "Val    40.4      65.7  53.8     57.7   63.5  92.0  60.1  78.6  58.6    81.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     78.0       76.5  \n",
            "Val       60.1       49.3  \n",
            "Train average mAcc: 0.8101875\n",
            "Val average mAcc: 0.634602\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.15535731224286178\n",
            "Val mAP: 0.43255427753603126\n",
            "==== Loss ====\n",
            "Train: 3.2323273845340896\n",
            "Validation: 5.174366474151611\n",
            "---Epoch 19:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  87.5      87.3  81.7     89.6   93.6  91.4  95.4  86.9  97.2    90.3   \n",
            "Val    78.0      78.4  71.9     86.3   85.7  91.6  92.5  68.7  92.5    70.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     80.4       77.0  \n",
            "Val       66.2       67.8  \n",
            "Train average Acc: 0.88182944\n",
            "Val average Acc: 0.7919223\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  73.0      78.9  72.6     69.4   88.5  91.0  83.3  86.2  85.9    90.3   \n",
            "Val    41.4      67.5  55.5     60.9   80.6  90.5  64.2  78.0  56.0    76.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     78.4       76.6  \n",
            "Val       63.5       57.9  \n",
            "Train average mAcc: 0.8118295\n",
            "Val average mAcc: 0.6601137\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.14612251304063065\n",
            "Val mAP: 0.4841392040890352\n",
            "==== Loss ====\n",
            "Train: 3.2066100783970044\n",
            "Validation: 5.426217642697421\n",
            "---Epoch 20:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  88.3      87.7  82.9     88.8   93.6  91.7  95.2  88.1  97.2    90.9   \n",
            "Val    73.7      79.6  72.8     90.5   94.2  91.5  92.1  83.2  97.0    82.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     80.7       78.0  \n",
            "Val       51.3       67.0  \n",
            "Train average Acc: 0.8859088\n",
            "Val average Acc: 0.8125639\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  78.0      79.4  73.6     66.2   88.8  91.2  83.2  87.6  86.4    90.9   \n",
            "Val    37.9      62.7  53.6     57.1   62.0  91.8  57.9  64.1  60.5    76.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     78.4       78.1  \n",
            "Val       58.8       47.3  \n",
            "Train average mAcc: 0.81831205\n",
            "Val average mAcc: 0.60814965\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.15405958907667625\n",
            "Val mAP: 0.39259046957504695\n",
            "==== Loss ====\n",
            "Train: 3.1383369031159773\n",
            "Validation: 5.155784541910345\n",
            "---Epoch 21:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  87.2      82.9  81.3     89.3   92.9  91.5  95.0  82.6  96.9    87.9   \n",
            "Val    73.5      77.9  72.3     93.4   93.7  89.7  92.6  81.1  96.8    83.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     79.5       75.6  \n",
            "Val       61.6       64.4  \n",
            "Train average Acc: 0.86880964\n",
            "Val average Acc: 0.8167817\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  74.2      69.3  71.1     67.2   87.5  91.1  82.2  81.5  85.0    87.9   \n",
            "Val    37.8      63.0  54.6     53.3   63.5  90.1  59.9  76.9  58.2    81.0   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     76.8       74.7  \n",
            "Val       59.7       46.9  \n",
            "Train average mAcc: 0.79046607\n",
            "Val average mAcc: 0.62078047\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.16183845283193776\n",
            "Val mAP: 0.440598015730943\n",
            "==== Loss ====\n",
            "Train: 3.3079510481461236\n",
            "Validation: 4.952917510812933\n",
            "---Epoch 22:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  87.9      86.1  82.9     90.3   93.9  92.3  95.9  87.1  97.4    90.8   \n",
            "Val    76.4      80.1  69.1     92.0   85.4  89.5  93.1  82.4  94.4    78.3   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     81.2       77.7  \n",
            "Val       64.0       63.4  \n",
            "Train average Acc: 0.88625115\n",
            "Val average Acc: 0.8067484\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  76.5      76.5  73.7     71.0   89.6  91.9  84.9  86.5  86.8    90.8   \n",
            "Val    41.4      67.0  54.2     57.9   75.3  88.7  58.0  82.4  61.4    80.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     78.7       78.1  \n",
            "Val       60.5       44.7  \n",
            "Train average mAcc: 0.82084274\n",
            "Val average mAcc: 0.64367247\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.17561872690590677\n",
            "Val mAP: 0.47235727494578966\n",
            "==== Loss ====\n",
            "Train: 3.05641800009686\n",
            "Validation: 5.628778457641602\n",
            "---Epoch 23:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  87.8      86.1  84.3     89.9   94.2  92.0  92.9  87.6  97.5    91.0   \n",
            "Val    76.8      78.9  70.2     89.7   92.8  91.9  93.5  83.1  96.2    82.5   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     81.4       79.2  \n",
            "Val       69.7       69.5  \n",
            "Train average Acc: 0.886582\n",
            "Val average Acc: 0.8289878\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  76.5      76.0  74.9     71.8   89.6  91.6  67.5  86.9  87.5    91.0   \n",
            "Val    37.8      67.5  54.7     57.9   77.5  91.4  61.7  81.6  55.6    82.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     79.2       79.1  \n",
            "Val       65.0       58.3  \n",
            "Train average mAcc: 0.8095156\n",
            "Val average mAcc: 0.6593735\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.18859734775324344\n",
            "Val mAP: 0.5115560090012672\n",
            "==== Loss ====\n",
            "Train: 3.0316200816113015\n",
            "Validation: 5.713067661632191\n",
            "---Epoch 24:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  88.5      88.3  84.6     90.7   94.5  93.1  95.2  89.2  97.6    91.6   \n",
            "Val    78.1      80.1  71.3     91.9   93.1  91.6  93.9  80.4  95.9    84.0   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     82.6       80.2  \n",
            "Val       68.2       65.5  \n",
            "Train average Acc: 0.89674914\n",
            "Val average Acc: 0.82834864\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  79.5      80.2  76.5     74.6   90.5  92.7  81.3  88.6  88.9    91.6   \n",
            "Val    38.6      72.0  53.0     59.7   81.0  91.3  65.9  83.0  55.5    84.3   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     80.5       80.0  \n",
            "Val       65.6       44.8  \n",
            "Train average mAcc: 0.837507\n",
            "Val average mAcc: 0.6622678\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.19873331101394784\n",
            "Val mAP: 0.4396185917091138\n",
            "==== Loss ====\n",
            "Train: 2.9274632744167164\n",
            "Validation: 6.460055177861994\n",
            "---Epoch 25:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  89.0      88.4  85.0     91.2   94.5  92.8  95.4  89.5  97.8    92.6   \n",
            "Val    74.5      79.9  74.8     89.3   92.5  91.3  92.3  85.7  96.4    81.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     82.8       80.4  \n",
            "Val       64.6       65.0  \n",
            "Train average Acc: 0.8994934\n",
            "Val average Acc: 0.8227889\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  77.9      81.0  77.3     75.9   89.9  92.5  84.0  89.0  89.5    92.6   \n",
            "Val    38.6      63.5  55.9     59.5   64.5  89.9  56.7  80.7  55.7    79.6   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     81.0       80.7  \n",
            "Val       62.7       48.3  \n",
            "Train average mAcc: 0.8426627\n",
            "Val average mAcc: 0.6297273\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.18088421599998133\n",
            "Val mAP: 0.5252369931723242\n",
            "==== Loss ====\n",
            "Train: 2.880578161322552\n",
            "Validation: 6.671650344675237\n",
            "---Epoch 26:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  89.5      88.8  86.0     91.4   94.8  93.5  95.7  89.8  97.7    92.6   \n",
            "Val    73.5      79.3  72.5     88.9   93.2  90.6  93.3  85.5  94.5    81.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     83.3       81.0  \n",
            "Val       62.5       67.9  \n",
            "Train average Acc: 0.9034414\n",
            "Val average Acc: 0.8195297\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  81.0      81.5  78.8     76.3   91.0  93.1  85.2  89.5  89.2    92.7   \n",
            "Val    36.9      63.1  56.5     59.3   77.7  91.5  62.4  83.3  61.5    83.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     81.2       81.7  \n",
            "Val       62.7       55.6  \n",
            "Train average mAcc: 0.8510518\n",
            "Val average mAcc: 0.66149414\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.18971204753608303\n",
            "Val mAP: 0.520777965762039\n",
            "==== Loss ====\n",
            "Train: 2.8271840800409733\n",
            "Validation: 6.226076277819547\n",
            "---Epoch 27:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  90.0      89.4  85.9     91.7   95.0  93.0  96.2  90.4  97.7    92.8   \n",
            "Val    75.8      78.0  66.5     91.2   94.2  89.5  93.2  86.3  96.2    82.3   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     83.5       80.7  \n",
            "Val       67.8       68.0  \n",
            "Train average Acc: 0.90519875\n",
            "Val average Acc: 0.8241948\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  81.2      82.6  78.9     77.2   91.2  92.7  85.5  89.9  89.9    92.8   \n",
            "Val    38.3      70.1  57.7     56.3   68.0  90.2  62.4  83.9  62.3    81.5   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     81.4       81.0  \n",
            "Val       61.5       48.9  \n",
            "Train average mAcc: 0.8536447\n",
            "Val average mAcc: 0.6509303\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.20380242049503416\n",
            "Val mAP: 0.49689763837893075\n",
            "==== Loss ====\n",
            "Train: 2.804244657184767\n",
            "Validation: 6.187073599208485\n",
            "---Epoch 28:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  90.7      90.8  86.4     91.9   95.5  93.7  96.4  90.7  98.0    92.8   \n",
            "Val    73.7      78.5  70.6     92.9   94.8  91.4  92.7  83.9  94.9    85.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     84.0       81.1  \n",
            "Val       58.4       61.4  \n",
            "Train average Acc: 0.9099743\n",
            "Val average Acc: 0.8155036\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  84.2      84.7  79.1     78.6   92.2  93.4  88.6  90.2  89.8    92.8   \n",
            "Val    37.5      72.0  54.9     57.8   66.6  90.7  66.5  77.8  63.9    82.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     82.2       82.2  \n",
            "Val       60.5       41.5  \n",
            "Train average mAcc: 0.8650358\n",
            "Val average mAcc: 0.64389867\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.20376795584666899\n",
            "Val mAP: 0.5144953191482584\n",
            "==== Loss ====\n",
            "Train: 2.6952293478924294\n",
            "Validation: 7.997863379391757\n",
            "---Epoch 29:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  89.4      90.0  86.4     91.7   94.6  93.1  96.2  90.2  97.7    93.0   \n",
            "Val    78.6      79.3  70.1     88.3   92.6  89.3  90.5  74.7  95.1    79.7   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     84.4       81.1  \n",
            "Val       66.4       63.7  \n",
            "Train average Acc: 0.9064996\n",
            "Val average Acc: 0.8068763\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  80.0      83.8  79.0     78.8   90.0  92.8  87.0  89.8  89.3    93.0   \n",
            "Val    39.7      72.1  57.4     62.0   69.7  89.4  65.3  79.8  64.0    82.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     82.7       81.5  \n",
            "Val       63.5       47.1  \n",
            "Train average mAcc: 0.8564309\n",
            "Val average mAcc: 0.6601136\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.21366042044616107\n",
            "Val mAP: 0.4424025003826636\n",
            "==== Loss ====\n",
            "Train: 2.7331005200095797\n",
            "Validation: 7.039296995509755\n",
            "---Epoch 30:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  90.6      90.5  86.3     92.5   95.4  93.8  95.8  90.6  98.0    93.2   \n",
            "Val    79.9      80.0  72.9     90.4   90.9  89.3  93.8  79.0  95.5    70.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     85.1       81.8  \n",
            "Val       57.9       69.2  \n",
            "Train average Acc: 0.91130924\n",
            "Val average Acc: 0.8074515\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  82.2      85.1  79.7     79.4   92.0  93.5  84.3  90.1  90.0    93.2   \n",
            "Val    40.0      70.9  59.0     59.5   73.1  89.5  65.8  81.9  64.2    75.5   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     83.2       82.1  \n",
            "Val       59.8       54.3  \n",
            "Train average mAcc: 0.8623427\n",
            "Val average mAcc: 0.6611895\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.2174511278380601\n",
            "Val mAP: 0.48673945263553575\n",
            "==== Loss ====\n",
            "Train: 2.651681945634925\n",
            "Validation: 8.437319560484452\n",
            "---Epoch 31:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  90.5      90.6  87.4     92.5   95.5  93.8  96.7  91.3  97.8    93.7   \n",
            "Val    78.7      75.0  74.8     90.7   94.1  89.0  92.0  81.2  95.6    79.7   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     84.8       82.6  \n",
            "Val       66.1       65.3  \n",
            "Train average Acc: 0.9142761\n",
            "Val average Acc: 0.8185072\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  83.3      84.5  80.8     79.7   92.4  93.5  89.1  90.9  90.4    93.7   \n",
            "Val    44.2      70.5  55.9     54.8   73.0  88.2  57.4  81.3  64.3    81.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     83.5       83.0  \n",
            "Val       61.5       50.9  \n",
            "Train average mAcc: 0.8707192\n",
            "Val average mAcc: 0.6531396\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.23231483038903097\n",
            "Val mAP: 0.4378927780895004\n",
            "==== Loss ====\n",
            "Train: 2.6439978309299637\n",
            "Validation: 8.000711289319126\n",
            "---Epoch 32:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  91.1      90.8  87.4     92.5   95.4  93.1  96.5  91.6  97.9    93.7   \n",
            "Val    78.5      80.3  71.2     92.2   91.5  90.6  92.9  74.2  93.4    76.5   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     84.6       81.9  \n",
            "Val       67.3       66.6  \n",
            "Train average Acc: 0.91373414\n",
            "Val average Acc: 0.8126917\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  84.9      85.3  80.8     80.5   92.0  92.7  88.1  91.2  89.3    93.7   \n",
            "Val    36.5      73.0  54.2     57.4   77.6  89.0  57.9  80.4  69.8    79.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     83.1       82.6  \n",
            "Val       61.1       48.1  \n",
            "Train average mAcc: 0.8701807\n",
            "Val average mAcc: 0.6536223\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.2390982065150905\n",
            "Val mAP: 0.488917906878967\n",
            "==== Loss ====\n",
            "Train: 2.630382714064225\n",
            "Validation: 8.247466260736639\n",
            "---Epoch 33:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  91.1      88.5  87.5     92.5   95.3  93.5  96.8  90.4  98.2    93.1   \n",
            "Val    77.8      81.9  72.2     91.7   92.6  93.9  92.3  80.4  96.5    79.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     85.2       82.4  \n",
            "Val       69.0       64.6  \n",
            "Train average Acc: 0.91204536\n",
            "Val average Acc: 0.82745403\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  83.6      81.4  80.9     81.0   92.2  93.1  89.8  90.0  90.9    93.1   \n",
            "Val    41.7      73.9  54.1     55.4   79.1  93.1  60.2  83.4  62.5    81.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     83.5       83.2  \n",
            "Val       66.2       51.0  \n",
            "Train average mAcc: 0.86885905\n",
            "Val average mAcc: 0.6681117\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.21410271104027878\n",
            "Val mAP: 0.5628567847166346\n",
            "==== Loss ====\n",
            "Train: 2.6274690939032515\n",
            "Validation: 8.986776460300792\n",
            "---Epoch 34:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  92.1      91.0  88.9     93.2   96.2  94.4  97.1  92.6  98.4    94.3   \n",
            "Val    75.2      77.9  71.5     89.6   92.6  90.8  92.4  85.1  96.9    85.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     86.6       84.0  \n",
            "Val       62.3       64.5  \n",
            "Train average Acc: 0.92404956\n",
            "Val average Acc: 0.819977\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  87.4      86.0  83.4     83.2   93.6  94.1  90.2  92.3  92.9    94.3   \n",
            "Val    41.8      72.0  54.5     57.3   68.0  89.6  57.2  81.9  62.7    84.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     85.2       84.7  \n",
            "Val       63.2       47.1  \n",
            "Train average mAcc: 0.88936925\n",
            "Val average mAcc: 0.6493802\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.24051785840272125\n",
            "Val mAP: 0.508740649867416\n",
            "==== Loss ====\n",
            "Train: 2.451631276503853\n",
            "Validation: 9.049084945158524\n",
            "---Epoch 35:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  92.1      91.4  88.8     93.5   95.8  93.6  97.2  91.9  98.4    94.4   \n",
            "Val    79.3      65.9  63.3     92.8   77.1  79.0  92.7  54.3  93.9    54.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     86.9       85.1  \n",
            "Val       62.5       58.4  \n",
            "Train average Acc: 0.924312\n",
            "Val average Acc: 0.72827196\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  87.2      85.7  83.5     83.3   92.7  93.3  91.2  91.6  92.0    94.5   \n",
            "Val    35.9      63.7  56.5     53.5   74.4  70.7  57.8  66.9  63.4    63.6   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     85.4       85.5  \n",
            "Val       52.3       38.3  \n",
            "Train average mAcc: 0.88812315\n",
            "Val average mAcc: 0.580881\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.24149702410994772\n",
            "Val mAP: 0.2760969236850631\n",
            "==== Loss ====\n",
            "Train: 2.4591969904692275\n",
            "Validation: 25.259629899805244\n",
            "---Epoch 36:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  89.5      88.4  86.6     88.0   93.3  92.8  96.4  89.3  96.9    91.3   \n",
            "Val    76.4      77.2  73.5     92.1   93.4  91.3  92.6  83.5  95.3    80.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     83.2       80.0  \n",
            "Val       64.3       66.8  \n",
            "Train average Acc: 0.89635533\n",
            "Val average Acc: 0.8227889\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  79.8      80.8  79.2     65.0   86.9  92.4  87.7  88.7  83.2    91.3   \n",
            "Val    39.7      68.9  55.2     53.2   76.1  90.1  60.4  82.8  59.6    80.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     81.4       80.5  \n",
            "Val       59.8       53.3  \n",
            "Train average mAcc: 0.83072984\n",
            "Val average mAcc: 0.6498139\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.21494724161207315\n",
            "Val mAP: 0.4749294690478295\n",
            "==== Loss ====\n",
            "Train: 2.9544871143672777\n",
            "Validation: 7.747816649350253\n",
            "---Epoch 37:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  91.9      90.9  88.4     91.2   95.4  94.1  96.9  91.9  97.6    93.7   \n",
            "Val    75.8      77.1  73.0     92.6   93.8  91.0  93.5  85.7  96.4    84.0   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     86.3       83.2  \n",
            "Val       68.6       64.6  \n",
            "Train average Acc: 0.91788197\n",
            "Val average Acc: 0.8300742\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  85.2      85.5  82.6     76.1   91.7  93.7  89.8  91.5  87.1    93.7   \n",
            "Val    40.4      72.7  55.4     52.8   72.9  90.9  62.1  79.7  60.2    81.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     85.0       83.8  \n",
            "Val       64.7       46.8  \n",
            "Train average mAcc: 0.8713808\n",
            "Val average mAcc: 0.6500753\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.24829551975981123\n",
            "Val mAP: 0.5006908505764237\n",
            "==== Loss ====\n",
            "Train: 2.5130481678506604\n",
            "Validation: 8.345441276376897\n",
            "---Epoch 38:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  92.1      91.7  89.1     92.6   95.8  94.8  96.4  92.2  98.1    94.0   \n",
            "Val    76.3      78.1  71.7     93.6   91.9  91.6  92.2  82.5  97.9    81.4   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     87.6       83.8  \n",
            "Val       69.0       66.5  \n",
            "Train average Acc: 0.92368436\n",
            "Val average Acc: 0.8272623\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  86.5      86.6  84.2     80.9   92.4  94.6  88.5  91.8  90.7    94.0   \n",
            "Val    43.4      66.4  57.6     54.0   76.2  90.6  61.9  83.1  60.9    82.0   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     86.3       84.8  \n",
            "Val       64.0       51.5  \n",
            "Train average mAcc: 0.8843553\n",
            "Val average mAcc: 0.6596594\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.25446209934587927\n",
            "Val mAP: 0.47451811272614564\n",
            "==== Loss ====\n",
            "Train: 2.431729743791663\n",
            "Validation: 10.549834684892135\n",
            "---Epoch 39:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  92.3      91.5  89.5     92.9   96.3  95.2  97.4  92.7  98.2    95.1   \n",
            "Val    75.7      79.1  74.7     91.5   92.3  90.2  91.7  79.4  97.8    76.8   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     87.8       85.1  \n",
            "Val       64.7       61.6  \n",
            "Train average Acc: 0.9282658\n",
            "Val average Acc: 0.8128196\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  87.0      86.5  84.6     81.3   93.5  94.9  91.4  92.3  90.9    95.1   \n",
            "Val    39.0      67.7  54.7     59.5   73.8  90.5  54.7  79.9  63.1    77.1   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     86.7       85.8  \n",
            "Val       57.1       46.0  \n",
            "Train average mAcc: 0.89163846\n",
            "Val average mAcc: 0.63593805\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.2533399904253026\n",
            "Val mAP: 0.4559026101401293\n",
            "==== Loss ====\n",
            "Train: 2.3562935124272886\n",
            "Validation: 12.40400533242659\n",
            "---Epoch 40:---------------------------------\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== ATTRIBUTES: Global Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  91.3      91.1  84.3     92.1   95.8  94.5  95.5  91.7  98.0    94.0   \n",
            "Val    76.2      80.6  73.5     91.3   92.8  88.5  91.9  85.6  97.1    82.2   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     86.5       84.3  \n",
            "Val       65.3       62.4  \n",
            "Train average Acc: 0.9160847\n",
            "Val average Acc: 0.8227889\n",
            "==== ATTRIBUTES: Class Averaged Accuracy % ====\n",
            "        Age  Backpack   Bag  Handbag  Cloth  Down    Up  Hair   Hat  Gender  \\\n",
            "Train  85.4      85.2  75.4     80.0   92.8  94.2  85.2  91.4  90.6    94.1   \n",
            "Val    38.7      66.8  54.8     54.0   68.9  89.6  57.8  82.6  58.3    79.9   \n",
            "\n",
            "       UpColor  DownColor  \n",
            "Train     85.0       84.9  \n",
            "Val       65.0       43.1  \n",
            "Train average mAcc: 0.87014866\n",
            "Val average mAcc: 0.632951\n",
            "==== RE-IDENTIFICATION: mean Average Precision % ====\n",
            "Train mAP: 0.2513943830580144\n",
            "Val mAP: 0.4787404593621182\n",
            "==== Loss ====\n",
            "Train: 2.5312469440957774\n",
            "Validation: 10.72757619077509\n",
            "The best weights have been saved as 'model_weights.pth'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hsYhUQxAaJUN"
      },
      "source": [
        "## Test classification\n",
        "Predicts the annotations on the test set and produces a csv file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I_CDjdBQaL_3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2bd76eee-23dd-466f-f169-fd72c3e658e4"
      },
      "source": [
        "predict(model.net, model.data, 0.82)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[3, 1, 1,  ..., 1, 1, 1],\n",
              "        [2, 1, 1,  ..., 1, 1, 1],\n",
              "        [2, 1, 1,  ..., 1, 1, 1],\n",
              "        ...,\n",
              "        [2, 1, 1,  ..., 1, 1, 1],\n",
              "        [2, 2, 1,  ..., 2, 1, 1],\n",
              "        [2, 1, 1,  ..., 1, 1, 1]], device='cuda:0', dtype=torch.int32)"
            ]
          },
          "metadata": {},
          "execution_count": 723
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(model.net.state_dict(), 'model_weights.pth')"
      ],
      "metadata": {
        "id": "9axyLAjWTAy4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "files.download('model_weights.pth')\n",
        "files.download('classification_test.csv')\n",
        "files.download('reid_test.txt')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "myapw0D5Vt0-",
        "outputId": "aeb700ea-44e9-4efd-b72a-283cb8412c72"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_feaf1c9a-05a8-4970-aeaa-98d40db979fc\", \"model_weights.pth\", 54413273)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_b4f1f2b0-aa1b-4585-8f15-bbb0c5dba094\", \"classification_test.csv\", 1279348)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_6dc0131d-e3cd-4c64-9693-8723029f335b\", \"reid_test.txt\", 13171664)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}