experiments_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "f2a7d0ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import sqlite3\n",
    "import warnings\n",
    "import random\n",
    "import sqlalchemy\n",
    "from math import comb\n",
    "\n",
    "# Make sure these are in the filepath.\n",
    "import factorization\n",
    "import sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "0f0415fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import entire yt graph.\n",
    "dcon = sqlite3.connect(\"yt_graph.db\")\n",
    "edges = pd.read_sql_query(\"SELECT * FROM ytGraphDirected\", con=dcon)\n",
    "G_full = nx.from_edgelist(edges.to_numpy())\n",
    "dcon.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dc8ef27",
   "metadata": {},
   "source": [
    "## Implementation.\n",
    "These functions implement and evaluate the link prediction process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "be250175",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove perc edges, for evaluation.\n",
    "# Does not mutate G.\n",
    "\n",
    "# DOES NOT WORK FOR DIRECTED GRAPHS.\n",
    "def remove_edges(G, perc=0.2):\n",
    "    \n",
    "    size = G.number_of_edges()\n",
    "    \n",
    "    # Num edges to remove.\n",
    "    N = np.floor(0.2*size)\n",
    "    \n",
    "    # Initialize output.\n",
    "    G_removed = G.copy()\n",
    "    removed_edges = []\n",
    "    bridges = []\n",
    "    j = 0\n",
    "    \n",
    "    while (len(removed_edges) < N) and (j < size):\n",
    "        \n",
    "        # Don't choose edges from the list of known bridges.\n",
    "        candidate_edges = list(set(G_removed.edges()) - set(bridges))\n",
    "        edge = random.choice(candidate_edges)\n",
    "        \n",
    "        # Check if the selected edge is a bridge.\n",
    "        G_removed.remove_edge(edge[0], edge[1])\n",
    "        is_bridge = not nx.has_path(G_removed, edge[0], edge[1])\n",
    "        \n",
    "        if is_bridge:\n",
    "            # If edge is a bridge, add it back to the graph and pass.\n",
    "            # Also add it to list of bridges so it isn't selected again.\n",
    "            G_removed.add_edge(edge[0], edge[1])\n",
    "            bridges.append(edge)\n",
    "        else:\n",
    "            # Otherwise, don't add the edge back and append it to output.\n",
    "            removed_edges.append(edge)\n",
    "        \n",
    "        j = j + 1\n",
    "    \n",
    "    # Return a new graph with edges remove, as well as what was removed.\n",
    "    return G_removed, removed_edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7fca6435",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function calculates embeddings for G.\n",
    "def calc_embeddings(G, directed=False, alg=\"factorization\", sim=\"autocovariance\"):\n",
    "    # G -- nx.Graph, or nx.DiGraph (if directed==True)\n",
    "    # Directed -- t/f\n",
    "    # alg -- embedding algorithm -- \"factorization\", \"sampling\"\n",
    "    # sim -- similarity metric -- \"autocovariance\", \"PMI\"\n",
    "    warnings.filterwarnings('ignore')\n",
    "    \n",
    "    if alg == \"factorization\":\n",
    "        \n",
    "        # factorization.embed(G, dimensions=128, markov_time, directed, similarity, average_similarity)\n",
    "        emb = factorization.embed(G, 128, 3, directed, sim, False)\n",
    "        \n",
    "    elif alg == \"sampling\":\n",
    "        \n",
    "        # sampling.embed(G, dimensions=128, markov_time, None, directed, similarity, average_similarity,\n",
    "        #       lr, iter, early_stop, batch_size, neg, walks, walk_length, damp, workers)\n",
    "        emb = sampling.embed(G, 128, 3, None, directed, sim, False,\n",
    "                            6e-3, 50, 5, 100, 1, 10, 80, 0.85, 32)\n",
    "        \n",
    "    return emb[:, ~np.isnan(emb).any(axis=0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "b83bf9d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Given a list of embeddings, return a list of top k predicted edges.\n",
    "def link_prediction(G, emb, k):\n",
    "    \n",
    "    existing_edges = [(min(edge), max(edge)) for edge in G.edges()]\n",
    "    vertices = sorted(list(G.nodes()))\n",
    "    \n",
    "    candidate_edges = [(u, v) for u in vertices for v in vertices]\n",
    "    candidate_edges = [edge for edge in candidate_edges if\n",
    "                       (edge[0] < edge[1])]\n",
    "    candidate_edges = [edge for edge in candidate_edges if\n",
    "                       (edge not in existing_edges)]\n",
    "    \n",
    "    dic = {}\n",
    "    for j in range(len(vertices)):\n",
    "        dic[vertices[j]] = emb[j, :]\n",
    "        \n",
    "    # I'm not sure if this should be sorted by asc or desc.\n",
    "    predictions = sorted(candidate_edges,\n",
    "                         key=lambda x: np.dot(dic[x[0]], dic[x[1]]), reverse=True)\n",
    "    \n",
    "    return predictions[0:k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "69cf08bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Do everything: Remove edges, link prediction, and evaluate.\n",
    "def eval_model(G, k, get_all_preds=False):\n",
    "    G_removed, removed_edges = remove_edges(G)\n",
    "    order = len(G_removed.nodes())\n",
    "    size = len(G_removed.edges())\n",
    "    \n",
    "    # If this is true, return the whole list of predictions rather than the confusion matrix.\n",
    "    if get_all_preds:\n",
    "        k = comb(order, 2) - size\n",
    "    \n",
    "    # Make predictions.\n",
    "    emb = calc_embeddings(G_removed)\n",
    "    predictions = link_prediction(G_removed, emb, k)\n",
    "    \n",
    "    if get_all_preds:\n",
    "        return predictions\n",
    "            \n",
    "    # Evaluate against removed_edges.\n",
    "    removed_edges = [(min(edge), max(edge)) for edge in removed_edges]\n",
    "    TP = len([edge for edge in predictions if (edge in removed_edges)])\n",
    "    FP = len(predictions) - TP\n",
    "    FN = len(removed_edges) - TP\n",
    "    TN = comb(order, 2) - size - TP - FP - FN\n",
    "            \n",
    "    return [TP, FP, FN, TN]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4c97da92",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function evaluates the model on many graphs of increasing order for fixed k.\n",
    "def test_on_order(start_vertex, iters=10, size_cap=5000):\n",
    "    \n",
    "    warnings.filterwarnings('ignore') # Ignore warnings.\n",
    "    \n",
    "    # Connect to graph db.\n",
    "    dcon = sqlite3.connect(\"yt_graph.db\")\n",
    "    \n",
    "    # Init some things.\n",
    "    order = -1\n",
    "    Cmatrices = {}\n",
    "    bads = {}\n",
    "    vertices = pd.Series([start_vertex], name=\"vertex\")\n",
    "    \n",
    "    for j in range(iters):\n",
    "        \n",
    "        # First, enlarge search by adding neighbors of the current subgraph.\n",
    "        vertices.to_sql(\"vertexList\", con=dcon, if_exists=\"replace\")\n",
    "        print(\"Executing SQL to expand target subgraph. (Initial vertex: \" +\n",
    "              str(start_vertex) + \")\")\n",
    "        neighbors = pd.read_sql_query(\n",
    "        \"\"\"\n",
    "        SELECT DISTINCT\n",
    "            x.v2\n",
    "        FROM\n",
    "            ytGraphUndirected x\n",
    "        WHERE EXISTS (\n",
    "            SELECT *\n",
    "            FROM vertexList y\n",
    "            WHERE y.vertex = x.v1\n",
    "            OR y.vertex = x.v2\n",
    "        )\n",
    "        \"\"\", dcon)\n",
    "        print(\"              SQL complete.\")\n",
    "        vertices = pd.Series(neighbors.squeeze(), name=\"vertex\")\n",
    "        \n",
    "        # If subgraph is maximal (represents an entire connected component), stop.\n",
    "        if len(vertices) == order:\n",
    "            print(\"SUBGRAPH IS MAXIMAL AND CANNOT BE ENLARGED.\")\n",
    "            print(\"------------------------------------------------------> STOPPING TESTING.\")\n",
    "            break\n",
    "        \n",
    "        # If graph is too big, stop.\n",
    "        order = len(vertices)\n",
    "        if order > size_cap:\n",
    "            print(\"ORDER OF SUBGRAPH (\" + str(order) + \") HAS EXCEEDED SPECIFIED SIZE LIMIT OF \" +\n",
    "                  str(size_cap) + \".\")\n",
    "            print(\"------------------------------------------------------> STOPPING TESTING.\")\n",
    "            break\n",
    "        \n",
    "        # Otherwise, continue by trying to make embeddings.\n",
    "        G = G_full.subgraph(list(vertices))\n",
    "        size = len(G.edges())\n",
    "        print(\"Calculating embeddings for subgraph of order \" + str(order) + \"...\")\n",
    "        \n",
    "        try:\n",
    "            # Try to evaluate the model.\n",
    "            k = int(np.ceil(0.2*size))\n",
    "            Cmatrices[(order, size-k)] = eval_model(G, k, get_all_preds=False)\n",
    "            print(\"              Embeddings complete.\")\n",
    "            \n",
    "        except Exception as E:\n",
    "            # Otherwise, save the error as well as the graph that produced it.\n",
    "            # We might need this for testing later.\n",
    "            print(\"              Embeddings failed. Moving to next iteration.\")\n",
    "            bads[G] = str(E)\n",
    "            \n",
    "        print(\"------------------------------------------------------> Iteration \" +\n",
    "              str(j+1) + \"/\" + str(iters) +\n",
    "              \" complete.\")\n",
    "        \n",
    "    \n",
    "    dcon.close()\n",
    "    return Cmatrices, bads"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3d868a4",
   "metadata": {},
   "source": [
    "## Test 1: Evaluate predictions vs size of graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "id": "27e10043",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[14987, 10285, 12294, 46028, 2603]\n"
     ]
    }
   ],
   "source": [
    "# Setup.\n",
    "num_trials = 5\n",
    "num_iters = 15\n",
    "sample_vertices = random.sample(G_full.nodes(), num_trials)\n",
    "print(sample_vertices)\n",
    "\n",
    "test_order_Cmatrices = {}\n",
    "test_order_bads = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "bae57adb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 3...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 1/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 31...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 2/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 55...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 3/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 114...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 4/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 402...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 5/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1141...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 6/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1260...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 7/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1478...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 8/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 2093...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 9/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 3430...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 10/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 4805...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 11/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 14987)\n",
      "              SQL complete.\n",
      "ORDER OF SUBGRAPH (7145) HAS EXCEEDED SPECIFIED SIZE LIMIT OF 5000.\n",
      "------------------------------------------------------> STOPPING TESTING.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 27...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 1/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 155...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 2/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 269...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 3/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 402...\n",
      "              Embeddings failed. Moving to next iteration.\n",
      "------------------------------------------------------> Iteration 4/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 615...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 5/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 717...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 6/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 945...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 7/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1369...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 8/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1991...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 9/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 3599...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 10/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 10285)\n",
      "              SQL complete.\n",
      "ORDER OF SUBGRAPH (5209) HAS EXCEEDED SPECIFIED SIZE LIMIT OF 5000.\n",
      "------------------------------------------------------> STOPPING TESTING.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 10...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 1/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 53...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 2/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 85...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 3/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 119...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 4/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 284...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 5/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 753...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 6/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1020...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 7/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1517...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 8/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 2241...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 9/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 3620...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 10/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 12294)\n",
      "              SQL complete.\n",
      "ORDER OF SUBGRAPH (5692) HAS EXCEEDED SPECIFIED SIZE LIMIT OF 5000.\n",
      "------------------------------------------------------> STOPPING TESTING.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 2...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 1/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 22...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 2/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 58...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 3/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 108...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 4/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 171...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 5/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 520...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 6/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 584...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 7/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 753...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 8/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1262...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 9/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1826...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 10/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 3199...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 11/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 46028)\n",
      "              SQL complete.\n",
      "ORDER OF SUBGRAPH (5489) HAS EXCEEDED SPECIFIED SIZE LIMIT OF 5000.\n",
      "------------------------------------------------------> STOPPING TESTING.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 2...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 1/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 18...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 2/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 58...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 3/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 176...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 4/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 848...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 5/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 876...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 6/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 966...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 7/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1257...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 8/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 1872...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 9/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "Calculating embeddings for subgraph of order 3183...\n",
      "              Embeddings complete.\n",
      "------------------------------------------------------> Iteration 10/15 complete.\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 2603)\n",
      "              SQL complete.\n",
      "ORDER OF SUBGRAPH (5845) HAS EXCEEDED SPECIFIED SIZE LIMIT OF 5000.\n",
      "------------------------------------------------------> STOPPING TESTING.\n"
     ]
    }
   ],
   "source": [
    "# Run tests.\n",
    "# Might take a while depending on test parameters.\n",
    "\n",
    "for j in range(num_trials):\n",
    "    Cmatrices, bads = test_on_order(start_vertex=sample_vertices[j], iters=num_iters)\n",
    "    test_order_Cmatrices.update(Cmatrices)\n",
    "    test_order_bads.update(bads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "2d46866d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_values([[0, 1, 0, 0], [3, 6, 6, 414], [2, 14, 14, 1391], [7, 36, 35, 6193], [45, 126, 125, 79622], [144, 366, 365, 647455], [136, 420, 419, 789974], [205, 469, 468, 1087667], [317, 685, 684, 2183585], [611, 1254, 1253, 5870157], [714, 1719, 1718, 11527730], [1, 12, 11, 276], [28, 93, 92, 11239], [40, 134, 133, 35043], [83, 213, 213, 187112], [93, 266, 265, 254626], [178, 375, 374, 442902], [269, 561, 560, 931686], [345, 761, 760, 1974756], [549, 1270, 1270, 6464236], [0, 5, 4, 19], [9, 29, 28, 1161], [16, 38, 37, 3264], [19, 45, 45, 6656], [43, 105, 104, 39342], [223, 291, 290, 280270], [280, 457, 456, 515551], [377, 620, 620, 1144281], [599, 910, 910, 2501465], [749, 1525, 1524, 6537499], [0, 0, 0, 0], [1, 10, 9, 167], [9, 20, 19, 1489], [7, 34, 34, 5539], [32, 36, 35, 14162], [87, 117, 116, 133807], [71, 148, 147, 168994], [105, 181, 180, 281519], [188, 373, 372, 792516], [302, 620, 619, 1660998], [407, 1033, 1032, 5106970], [0, 4, 1, 131], [2, 13, 12, 1568], [11, 34, 34, 15141], [96, 159, 158, 357696], [115, 149, 148, 381785], [120, 176, 176, 464439], [157, 266, 265, 787018], [258, 569, 568, 1746553], [445, 1023, 1022, 5055793]])"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_order_Cmatrices.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "902af0c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save good data.\n",
    "with open(\"test_results/test_order_Cmatrices.pk\", \"wb\") as file:\n",
    "    pickle.dump(test_order_Cmatrices, file)\n",
    "\n",
    "# Save bads.\n",
    "with open(\"test_results/test_order_bads.pk\", \"wb\") as file:\n",
    "    pickle.dump(test_order_bads, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "id": "ee88e3c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Precision')"
      ]
     },
     "execution_count": 302,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAElCAYAAAD3KtVsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAArOklEQVR4nO3debwcVZ338c83IQECASEJKoTcgOASBR0MoA8oKC6AIsy4DExcwCXgzqOjIDijqFFkRgccVIwKLgQQFRlw4EEFQRkVCC4gg5EtgQhICDuRNb/nj3M6qXSqu6vv7e7bfe/3/Xr163bXeupU3fpVnXPqlCICMzOzViaMdgLMzGwwOGCYmVklDhhmZlaJA4aZmVXigGFmZpU4YJiZWSUOGNZzkuZJ+kmF6U6R9C+9SNNYJykk7dCB5XifjGPycxhWJGkp8FTgSeBh4ALg/RHx0Gimq99Img3cAlwQEa8pDD8duDEiPllhGUuBd0bEz7qUzOK6AtgxIm6sMO07gI8A2wCrgMXAwRHxYHdTaf3OdxhW5oCI2BTYBdgV+Hj9BJI26Hmq+tOLJO0x2olopN39JGkv4LPAIRExFXgOcHY30maDxwHDGoqIvwAXAs+DNcUa75V0A3BDHvZaSb+XdJ+kX0nauTa/pG0lnSNphaSVkk7Oww+VdHn+Lkn/IekuSfdLukZSbX3fkvSZwvLeJelGSfdIOk/S1oVxIekISTdIulfSlyWpfpskbS3pb5K2LAz7O0l3S5okaQdJl+W03C3pey2y6QTgM41GNsofSd8FZgHnS3pI0kclfVvSh/P4bfI2vSf/3iFvtyrmxTr7qS5Ne0q6TdLLSpK8K/DriPgdQETcExHfrt1dFPeJpFraa5/Vkg7N454t6ac5fUskvalFPtoAcMCwhiRtC+wP/K4w+CBgd2COpF2AU4HDgWnA14DzJG0oaSLwY2AZMJtUvHFWyWpeBbwUeCbwFOAfgZUlaXk58DngTcDT83Lrl/da0gnv+Xm6V9cvJyJuB34NvL4w+J+AH0TE48CngZ8AWwAzgf8sSXPRl4FnSnpFSZob5k9EvAW4lXw3FxEnAJcBe+fZ9wJuzn8h5dEvIyIq5sVB5P1Ul6ZXA2cCr4+In5dszxXAqyUdJ2kPSRs22vCIqKV9U+ANwJ3AxZI2AX4KnAFsBRwCfEXScxstywaDA4aVOVfSfcDlpJPYZwvjPpevOv8GvAv4WkRcERFPRsS3gUeBFwG7AVsDH4mIhyPikYi4vGRdjwNTgWeT6tSuj4g7SqabB5waEb+NiEeBjwEvznUJNcdHxH0RcSvwc+AFDbbvDNJJjHzFfnAeVkvPELB1kzQXPQIsoPwuo1n+lLkMeImkCaQAcQJQK+7aK4+HanlR3E81bwQWAvtHxJVlCYiIXwL/QCqO/G9gpaQv5guAUpKeCXwH+MeIuI0UuJdGxGkR8URE/Bb4ISmo2ABzwLAyB0XEUyJiKCLeU3fSua3wfQj4cC5uuS8HmW1JgWJbYFlEPNFsRRFxCXAy6Ur9r5IWStqsZNKtSVfStfkeIt2JbFOY5s7C91XApg1W+wPSCXZr0ok5gF/mcR8FBFwp6TpJb2+W/uzrwFMlHVA3vFn+rCcibgIeIgW6l5Du0G6X9CzWDRhV8qK4n2qOBM6OiGubbUxEXBgRBwBbAgcChwLvLJtW0ubAfwH/koNNbbt3r9vuecDTmq3X+p8DhrWr2KzuNmBBDi61z5SIODOPm6UKla4R8aWIeCHwXFLR1EdKJruddCICIBd7TAP+0vYGRNxHKnZ6E6k46szIzQUj4s6IeFdEbE0qSvqKWjRHzUVZx5GKs4r1Js3yB9bNy5rLSFfik3Md0mXAW0lFZL/P01TJi7JlvxE4SNKRzbansF2rI+Ji4BJyPVZRvhM6A/h5RHytMOo24LK67d40It5dZb3WvxwwbCS+DhwhafdUd61NJL1G0lTgSuAO4Pg8fCOVtCaStGuefxKpGe8jpCa99c4ADpP0glyu/lngiohYOsy0n0E6Eb+etcVRSHqjpJn5572kE29Zeup9F9gQ2LcwrFn+APwV2L5uOZcB7wN+kX9fCrwfuDwiaukYbl7cDuwDfKBWmV5P0oGSDpa0RU7zbqS7m9+UTL4A2AT4YN3wH5Pqdd6i1JBgUt7Pz2mRPutzDhg2bBGxmFROfzLp5HojqfiCfHI7ANiBVLm7nFShXW8z0on1XlIxy0rg30vWdTHwL6Sy8DuAZ5DqHobrPGBH4K8R8YfC8F2BKyQ9lKf5YETc0mpheXs/QSrGqQ1rmD/Z54CP52Kbf87DLiPV6dQCxuXAlMLvEeVFrt/ZBzhKUlkx0705zTcADwCnA/8WEYtKpj2EVB9zb6Gl1LzcoupVOU23k4oKP08KqEg6RtKFVdJr/cUP7pmZWSW+wzAzs0ocMMzMrBIHDDMzq8QBw8zMKhm4gCHpTEkH5e9r+iSqMN8xkr7R1cT1AfWo+2lJS8u6w+ji+jrSPXcvSZqd0z0qHTXmrj1uyK2XDhrhsvaWtLxDSau6zr7qSr2d802vtXusKfXxtm/rKdc1UD2OKnXc9nzSw1ZtiYjPtp5qsCh19PbOiNizNiwijhi9FCWSvgUsj4j1ermtOP+lwOkRMeYDfJd9Cjg5Ik4a7YQMRz8cy2PY8cBXgf/XzkyDdodxOLAo+rQtcC+vJEfrqtVGxzD39xBwXafTYp3X6//n3JfYZpLmtjvjwHxIvXfuWfh9KOkJ2NrvfyM96LR5ybyfJF21AmxEeiBpJXAfcBXw1AbrXErq3O1/SQ81nQZslMftTXog7SjSw0m1p31PJD2wdHv+vmHd9McAd+dlzyusa3NSJ24rSA+xfRyYUNjW/wH+A7iH9NBW7anoh4D78nTfAj5TWOa7SA+M3UN6EG3rwrgAjiA9pHUvqT+n2rM5zyB1CbEyp3UR8JS6fHlFSX7NJ3Xg91hO1/l5+HNITy3fRzqJva5Bfi/I2/RInv/kVmnN498OXJ/HXQQMNVj+7Lyst5EeKLwbOLYwvj7/9ibdLRW3+yPANaQn079JeuHUhcCDwM+ALerWNT8fC3cAHy4sawJwNHBTzuezgS3r5n1HTucvGmxP6f7Ny1wN/C3n44Yl825NOo5WkF4G9YHCuI1zXtxLOvY/UpcPu5B6MX4Q+D7wvbp8ey2pK5P7gF8BOxfGHUXqxuRBYAmwT4NtW7MvWPu/82HgrpyXhzU5V2ye980deV2fASZWPLa3Bc7J+bKStcfgoaTzy7/nfLkF2K9JGhrmEeXnji1IT8mvyMv/MTCzsLxLSQ97XgncT+rDq/54KT2uG6Tv68An2joHtzPxaH5IXRAEMKMwrLYDJ+SNvwiY0mD+T7I2YBwOnE96gnYi8EJgswbzLQX+mA+iLUkn7eJOf4K1T7FuTCoG+A2pW+cZpH+WT9dN/8U8/V6kk86z8vjv5INgaj4A/gy8o7CtT5C6idggr+tQCgGz5J/s5fnA2SWv7z8pnHhyfv6Y1K34rHyg7pvH7QC8Ms83g/Sk8Yl1+bJewKhPQ/49iXRSOwaYnNP1YG27S+a/lFTURsW0HpSX/5ycNx8HftVg2bPzsr6e8/D5pB5kn9Mg7XuzfsD4DSlIbEM6ef0W+LucV5eQ/wkL6zqTdPzulNP9ijz+yLysmXner5H6tSrO+50878Yl29Jq/zbbRxOAq4F/zftke9IF2avz+ONJHTJuSTr2/1jLhzz9MlKXIJNIvds+xtrjbpecL7uT/r/eltOyIfAsUl9TWxe28xmtjiPW/u98Kq9zf1IHk1s0mPfcnJ+bkP4XrwQOb3Vs5/T+gXRhtgnp4nLPwv/g46QgPRF4N+lCQCXrb5VHte0pnjumkbqqmUI6B3wfOLfu/+IvpH69NiEF+9ML+djwuG6QRx8CzmnrPNypE3q3P6R/ziBf3Rd24BWkyP1DUodtjeb/ZCFz307dVU+T+ZYCRxR+7w/cVNjpj9Wl6SZS99G1368mdfVcPEg2KYw/m9TNw8S8g+cUxh0OXFrY1lvr0nYozQPGN4ETCuM2zQf87Pw7WPeO7Wzg6Ab5cBDwu7p8qRowXkK6ippQGHYm8MkG819KecAoTSvp6v4dhXETSCeToZJl1/6xilduV5JeQVqW9r1ZP2AU7wp/CHy18Pv95H/ywrqeXRh/AvDN/P16ClfXpHdbPE4KerV5t29ybLbav8320e4lx9PHgNPy95vJATn/ns/agPFS0omreId3eeG4+yr5IqkwfgnpAmkHUjB5BTCpxf/et1j3BPs3YIPC+LuAF5XM91TS/9LGhWGHkDpJbHpsAy8mBfUNSqY7lPT63drvKXkfPa1k2lZ5tDd1546SZbwAuLfu/+L4wu85eRkTaXFcN1j+u4BLmu2D+s8glYPfl/9OJRVX1OxAiqa7RcRjFZf1XdJV01mSnkIqnjo2Uq+jZYpdRS9j3e6pV0REMT3rdD1dMv29EfFwyfjprL0qKY5r1WV1M1uTrn6B1A22pFo32Evz4NIuwSVtBXyJdLKfSjoJ39vm+ovpuC0iVheG1W9bFY26Lx8CTpL0hcJ45eUX87PKsqr4a+H730p+1y+r/vjZKX8fAn4kqZgvT5JOeGXz1quyfxsZArbOXY/XTGRtN+9bl6S7uN6/RD7rlKRzCHibpPcXhk0m3VVclnvL/STwXEkXAR+K9GKrVlbGut3lN9pvQ6Sr+ju09qWLE2ppbHFst+qWf81xExGr8vLL0tAqj6Du3CFpCunOZl9S8RTAVEkTY23Hk/X7ZBLp3LFe+mh9XE9l7Xm1koGp9M4n2ZtI3V8XXQ8cBlyY3xtQZVmPR8RxETEH+D+k8ta3Npll28L3WaTb0DWLq5t2na6nS6bfIndHXT/+bta+vKc4rlmX1fW/642kS/DP5eXvHBGbAW9m3a67mynLk21zd9g19dvWbP5WbiMVNzyl8Nk4In7V5nIgFRFOKfzuxDscGh0/t5HKwIvp3ihSt+Y1zfJiJPv3NuCWunVPjYj98/g7StJNYdw20jqvwC1O27Rb94g4I1LLvqG8fZ+vkN523Ea6w5heWP9mEVF741+zY7tyt/wttMojWH/ffphUZLd7TtdL8/BGy5hFOmfcPcw0PodU/FbZwASM7ALWvrJyjXwgHgP8TNIzWi1E0ssk7aT0FrEHSJnerAvr90qaqfQe6GNIRWCNnEnqgXSGpOmkMuLT66Y5TtJkSS8hBavv5yuIs4EFkqZKGiKVMdbPW/RXYKakyQ3Gj6RL8KnkynRJ21D+jopm6Sp2230F6UT80dzV9d6knmzLXtlaNn8rpwAfU34FqKTNJb2xjfmLfg/sL2lLSU8j1TOM1L9ImpLTdxhrj59TSPt7CCAfMwe2sdyR7N8rgQckHSVpY0kTJT1P0q55/NmkPN1Cqbv34t3Cr0n/L++TtEFO826F8Q27dZf0LEkvz+l9hHRHVqX7+MoivbHxJ8AXJG0maYKkZ0iqnTuaHduVuuWvoFUelZlKyo/78rnmEyXTvFnSnHw38inSq4WHm397kYpzKxu0gLEQmFcXtQGI9PrLTwGXaN1XVZZ5Gumtaw+Q7lAuo/mJ+QzSAXhz/pS9jrPmM8BiUiuaa0lFBsXp7yTd/t5Oap1xRET8KY97P+nEejOpvPMM0juhG7mE1OLoTknrXWXEyLoEP45UeXk/6VWd51ScD1LZ+hylbrvPzUWFrwP2I10NfQV4a2G7650EvEHSvZK+1GplEfEj0lXqWZIeIFXQ7tdGeou+S7rqWkra580uDqq6jFQpfzHw7xHxkzz8JFLLpp9IepBUAb571YWOZP/G2u7nX0Bq7XM38A1S6yJI+39ZHvcTUr7U5n2MVIn7DlKRxptJDRIezeObdeu+IalC/W7S/8JWpIuwTnsrqRis1rrxB6Q6otq2lR7bUb1b/qZa5VEDJ5IqrO8mHQtlz0h8l1S3cyepQv4DVdIjaZ6k6wq/dwUejgav6m24nHWL2PqfpDNIr5k8t0frW0qqgP1ZB5a1N6nifWaLSc0GiqQrgFMi4rTRTku/GmkedfKBVkk/JDW+uKCd+Qap0huAiGj7KW8z66xcvLOEdDU8D9iZNp8aHuv6OY8i4vXDmW/gAoaZ9YVnkeo5NiU1RnlDrjuwtcZcHg1ckZSZmY2OQav0NjOzUTJwRVLTp0+P2bNnj3YyzMwGytVXX313RMwYyTIGLmDMnj2bxYsXj3YyzMwGiqRGvR5U5iIpMzOrxAHDzMwqccAwM7NKHDDMzKwSBwwzM6tkXAeMRdcuYvaJs5lw3ARmnzibRdcuGu0kmZn1rYFrVtspi65dxPzz57Pq8VUALLt/GfPPnw/AvJ3mjWbSzMz60ri9wzj24mPXBIuaVY+v4tiLjx2lFJmZ9bdxGzBuvf/WtoabmY134zZgzNp8VlvDzczGu3EbMBbss4Apk6asM2zKpCks2GfBKKXIzKy/jduAMW+neSw8YCFDmw8hxNDmQyw8YKErvM3MGhi492HMnTs33PmgmVl7JF0dEXNHsoxxe4dhZmbtccAwM7NKHDDMzKwSBwwzM6vEAcPMzCpxwDAzs0ocMMzMrBIHjAHjLtnNbLQ4YPSZZgGh1iX7svuXEcSaLtkdNMysFxww+kirgOAu2c1sNDlg9JFWAcFdspvZaHLA6ILh1jO0Cgjukt3MRpMDRoeNpJ6hVUBwl+xmNpocMDpsJPUMrQKCu2Q3s9G0wWgnYKwZST1D7cR/7MXHcuv9tzJr81ks2GfBOgFh3k7zHCDMbFQ4YHTYrM1nsez+ZaXDq3BAMLN+5SKpDnM9g5mNVQ4YHeZ6BjMbq/yKVjOzccCvaDUzs55xwBgj3CmhmXWbW0mNAbWHBWvPf9QeFgRcd2JmHdPVOwxJ+0paIulGSUc3mW5XSU9KekM30zNWuVNCM+uFrgUMSROBLwP7AXOAQyTNaTDd54GLupWWsc6dEnaOi/bMGuvmHcZuwI0RcXNEPAacBRxYMt37gR8Cd3UxLWOaOyXsDL9vxKy5bgaMbYDbCr+X52FrSNoG+HvglGYLkjRf0mJJi1esWNHxhA46PyzYGS7aM2uumwFDJcPqH/o4ETgqIp5stqCIWBgRcyNi7owZMzqVvjHDDwt2hov2zJrrZiup5cC2hd8zgdvrppkLnCUJYDqwv6QnIuLcLqZrTHIfVCM30n7AzMa6bt5hXAXsKGk7SZOBg4HzihNExHYRMTsiZgM/AN7jYGGjxUV7Zs11LWBExBPA+0itn64Hzo6I6yQdIemIbq3XbLhctGfWnPuSGqMWXbuo6Xs1zGx8cV9SVqofmof6eQazsccBY4BUPQmPdvPQfghYZtZ5DhgDop2T8Gg3Dx3tgGVm3eGAMSDaOQmP9MnvkRYnjXbAMrPucMAYEO2chEfSPLQTxUnuqsRsbHLA6DONru7bOQmPpHloJ4qT/DyD2djkZrV9pP69FpBOtAsPWAjQcFwnm8tOOG4CsV4PLiDE6k+srrwcN+s16y+daFbrFyj1kWZX90uPXLpmmm6ehDvVPYa7KjEbexww+kireopenIQX7LOg9E7GxUlm5jqMPtIPlcXuHsPMGvEdRh/pl6t7FyeZWRnfYfQRX92bWT9zKykzs3HAnQ+amVnPOGCYmVklDhhmZlaJA4aZmVXigGFmZpU4YNh6/LY8MyvjB/dsHfUdINa6Nwf8PIjZOOc7DFuH35ZnZo04YNg6+vVteS4m61/t7hvvy8HlIilbR6e6N+8kF5P1r3b3jfflYPMdRo8MylVVq7fljcZ2uJisf7W7b7wvB9vA9SW15dBz4pXHnDrayWjLXQ/fxZKVS1gda99YN0ETeNa0Z7HVJlt1ZPm33Hszjzz5KBtN3JDttth+RMtttLxub0cjly27rOG4vYb26tp6rbV294335eg5+4j/4zfuDYJb7r15nZMswOpYzS333jziE239SfyRJx9lycolAMNe9labbFU6bze3o5mNJm7II08+WjrcRle7+8b7crANXMDYfsYmfO/wF492Mtoy4bg9iA3K35O99PDq78kuM/vE2dwxef06h8kThvj54UtHtOx63dyOZhZde3Ppe0K+cMBC5u1U/Vjwe8Y7r91906l9ae07+4iRL8N1GD3QzTfp9bJV02i9EbAT7wmpVbYuu38ZQaypbO3XuqRB0e6+8TtfBtvA1WEM4vsw6luGQLqq6sQ/yuwTZ5e2ahrafIilRy4d0bLrdXM72k1HozuFRuN6mU9m/agT78MYuCKpQVQ7mXWjOKSXr3Xt5nZU1axZJtBwXL8+X2I2SHyHMQaMp7L5ZncKwLDG+Q7DxgPfYRiQrvz7IUD0InAN507h1vtv5bv/8N2e3YmZjVWu9LaO6FWlcrOK92bjXNnaWC8fxhyUB1itXFeLpCTtC5wETAS+ERHH140/EPg0sBp4AjgyIi5vtkwXSfWnXlUqN6t4B/qiUn6Q9LIhQ780mhivOlEk1bU7DEkTgS8D+wFzgEMkzamb7GLg+RHxAuDtwDe6kZbiVc30E6Yz/YTpvsLpsF5VKje7U/BdRPt62VWHuwUZfN2sw9gNuDEibgaQdBZwIPC/tQki4qHC9JsAHb/dqb+qWfm3lWvGueOzzullp4XN6mz6pT5nUPSy9Zhbqg2+btZhbAPcVvi9PA9bh6S/l/Qn4L9JdxnrkTRf0mJJi1esWNFWIsquaop8hdMZrTottP7Uy4cxR+vBz7GgX+p+uhkwVDJsvTuIiPhRRDwbOIhUn7H+TBELI2JuRMydMWNGW4mocvVSNk2/7KBB4eKgwdTLQO+LiuHpp14KulkktRzYtvB7JnB7o4kj4heSniFpekTc3alENCoqqZ+myH32D4+LgwZPLx/G7IcHPwdRs7qfXudd11pJSdoA+DOwD/AX4CrgnyLiusI0OwA3RURI2gU4H5gZTRLVbiupspYZRWWtNNyNhJn1iwnHTSBKqneFWP2J6p1+9nUrqYh4AngfcBFwPXB2RFwn6QhJtX4TXw/8UdLvSS2q/rFZsBiO+qKSaRtPY9rG05oWm4yHyjkXuVmRj4f+1U91P+4apEQ37zD6oRsPt4e3Ih8P/a1T+6ev7zAGWbcq5/ql8srt4a3Ix0N/66cGJb7DaKAbdwL9UjfSqTJRGxt8PIwPPet8UNIewCeBoTyPgIiI7Uey8n7WjRY//VI30suH7Kz/+XiwqqoWSX0T+CKwJ7ArMDf/tTb0S+WV28NbkY8Hq6pqwLg/Ii6MiLsiYmXt09WUjUH98o/ZT2WiNvp8PFhVleowJB1P6nH2HODR2vCI+G33klZu0Hur7YdWUmY2/nSiDqNqwPh5yeCIiJePZOXD0S8Bwyd+MxskPav0joiXjWQlY427DjGz8ahSHYakzSV9sdZjrKQvSNq824nrV91ot+4nbc2s31Wt9D4VeBB4U/48AJzWrUT1u043j+2XB/rMzJqpGjCeERGfiIib8+c4YMw+g9FKp5vH+klbMxsEVQPG3yTtWfuRH+T7W3eS1P863Ty2Xx7oMzNrpur7MN4NfDvXWwi4Bzi0W4nqd53u199P2prZIGirLylJmwFExANdS1EL/dKstpPcW6iZdVvXe6uV9Ob890OSPgS8E3hn4fdA65eWSf32pG2/5IuZ9ZdWRVKb5L9Tu52QXuu3Zyn65fWm/ZYvZtY/xm335v3S1Xi/cb6YjU09e4GSpBMkbSZpkqSLJd1dK64aVG6ZVM75YmaNVG1W+6pc0f1aYDnwTOAjXUtVD/RLV+P9xvliZo1UDRiT8t/9gTMj4p4upadn+qWr8X7jfDGzRqoGjPMl/Yn04qSLJc0AHulesnpj4w02XvN92sbT1rRMGs+thPqtxZaZ9Y/Kld6StgAeiIgnJU0BNouIO7uauhKdqPRu9twD4GcizGzM6fr7MCS9PCIukfQPZeMj4pyRrHw4OhEwmrUEAtxKyMzGnF68D2Mv4BLggJJxQXoD38AZTksgtxIys/GuacCIiE/kv4f1Jjm90arvJvfrZGa2vqrPYXxW0lMKv7eQ9JmuparLmrUEcishM7NyVVtJ7RcR99V+RMS9pCa2A6lZSyC3EjIzK1eplZSka4BdI+LR/HtjYHFEPLfL6VvPWOyt1sys23pR6V1zOun5i9NIld1vB749khWbmdlgqRQwIuKEfJfxCtILlD4dERd1NWVmZtZXqt5hAFwPPBERP5M0RdLUiHiwWwkzM7P+UrWV1LuAHwBfy4O2Ac7tUprMzKwPVW0l9V5gD+ABgIi4AdiqW4kyM7P+UzVgPBoRj9V+SNqAVPndlKR9JS2RdKOko0vGz5N0Tf78StLzqyfdzMx6qWrAuEzSMcDGkl4JfB84v9kMkiYCXwb2A+YAh0iaUzfZLcBeEbEz8GlgYTuJNzOz3qkaMI4CVgDXAocDFwAfbzHPbsCNEXFzvjs5CziwOEFE/Co/BAjwG2Bm1YSbmVlvtWwlJWkCcE1EPA/4ehvL3ga4rfB7ObB7k+nfAVzYIA3zgfkAs2a5Tyczs9HQ8g4jIlYDf5DU7plaZYsrnVB6GSlgHNUgDQsjYm5EzJ0xY0abyTAzs06o+hzG04HrJF0JPFwbGBGvazLPcmDbwu+ZwO31E0naGfgGqb+qlRXTY2ZmPVY1YBw3jGVfBewoaTvgL8DBwD8VJ8h3LecAb4mIPw9jHWZm1iNNA4akjYAjgB1IFd7fjIgnqiw4Ip6Q9D7gImAicGpEXCfpiDz+FOBfgWnAVyRBepJ8RJ1jmZlZd7R6Rev3gMeBX5Kaxy6LiA/2KG2l3FutmVn7etFb7ZyI2Cmv7JvAlSNZmZmZDa5WraQer32pWhRlZmZjU6s7jOdLeiB/F+lJ7wfy94iIzbqaOjMz6xtNA0ZETOxVQszMrL9V7RrEzMzGOQcMMzOrxAHDzMwqccAwM7NKHDDMzKwSBwwzM6vEAcPMzCpxwDAzs0ocMMzMrBIHDDMzq8QBw8zMKnHAMDOzShwwzMysEgcMMzOrxAHDzMwqccAwM7NKHDDMzKwSBwwzM6vEAcPMzCpxwDAzs0ocMMzMrBIHDDMzq8QBw8zMKnHAMDOzShwwzMysEgcMMzOrxAHDzMwqccAwM7NKHDDMzKySrgYMSftKWiLpRklHl4x/tqRfS3pU0j93My1mZjYyG3RrwZImAl8GXgksB66SdF5E/G9hsnuADwAHdSsdZmbWGd28w9gNuDEibo6Ix4CzgAOLE0TEXRFxFfB4F9NhZmYd0M2AsQ1wW+H38jysbZLmS1osafGKFSs6kjgzM2tPNwOGSobFcBYUEQsjYm5EzJ0xY8YIk2VmZsPRzYCxHNi28HsmcHsX12dmZl3UzYBxFbCjpO0kTQYOBs7r4vrMzKyLutZKKiKekPQ+4CJgInBqRFwn6Yg8/hRJTwMWA5sBqyUdCcyJiAe6lS4zMxuergUMgIi4ALigbtgphe93koqqzMysz/lJbzMzq8QBw8zMKnHAMDOzShwwzMysEgcMMzOrxAHDzMwqccAwM7NKHDDMzKwSBwwzM6vEAcPMzCpxwDAzs0ocMMzMrBIHDDMzq8QBw8zMKnHAMDOzShwwzMysEgcMMzOrxAHDzMwqccAwM7NKHDDMzKwSBwwzM6vEAcPMzCpxwDAzs0ocMMzMrBIHDDMzq8QBw8zMKhkfAWPRIpg9GyZMSH8XLRrtFJmZDZwNRjsBXbdoEcyfD6tWpd/LlqXfAPPmjV66zMwGzNi/wzj22LXBombVqjTczMwqG/sB49Zby4cvWwYbbACSi6nMzCoY+wFj1qzG4558Mv2tFVM5aJiZNTT2A8aCBTBlSuvpVq2CD34wfV+0CKZPT3cfUvruYGJm49zYDxjz5sHChTA01HralSvhPe+Bww5L34vD3/72dYNGs5ZXZeNqw6TyorBW45tptj63DDOzTomIrn2AfYElwI3A0SXjBXwpj78G2KXVMl/4whfGsA0NRUDzj9R43NBQWs7pp0dMmbLuuClT0vCycZMmRUyeXL7MKVMi3v3u9ecpznv66WvXO23a2nHTpjWed+LE8vTVnH562h4p/S2O22efdefdZ59q8xXHTZuWPmXTDUfVZVdNX6M0VZlmrCk7rtrZ7nbybKTrGq86cFwCi2Ok5/SRLqDhgmEicBOwPTAZ+AMwp26a/YELc+B4EXBFq+WOKGCUnczb+UhpOY0Cz9BQtaDU6uRe/6n9U02aNPy0Vw149cGiGDTaDZTNAlYn91uzNFQZ12w9I0n3IDj99PKLmeKFSqv5q+ZZo2N48uSxnccj1aHjst8DxouBiwq/PwZ8rG6arwGHFH4vAZ7ebLkjChgRayP1SE64je5CpOZ3KCM92Y90GVUC3nDSUDVQ1vKvXVWXPZz0FdNUZZqxplneVtnudvJspOsarzp0XHYiYCgtp/MkvQHYNyLemX+/Bdg9It5XmObHwPERcXn+fTFwVEQsrlvWfGA+wKxZs164bNmykSdw+vR16ylqNt0UHn0UHn983eGTJ8Opp6Y6kdmzU8uqerV6knbTN3Hi2hZbjUjpMBmJoSFYujTVa5Qtq9U6Go2X0t9W6ZNg9erKyV2jUXqrpqHVuFqamuXLcNI9CJrlbZXtbifPRrqu8apDx6WkqyNi7oiSMpKZW1DJsPqtrjINEbEwIuZGxNwZM2Z0JHGcdFIKAkWTJ8Mpp8Bpp8G0aWuHT5u2NlhAecurKVPS8LJxkyatv67ifPPnN2/JNW1a8+bBVdTSB42X1Wodzearkr7hbkPVZQ8nfcXhw82XQdZs20ayT8uGj3Rd41U/HZcjvUVp9KFfi6SKRlKR1G7larEorFZnUZyvvjKw9qmV7zYr/91nn/WLwiZPbl4x7DoM12FEuA5jEIyTOowNgJuB7Vhb6f3cumlew7qV3le2Wm5HA0Y/ahWIGrUwaTf4uZWUW0nVuJVU/+uTVlJdq8MAkLQ/cCKpxdSpEbFA0hH5zuYUSQJOJjW/XQUcFnX1F/Xmzp0bixc3ncTMzOp0og6jq73VRsQFwAV1w04pfA/gvd1Mg5mZdcbYf9LbzMw6wgHDzMwqccAwM7NKHDDMzKySrraS6gZJK4Aqj1JPB+7ucnL62XjffnAejPftB+dBcfuHImJETz4PXMCoStLikTYhG2TjffvBeTDetx+cB53efhdJmZlZJQ4YZmZWyVgOGAtHOwGjbLxvPzgPxvv2g/Ogo9s/ZuswzMyss8byHYaZmXWQA4aZmVUy5gKGpH0lLZF0o6SjRzs9nSTpVEl3SfpjYdiWkn4q6Yb8d4vCuI/lfFgi6dWF4S+UdG0e96Xca3Dfk7StpJ9Lul7SdZI+mIePizyQtJGkKyX9IW//cXn4uNj+IkkTJf0uv7VzXOWBpKU53b+XtDgP6832j7R/9H76kLpRvwnYnrXv4Jgz2unq4Pa9FNgF+GNh2AnA0fn70cDn8/c5efs3JL2T5CZgYh53JekFVyK9j2S/0d62itv/dGCX/H0q8Oe8neMiD3JaN83fJwFXkN4jMy62vy4vPgScAfw4/x43eQAsBabXDevJ9o+1O4zdgBsj4uaIeAw4CzhwlNPUMRHxC+CeusEHAt/O378NHFQYflZEPBoRtwA3ArtJejqwWUT8OtJR853CPH0tIu6IiN/m7w8C1wPbME7yIJKH8s9J+ROMk+2vkTST9PK1bxQGj6s8KNGT7R9rAWMb4LbC7+V52Fj21Ii4A9IJFdgqD2+UF9vk7/XDB4qk2cDfka6yx00e5KKY3wN3AT+NiHG1/dmJwEeB1YVh4ykPAviJpKslzc/DerL9XX2B0igoK4Mbr+2GG+XFwOeRpE2BHwJHRsQDTYpex1weRMSTwAskPQX4kaTnNZl8zG2/pNcCd0XE1ZL2rjJLybCBzgNgj4i4XdJWwE8l/anJtB3d/rF2h7Ec2LbweyZw+yilpVf+mm8vyX/vysMb5cXy/L1++ECQNIkULBZFxDl58LjKA4CIuA+4lPR64/G0/XsAr5O0lFTk/HJJpzOO8iAibs9/7wJ+RCqK78n2j7WAcRWwo6TtJE0GDgbOG+U0ddt5wNvy97cB/1UYfrCkDSVtB+wIXJlvVx+U9KLcKuKthXn6Wk7vN4HrI+KLhVHjIg8kzch3FkjaGHgF8CfGyfYDRMTHImJmRMwm/X9fEhFvZpzkgaRNJE2tfQdeBfyRXm3/aNf4d/oD7E9qPXMTcOxop6fD23YmcAfwOOkK4R3ANOBi4Ib8d8vC9MfmfFhCoQUEMDcfZDcBJ5Of+O/3D7An6bb5GuD3+bP/eMkDYGfgd3n7/wj8ax4+Lra/JD/2Zm0rqXGRB6QWoH/In+tq57hebb+7BjEzs0rGWpGUmZl1iQOGmZlV4oBhZmaVOGCYmVklDhhmZlaJA4ZZA5JmSvqv3APoTZJOys/3tJrvUklze5FGs15ywDArkR9mOgc4NyJ2BJ4JbAosqJtuxN3rSJo40mWY9cJY60vKrFNeDjwSEadB6sNJ0v8FbpF0C/AyYCNgE0mvAU4jdSV9PbBxbSGSXgUcR+pe+ibgsIh4KHdtcSrpSd2TSd1cmPU1Bwyzcs8Fri4OiNTR4a2k/5sXAztHxD2SPgSsioidJe0M/BZA0nTg48ArIuJhSUeR3uPwqbzIRyJizx5tj9mIOWCYlRPlvXfWhv80ImrvJnkp8CWAiLhG0jV5+ItIdx3/k3vUnQz8urCs73Uh3WZd44BhVu464PXFAZI2I/X8+STwcN30jYLLTyPikAbrqF+GWV9zpbdZuYuBKZLeCmsqpr8AfAtYVTftL4B5ebrnkToJBPgNsIekHfK4KZKe2f2km3WHA4ZZiUi9cv498EZJN5B6QH4EOKZk8q8Cm+aiqI+S3pVMRKwADgXOzON+Azy7+6k36w73VmtmZpX4DsPMzCpxwDAzs0ocMMzMrBIHDDMzq8QBw8zMKnHAMDOzShwwzMyskv8PQa9vzq1VyyUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#[TP, FP, FN, TN]\n",
    "\n",
    "def precision(x):\n",
    "    return x[0]/(x[0]+x[1])\n",
    "\n",
    "def random_precision(x):\n",
    "    # Can calculate precision of random guessing analytically from hypergeometric distribution.\n",
    "    K = np.floor(0.25*x[1])\n",
    "    N = comb(x[0], 2) - x[1]\n",
    "    TP = K**2/N\n",
    "    FP = K-TP\n",
    "    return (TP/(TP+FP))\n",
    "    \n",
    "\n",
    "keys = [x for x in test_order_Cmatrices.keys() if not test_order_Cmatrices[x][0]<10]\n",
    "\n",
    "orders = [x[0] for x in keys]\n",
    "precisions = [precision(test_order_Cmatrices[x]) for x in keys]\n",
    "random_precisions = [random_precision(x) for x in keys]\n",
    "\n",
    "# Plot precision\n",
    "fig, axs = plt.subplots()\n",
    "axs.scatter(orders, precisions, color=\"g\")\n",
    "axs.scatter(orders, random_precisions, color=\"r\")\n",
    "plt.axhline(y=np.mean(precisions))\n",
    "axs.set_title(\"Precision vs Network Size. \\n (k is proportional to the number of edges in each graph.)\")\n",
    "axs.set_xlabel(\"Order\")\n",
    "axs.set_ylabel(\"Precision\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d5850c8",
   "metadata": {},
   "source": [
    "## Test 2: Evaluate for differnet values of k on a single graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "id": "76313b0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 1\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 10\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 48\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 195\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 198\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 225\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 266\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 347\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 533\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 890\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 1159\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 1569\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 2293\n",
      "Executing SQL to expand target subgraph. (Initial vertex: 0)\n",
      "              SQL complete. Current size: 3904\n"
     ]
    }
   ],
   "source": [
    "warnings.filterwarnings('ignore') # Ignore warnings.\n",
    "    \n",
    "# Connect to graph db.\n",
    "dcon = sqlite3.connect(\"yt_graph.db\")\n",
    "    \n",
    "# Init vertex list.\n",
    "vertices = pd.Series([0], name=\"vertex\")\n",
    "whileBreak = 0\n",
    "    \n",
    "# Explode.\n",
    "while len(vertices)<5000:\n",
    "        \n",
    "    # First, enlarge search by adding neighbors of the current subgraph.\n",
    "    vertices.to_sql(\"vertexList\", con=dcon, if_exists=\"replace\")\n",
    "    print(\"Executing SQL to expand target subgraph. (Initial vertex: \" +\n",
    "            str(0) + \")\")\n",
    "    neighbors = pd.read_sql_query(\n",
    "    \"\"\"\n",
    "    SELECT DISTINCT\n",
    "        x.v2\n",
    "    FROM\n",
    "        ytGraphUndirected x\n",
    "    WHERE EXISTS (\n",
    "        SELECT *\n",
    "        FROM vertexList y\n",
    "        WHERE y.vertex = x.v1\n",
    "        OR y.vertex = x.v2\n",
    "    )\n",
    "    \"\"\", dcon)\n",
    "    print(\"              SQL complete. Current size: \" + str(len(vertices)))\n",
    "    vertices = pd.Series(neighbors.squeeze(), name=\"vertex\")\n",
    "    whileBreak = whileBreak+1\n",
    "    if whileBreak>20:\n",
    "        break\n",
    "        \n",
    "# Turn vertices into a graph.\n",
    "G_short = G_full.subgraph(list(vertices))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "id": "447947ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions on this graph.\n",
    "preds = eval_model(G_short, 1, get_all_preds=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "id": "d71db8f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Figure out which edges were removed because I'm dumb.\n",
    "G_short_edges = [(min(edge), max(edge)) for edge in G_short.edges()]\n",
    "removed_edges = [edge for edge in preds if edge in G_short_edges]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "id": "078913be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAt+klEQVR4nO3deXxU1fnH8c+TBRKCgEDCFhCQVXaI4C5iUcAFtNriUlBQpK21ttXWWltbi/6qtgoKrtTijrug4oIIoiKr7CL7jpCw74Ek5/fH3MQhTsIAmdyZzPf9es0rd+49c+Y5c2GeOefee6455xARkfiV4HcAIiLiLyUCEZE4p0QgIhLnlAhEROKcEoGISJxTIhARiXNKBFImzOw6M/skjHJPmdlfI/D+Zmb/M7MdZjazrOsva2Y2xsyG+R0HgJlNMbObvOWw9mMJ9XxoZgPLNjopD0l+ByCRZ2ZrgDpAPrAPmAD8xjm3t6zewzn3MvByGOWGltV7FnMO0BPIdM7tO9HKzKwxsBpIds7lnWh9sSLc/WhmfweaOeeuD3pt7wiGJhGkHkH8uMw5VxXoDJwO3FO8gJnF8g+DU4A1x5MEYrzdR6hIbZHyo0QQZ5xzG4EPgbYAZubM7NdmthxY7q271MzmmdlOM5tmZu0LX29mDc3sbTPLMbNtZjbSW3+DmX3pLZuZPWpm2Wa2y8wWmFnh+x0xJGJmN5vZCjPbbmbjzax+0DZnZkPNbLk35DPKzKx4m8xsMDAaONPM9prZP8Ks+4h2FzPV+7vTq/NMM0sws3vMbK3XthfMrLpXX2OvziFmtsnMvjezP4SzT8zsJDObbGaPldC+KWb2f2Y20/s8x5lZzWLvO9jM1gGfeesHmdkS73P72MxOCaqvp5l959U1ErCgbUX70Xvexswmep/hFjO728x6AXcDP/c+m/lBcRYOMYXzWQ00s3VmttXM/hL0nl3NbLaZ7fbe85FwPkc5Ac45PSr4A1gD/MRbbggsBv7pPXfARKAmkEqgx5ANdAMSgYHe6yt7z+cDjwJpQApwjlfPDcCX3vLFwBygBoEvmdZAPW/bGGCYt9wD2Oq9Z2XgcWBqUNwOeN+rpxGQA/QqoY1F738MdRe1O0R9jb0ySUHrBgErgKZAVeBt4MVi5V/1Ppt2Xrw/KSHeMcAwoBYws/AzKaHsFGAjgeSdBrwFvFTsfV/wtqUC/bw4WxMY/r0HmOaVrw3sBq4CkoHfAXnATSH240nA98AfvH19EtDN2/b3whiKxXnTMXxWz3rxdgBygdbe9q+BX3jLVYEz/P4/VNEfvgegRzns5MAX+V5gJ7AWeKLwy8/7D9kjqOyTeEkiaN1S4HzgTO/LLSnEewR/gfQAlgFnAAnFyo3hh0TwX+ChoG1VgcNA46DYzgna/jpwVwltLHr/Y6i7R6i6vO2FX1bBiWAS8Kug5y29OpOCyrcK2v4Q8N8S6h8DPAcsAu48yv6bAvwr6PlpwCECibnwfZsGbf8QGBz0PAHYT2D4bAAwPWibARsInQiuAeaWENPfKT0RhPNZZQZtnwn095anAv8Aavv9fydeHhoaih/9nHM1nHOnOOd+5Zw7ELRtfdDyKcAfvGGhnWa2k0Avor73d607ysFT59xnwEhgFLDFzJ4xs2ohitYnkJgKX7cX2AY0CCqzOWh5P4Ev9HCEU/f64i86ljq95SQCB+JD1bnWe01JLiHwi/ipMN67eL3JBH7dh9p+CjAiaP9tJ/CF38CLp6isC3zzlvQ5NARWhhFbKOF8ViXt28FAC+A7M5tlZpceZwwSJiUCgcCvs0Lrgfu9pFH4qOKce9Xb1sjCOCDpnHvMOdcFaEPgP/WdIYptIvClBYCZpREYKtl4Am05lrpLm3o31LYj6iQwXJUHbAla17DY9k2lvMezwEfABC++0hSv9zCBoa9Q8a4Hbim2D1Odc9MIDPUU1eUdkwium2L1nFrCtqNNWxzOZxW6YueWO+euATKAB4E3w/h85AQoEUhxzwJDzaybd9A3zcwuMbOTCHTfvwf+5a1PMbOzi1dgZqd7r08mcLrqQQKnrhb3CnCjmXU0s8rAA8AM59yaMmjHidadAxQQGOMu9CrwOzNrYmZVvTpfK9ZD+quZVTGzNsCNwGtHeZ9bCQy9vW9mqaWUu97MTjOzKsB9wJvOuVCfKQR6GH/2YsDMqpvZ1d62D4A2Znall9BvA+qWUM/7QF0zu93MKnsHtbt527YAjc2spO+QcD6rkMzsejNLd84VEBjOhND/fqSMKBHIEZxzs4GbCQzt7CBwwO8Gb1s+cBnQDFhHYGz55yGqqUYgoewgMCSwDfh3iPeaBPyVwMHP7wn8+uxfRu04obqdc/uB+4GvvCGWMwiM6b9IYAx7NYEE95tiL/2cwGc2Cfi3c67Ui7O8oZkhBH59jzOzlBKKvkjguMJmAgdubyulzncI/JIea2a7CRyH6O1t2wpcDfyLwH5pDnxVQj17CFybcZn3vsuBC7zNb3h/t5nZNyFeHs5nVZJewGIz2wuMIHDs4CCAd5bSuWHWI2GywL9DETkRFsEL0MxsCoEDs6PLsl6RQuoRiIjEOSUCEZE4p6EhEZE4px6BiEici7kJqmrXru0aN27sdxgiIjFlzpw5W51z6aG2xVwiaNy4MbNnz/Y7DBGRmGJma0vapqEhEZE4p0QgIhLnlAhEROKcEoGISJxTIhARiXMRTQRm1svMllrgdoF3hdh+pwVuiTjPzBaZWX7hLfhERKR8RCwRmFkigRuT9CZwR6VrzOy04DLOuYedcx2dcx2BPwOfO+e2RyomERH5sUj2CLoCK5xzq5xzh4CxQN9Syl9DYA7ziPly3Zcszl4cybcQEYk5kUwEDTjyFngbOPI2gUW8m230IjB3fKjtQ8xstpnNzsnJOe6Azv3fubR9su1xv15EpCKKZCKwEOtKmuHuMuCrkoaFnHPPOOeynHNZ6ekhr5AWEZHjFMlEsIEj74WaScn3b+1PhIeFREQktEgmgllAc++epZUIfNmPL17IzKoD5wPjIhiLiIiUIGKTzjnn8szsVuBjIBF4zjm32MyGetuf8opeAXzinNsXqVhERKRkEZ191Dk3AZhQbN1TxZ6PIXBTbhER8YGuLBYRiXNKBCIicU6JQEQkzikRiIjEOSUCEZE4p0QgIhLnlAhEROKcEoGISJxTIhARiXNxmQiGTx/OKwtf8TsMEZGoENEpJqKJcz/MgP27j38HwLXtrvUrHBGRqBE3PYJ8l+93CCIiUSl+EkHBjxNBcC9BRCRexU0iKHAFP1q37cA2HyIREYkucZMIQg0NbdpT0g3TRETiR/wkghBDQ/sP7/chEhGR6BI/iSBEj+Bg3kEfIhERiS5xkwhCHSM4cPiAD5GIiESXuEkEoYaG1CMQEYmnRBBiaOib77/xIRIRkegSP4kgRI9g2BfD+HD5hz5EIyISPeImEYQ6RgDw7nfvlm8gIiJRJm4SQUlTTHy37btyjkREJLrETyIIGho6u+HZRct5BXl+hCMiEjXiJxG40IngcP5hP8IREYkacZMIgo8RJCcmFy3vObTHj3BERKJG3CSC4KGhREssWt57aK8f4YiIRI34SQRBQ0MJ9kOz9+SqRyAi8S1uEkHw0FBiwpE9At2XQETiWdwkgpKGhvJdvqaaEJG4Fj+JoNjQ0Le/+pY7zrwD0HECEYlv8ZMIgnsECYm0Tm9N24y2QOlnDs3YMAP7hzFn05yIxygi4oe4SQRHHCPwhoaqVqoKwPd7vi/xdS8ueBGAKWumRC44EREfxU0iCB4aKjxYfO4p51IjpQYjZ438Ufnv93zPOc+dw//m/Q/4IWmIiFQ08ZMICn58+mhGWganpZ9Gzr6cH5V/9ptn+Wr9V0W3s1y/e71ONRWRCil+EkEJ1xGkJKWEPGuoRkqNI57f/8X9VPtXNXq/3Jste7dELE4RkfIWN4kg1DECgNSkVA7k/fiWlcHlM6tlFi1/tOIjPl75cYSiFBEpf3GTCIqfNVSopB7BjgM7ipZrV6l9xLYFWxZEIEIREX9ENBGYWS8zW2pmK8zsrhLKdDezeWa22Mw+j1QsRxwsDu4RJKeGvIn9joOBRDB54GTSq6Qfse0/X/+HldtXRihSEZHyFbFEYGaJwCigN3AacI2ZnVasTA3gCeBy51wb4OpIxRPqYDEEhoZC9ggO7qBxjcZ0b9yd9LRAIvhZm5/xfL/nATQ8JCIVRiR7BF2BFc65Vc65Q8BYoG+xMtcCbzvn1gE457IjFUxJcw2FGhoqcAVMXDmRk1NOBqBWai0AutTrwi/a/4L0KulMWD6BTXs2AZCzL4eNuzdGKnQRkYiKZCJoAKwPer7BWxesBXCymU0xszlmNiBURWY2xMxmm9nsnJwfn+oZjuChocIriiH0weKpa6eyZd+WomsH+jTvA0DHuh0xM7LqZ/HB8g9o8EigOW2eaEPmo5mIiMSiSCYCC7Gu+DSfSUAX4BLgYuCvZtbiRy9y7hnnXJZzLis9Pb345rD0bNqTWTfPYu3ta+lYt2PR+sIeQfAMpIXj/09c8gQAvZr1YvMfNnPRqRcB0Lle5yPqztkfSE5frP3iuGITEfFTJBPBBqBh0PNMYFOIMh855/Y557YCU4EOkQjm5NSTyaqfRaPqjY5Yn5qcCkBufm7RuvW712MYLWu1LFpXp2qdouXgOg7lHypavuata8o8bhGRSItkIpgFNDezJmZWCegPjC9WZhxwrpklmVkVoBuwJIIx/UhKUgrAEccJ1u9aT92qdY+4pWWwAR0G0OCkwLDQ4uzFRes37tmos4lEJOZELBE45/KAW4GPCXy5v+6cW2xmQ81sqFdmCfARsACYCYx2zi2KVEyhpCYFegSFp5BOXDmRVxa9QsPqDUt8TUpSCs/1fQ4IHE8AeKDHA4HXr5oYyXBFRMpcRK8jcM5NcM61cM6d6py731v3lHPuqaAyDzvnTnPOtXXODY9kPKEU9ggKDxhf9NJFHMw7eMTVxKG0qBU4lDF67mgAft725zSq3ohPVn4SwWhFRMpe3FxZXJLCYwSnPnYq63f9cJJTw2ol9wgATql+CnXS6rAoexF10urQpEYTzm10LrM3zY5ovCIiZS3uE0FaclrR8vLty4uW66TVCVW8iJkxuNNg2qS3YXiv4ZgZLWq1YP3u9SGvVBYRiVZxnwh6NOnBr0//NQCrd6wuWh989XFJ7r/wfhb9ahH92/YHoHnN5gCs2L4iApGKiERG3CeC1ORUhmYNBWDZtmVF68/IPOOY6yo8bhDcsxARiXZxnwjgh+Ghx2c+DgQmmju/8fnHXE/zWoEewfJtSgQiEjuUCPjhNpQH8g7Qs2lPujfuflz1VKtcjYy0DJZvX86CLQuYvHpyGUYpIhIZSgRAWqUfDhg3qdHkhOpqXrM5y7Yto8NTHejxQg8em/EY+w7twznH8m3Lj5jKQkQkGigR8MNFZUDRlNPHq0WtFnyx7oc5h3770W+55q1r6PxMZ1qMbMHTc54+ofpFRMqaEgGBU0ELFb8JzbEqPHMI4M6z7gTgvWXvMW/zPGql1mLEjBF8ue5LDRuJSNRQIijmRHsETU9uCkBW/Swe6vnQEdse6vkQ3239jnP/dy49XujBFa9dwZKccp1aSUTkR5QIijnRHkHr9NYADOwwEICHez5M53qdmXHTDK5td+0RZd/97l3GLR13Qu8nInKikvwOINoUfpEfr/Z12rP6t6s5pfopANxx1h3ccdYdRduf6PME8zbP44ELH6D1qNa6+ExEfKdE4PnjWX/kcMHho042F47GNRqXuO2Xp/+yaLl5reas3KFpq0XEX0oEngd7Plju79miVgvGfTeOWyfcys2db6Zu1bpH3ABHRKQ86BiBj27seCM7Du5g1KxRdHy6I52f6UyBK/A7LBGJM0oEPjrvlPOYNmha0fNNezbpLCIRKXdKBD47s+GZPNzz4aKzld5a8pbPEYlIvFEiiAJ3nHUH2Xdm069VP+6dci+/mfAbTUUhIuVGiSCK/LvnvwEYOWskQ94bwtOzNR2FiESexdovz6ysLDd7dsW9HeSBwweo8kCVoudb79xKrSq1fIxIRCoCM5vjnMsKtU09giiTmpx6xNXNw6YO44lZT/gYkYhUdLqOIApNHjg5cPXxlw8wfMZwAH7S9CdFd0ATESlL6hFEoTYZbbiu/XU83+/5onUtR7bkUP4hH6MSkYpKiSCKZdXPYv3v1tOtQTcAPlrxkc4mEpEyp0QQ5TKrZTJpwCROqnQSfcf2JeG+BG4cdyOb9272OzQRqSCUCGJAWqU03u3/btG9lcfMG0Ovl3rxszd+xicrP/E5OhGJdTpYHCN6NOnBnj/vYdv+bdzy/i28teQt5m+Zz86DO7no1Iv8Dk9EYlhYPQIzO9vMJprZMjNbZWarzWxVpIOTH6tVpVbRLTABJq2eRPa+bB8jEpFYF+7Q0H+BR4BzgNOBLO+v+KBD3Q4A9G7WmwJXwNtL3vY5IhGJZeEmgl3OuQ+dc9nOuW2Fj4hGJiVKSUph/937+eDaD8islsnnaz/3OyQRiWHhHiOYbGYPA28DuYUrnXPfRCQqOarU5FQAzsg8g+kbpvscjYjEsnATQTfvb/A8FQ7oUbbhyLE6M/NM3vz2TTbv3UzdqnX9DkdEYlBYicA5d0GkA5Hjc2bmmQBM3zCdfq36+RuMiMSkcM8aqm5mj5jZbO/xHzOrHung5Og61etESlIKE1dO9DsUEYlR4R4sfg7YA/zMe+wG/hepoCR8KUkp9GvVj7GLx5Kbl3v0F4iIFBNuIjjVOXevc26V9/gH0DSSgUn4BnYYyPYD23nz2zf9DkVEYlC4ieCAmZ1T+MTMzgYORCYkOVY9m/akXUY7bnrvJqaunep3OCISY8JNBL8ERpnZGjNbC4wEhkYuLDkWiQmJfDbwMzLSMrh3yr1+hyMiMSasROCcm+ec6wC0B9o55zo55+Yf7XVm1svMlprZCjO7K8T27ma2y8zmeY+/HXsTBKB2ldoM7TKUKWumsGzbMr/DEZEYUmoiMLPrvb+/N7PfAzcBNwU9L+21icAooDdwGnCNmZ0WougXzrmO3uO+42qFAHBjpxtJSkji2TnP+h2KiMSQo/UI0ry/J5XwKE1XYIV3cPkQMBboewKxylHUrVqXy1tezpj5Y8gryPM7HBGJEaVeUOace9r7+4/jqLsBsD7o+QZ+uEI52JlmNh/YBNzhnFt8HO8lnp+d9jPeXvI2c7+fy+kNNC+giBxduBeUPWRm1cws2cwmmdnWwmGj0l4WYl3x+yx+A5ziHX94HHi3hPcfUngxW05OTjghx63ujbsDMHnNZH8DEZGYEe5ZQxc553YDlxL4Zd8CuLP0l7ABaBj0PJPAr/4izrndzrm93vIEINnMahevyDn3jHMuyzmXlZ6eHmbI8alO1Tq0rt2aKWumMHvTbK5/+3pWbl/pd1giEsXCTQTJ3t8+wKvOue1hvGYW0NzMmphZJaA/MD64gJnVNTPzlrt68Wh66xPUvXF3vlj3BY98/QgvL3yZjk935IX5L/gdlohEqXATwXtm9h2B2UcnmVk6cLC0Fzjn8oBbgY+BJcDrzrnFZjbUzAqvQbgKWOQdI3gM6O+cKz58JMfop61/yt5De3l10aucf8r5dK7XmYHvDmTBlgV+hyYiUcjC/d41s5OB3c65fDOrAlRzzm2OaHQhZGVludmzZ5f328YU5xyXj72cT1d9ygfXfkCnup1oNLwRl7e8nJevfNnv8ETEB2Y2xzmXFWpbqWcNmVkP59xnZnZl0LrgIrpHYhQyM8b3H8++w/uoWqkqALd0uYX/fP0fLmh8ATd1vsnnCEUkmhztfgTnA58Bl4XY5lAiiFpmVpQEAO674D6+3vA1d068kzbpbaiUWIku9bv4GKGIRIuwh4aihYaGjt/CLQs5b8x57Dy4E4Bpg6ZxZsMz/Q1KRMpFaUND4V5H8ICZ1Qh6frKZDSuj+KSctKvTjmcufYbGNRpTu0pt7v7sbmLth4CIlL1wzxrq7ZzbWfjEObeDwKmkEmOubnM1q3+7mr+d9zemrJnCpNWT/A5JRHwWbiJINLPKhU/MLBWoXEp5iXJDugyhUfVG/OWzv6hXIBLnwk0ELxG4fmCwmQ0CJgLPRy4sibTKSZW59/x7mblxJqO/Ge13OCLio3DvR/AQMAxoDbQB/umtkxg2oMMAOtXtxJD3hzB5teYmEolX4fYIIHB18EfOuT8AX5jZ0aahliiXlJDE5IGTyayWyYUvXEj/N/uTm5frd1giUs7CPWvoZuBN4GlvVQNKmClUYkv1lOrMGTKHO8+6k9cWv8ag8YN0LwOROHO0C8oK/ZrAjWZmADjnlptZRsSiknKVkZbBgz0f5GDeQR6b+RjVKlVjZJ+RJCYk+h2aiJSDcIeGcr27jAFgZkn8+N4CEuOG9xrOgA4DeGrOU9R/pD6XvnIpczbN8TssEYmwcBPB52Z2N5BqZj2BN4D3IheW+MHMGNN3DNe1u46t+7cyec1k+o7ty+7c3X6HJiIRFG4i+BOQAywEbgEmAPdEKijxj5nx0pUvkffXPD4b8Bmb9mziz5/+2e+wRCSCjpoIzCwBWOice9Y5d7Vz7ipvWUNDFZiZ0S2zG7d1u40nZz/JV+u+8jskEYmQoyYC51wBMN/MGpVDPBJlhvUYRsPqDRk8fjATV07koxUf0f/N/kUT14lI7Av3rKF6wGIzmwnsK1zpnLs8IlFJ1KhaqSrPXf4c/V7rx0UvXVS0PiMtg4y0DO446w5SklJ8jFBETlS4ieAfEY1CotqFTS8k+45sPl31Kcu3L+e1xa/x+MzHAah/Un0GdRrkc4QiciJKvR+BmaUAQ4FmBA4U/9e7F7FvdD8C/y3btowHvniArzd8TW5eLvOGzqNGSg2/wxKRUpzI/QieJ3DD+oVAb+A/ZRybxKAWtVowpt8YXuj3Ahv3bKTz0535Nudbv8MSkeN0tERwmnPueufc08BVwLnlEJPEiG6Z3Xjj6jfYun8rnZ7uxIvzX9RBZJEYdLREcLhwwe8hIYlO/Vr1Y8ZNM2hSowkD3h1AzQdr8qeJf/I7LBE5BkdLBB3MbLf32AO0L1w2M11uKgC0Tm/N3FvmMmnAJC5reRkPTXuI5+Y+53dYIhKmUs8acs5p1jEJS2pyKj2a9KBV7Vas3bmWweMHM2H5BJ6+9GlqVanld3giUopjuR+ByFHVP6k+0wZP49bTb+WtJW+R9WwW2fuy/Q5LREqhRCBlrkpyFR7v8zgfXfcRm/dupumIpkxZM8XvsESkBEoEEjEXN7uYSQMmkZGWwaBxuuGNSLRSIpCIOqvhWTx68aOs3rma9IfTuX/q/RzMO+h3WCISRIlAIu6ylpfRJr0NOw/u5J7J9zDw3YG6N7JIFFEikIhLsASm3DCF7Duyubbdtby++HXaPdmOb77/hsP5h48oq96CSPlTIpByUbtKbdLT0nnpipcY03cMu3J30eWZLlQaVgn7h3Hf5/cxYvoIavyrBte/fT07DuzwO2SRuFHqpHPRSJPOVQwrt6+k1ahWJR5AHtplKE9e+mQ5RyVScZU26Vy401CLlKlTa57K5j9sZs3ONSQlJDFlzRQ61etEq9qt+Ofn/+TJ2U9yKP8QQ7oMoW1GW9IqpfkdskiFpR6BRJ2cfTkMeHcAk1dPJjc/lwRL4N7z7+Wuc+4iOSGZaeun0bVBV5ITk/0OVSRmlNYjUCKQqLVu1zo+XfUpHyz/gLeXvE3N1JrkFeSxO3c3F596MS9d+RK1q9T2O0yRmKBEIDGtwBXw0oKXeOTrR6hWuRqLcxaz/cB2DOPylpczuNNgLmt5md9hikQ1JQKpcGZtnMV9U+/j/WXvA/DNkG/oVK+Tz1GJRK8TuUOZSFQ6vcHpjO8/ntevep3UpFQGvjuQrfu3+h2WSExSIpCYZWZc3eZq3v752yzdtpRB4wb96AI1ETm6iCYCM+tlZkvNbIWZ3VVKudPNLN/MropkPFIx9WrWiwd/8iDvLXuPS165hKdmP8Wug7v8DkskZkTsOgIzSwRGAT2BDcAsMxvvnPs2RLkHgY8jFYtUfLefcTtrd65l+IzhTFw1kY9WfMQ7P38HM/M7NJGoF8keQVdghXNulXPuEDAW6Bui3G+AtwDdvUROyKO9HmXeLfM4p9E5jFs6jkenP+p3SCIxIZKJoAGwPuj5Bm9dETNrAFwBPFVaRWY2xMxmm9nsnJycMg9UKo4OdTsweeBkLm95OX/45A88+OWDbD+w3e+wRKJaJBNBqD558XNVhwN/cs7ll1aRc+4Z51yWcy4rPT29rOKTCiopIYlXf/oq3Rp0465Jd9FtdDd25+72OyyRqBXJRLABaBj0PBPYVKxMFjDWzNYAVwFPmFm/CMYkcaJKchWmDZ7GOz9/h1U7VnHhCxdqimuREkRy0rlZQHMzawJsBPoD1wYXcM41KVw2szHA+865dyMYk8SRBEugX6t+vHbVa1z9xtU0f7w5p9c/nQJXwMY9GxnUcRBDs4bqgLLEvYglAudcnpndSuBsoETgOefcYjMb6m0v9biASFm56rSrePWnr3LL+7fwznfvFK2fvWk2y7cvZ1CnQSQlJNGqdisfoxTxj6aYkLiRvS+byasn06h6I7plduOW925h9NzRRdtH9BrBbd1u8zFCkcjRXEMiJZi9aTbvfvcuYxeNZeWOlQAMu2AYd597t4aMpELRXEMiJciqn8WwHsNY+MuF/PGsPwJwz+R76Du2Lwu2LKDAFYRVj3OObfu3RTJUkYhRj0AkSF5BHiOmj+BvU/7G/sP7qVa5Gq1qt+L3Z/weh2PL3i1USa5C31Z9ueK1Kzicf5jKSZX5ct2XAAzoMIDezXrTuV5nWtRq4XNrRH6goSGRY7Ru1zrGLhrLqFmjWLdrXVivSUtOI68gj9z8XBItkf5t+9OrWS+uaXsNiQmJEY5YpHRKBCInIDcvl+fnP09GWgYFroCqlary3tL3uKL1FXRr0I3kxGSSE5IxM9buXMsb377BG9++weLsxew7vI/LWlzG2z9/m6QE3SJc/KNEIOKDAlfAiOkj+P0nv6dn0568fvXr1Eip4XdYEqd0sFjEBwmWwO/O/B2j+oxiypoptH+yfdEd1USiiRKBSIT96vRf8emATzk59WT6ju3L2c+dzdKtS/0OS6SIEoFIOTjvlPOYesNUftvtt8z9fi5tn2zLrRNuZdm2ZX6HJqJEIFJeqqdU55GLH2H6TdPpVLcTo2aNotXIVtz+0e3E2rE6qViUCETKWfs67Zlx0wwW/nIht3S5hREzRtDw0YY8/NXD5BXk+R2exCElAhEfmBltM9ryxCVP8OjFj9KoeiP++OkfueD5CxgxfQT7D+/3O0SJI0oEIj4yM24/43amDZ7GM5c+w4rtK7j949vpNrob//3mv0oIUi6UCESixM1dbmbj7zcypu8Y1u5cy03v3UTmI5n0G9uPGRtmkJuX63eIUkHpgjKRKOSc48t1XzJq1ije/PZN8l0+SQlJ9Gzakxs63sCe3D1c0OQCmp7c1O9QJUboymKRGLZpzybeWfIO0zdO56UFLx2xrWfTngzpMoTLW15OpcRKPkUosUCJQKSC+HzN53yb8y0tarVg2vppjJ47mnW71tEmvQ0TrptAo+qN/A5RopQSgUgFlV+Qz7il47hx3I3k5uVyZesrebjnwzSo1sDv0CTKlJYINB2iSAxLTEjkytZX0i6jHY/NeIz/zfsfU9dO5Zq213BWw7PoltmNulXrAoG5j0RCUY9ApAKZt3kev3jnFyzOXozDkWiJJCcmk5acxv9d+H/c2OlGTYcdpzQ0JBJncvNymbhqImPmjQFg/e71zNw4k64NuvL0pU/TsW5HX+OT8qdEIBLnClwBY+aN4S+f/YXdubu5rt11NKzWkK4NutI2oy3JiclsP7Cd1KRUGlVvhJn5HbKUMR0jEIlzCZbAoE6D6NO8D7d9eBsvLniRg3kHQ5a9sMmFDOo0iCtbX0lKUko5Ryp+UI9AJA4559h2YBsvL3iZzXs3UyOlBvkun237t/HYzMfIK8ijWc1mXH3a1WzZu4Wl25by09Y/5bZut+n+yzFKQ0MiErZdB3cxafUkhk8fzlfrv6JSYiUyq2WyYvsKLmh8Af++6N/Uq1qPeifV8ztUOQZKBCJyXPYd2kdKUgqJCYmMmTeGX33wKw7kHQAgq34W93W/jwubXqirmmOAEoGIlImcfTk8PvNx8gvyGT13NNn7smlWsxnDLhhGv1b9qJxU2e8QpQRKBCJS5nL25fDa4td49ptnWbBlAQCZ1TK5tPmlbNq7ictbXM6gToN0BlKUUCIQkYg5nH+Y5+c/z4rtK/h01acs27aMmqk1WbtrLa1rt+buc++mT/M+LN+2nG6Z3fwON24pEYhIuSpwBbw4/0Xu/+J+lm9fXrS+TXob2ma0JcES6NuyL+c0OkfzIpUTJQIR8YVzjreWvMXc7+ey//B+FuUsYtm2ZWzZu4Xc/FxSklLo2bQnd597N5UTK9P05KacVPkkzYsUAUoEIhJV9uTuYdamWTw1+yk+XPEhew/tLdrWrGYzmtdszrXtruW6dtfpGEMZUSIQkai1df9WXl34Kut3r6dSYiW+WPcFG3ZvYNWOVZxU6SSantyUW7veyi/a/0JnJZ0AJQIRiSkFroAX5r/A9A3T+Xzt53y39TuqV67OzZ1v5s6z7yQjLcPvEGOOEoGIxCznHJNWT+LxmY8zYfkE8gry6NagGyP7jCSrfsjvNQlBiUBEKoQlOUsY/c1oXl74Mtn7srno1IuomVqTi069iIy0DJbkLKFl7ZZc0vwSHVsoRolARCqUXQd38cAXDzBhxQS27t/K5r2bj9h+QeMLuLL1lfRs2pOWtVv6FGV0USIQkQqrwBXw1bqvAGhVuxWvL36dv07+KzsO7gCgbUZbep3ai7vOuYtaVWr5GaqvfEsEZtYLGAEkAqOdc/8qtr0v8E+gAMgDbnfOfVlanUoEInI0+w/vZ8PuDby++HXe/PZNFmYvpHrl6gzuNJgu9bvQvXH3ons5xwtfEoGZJQLLgJ7ABmAWcI1z7tugMlWBfc45Z2btgdedc61Kq1eJQESO1aLsRfzp0z8xceVEDhccJjUplUtbXEqNlBq0r9OeDnU6UK1yNcyMU08+lbRKaX6HXOb8ukNZV2CFc26VF8RYoC9QlAicc3uDyqcBsTVOJSIxoW1GWz649gMO5R9iUfYihk8fzudrP2fvob08+82zR5StmVqTmzvfzNCsoTSu0difgMtZJHsEVwG9nHM3ec9/AXRzzt1arNwVwP8BGcAlzrmvQ9Q1BBgC0KhRoy5r166NSMwiEl+cc2zcs5H5m+ez59Ae8gvyeXnhy3yy8hMAujfuzm+6/obezXvH/D0X/Boauhq4uFgi6Oqc+00J5c8D/uac+0lp9WpoSEQibcPuDTwx6wmem/scW/ZtIS05je6Nu3PPefdwRuYZfod3XEpLBJGc2WkD0DDoeSawqaTCzrmpwKlmVjuCMYmIHFVmtUweuPABVv92NeP7j+eGjjcwY+MMzvzvmVzyyiV8svIT8gry/A6zzESyR5BE4GDxhcBGAgeLr3XOLQ4q0wxY6R0s7gy8B2S6UoJSj0BE/LD30F5GzhzJw9MeZvuB7aQkpXByysk0qt6IlrVbckaDM2iT0QbnHFn1s6iUWInkxGS/wy7i5+mjfYDhBE4ffc45d7+ZDQVwzj1lZn8CBgCHgQPAnTp9VESi2Z7cPXy88mOmrJnC2l1r2X94P0u3LmXjno1HlCucMK9n055c3OxiOtTpQHpauk9R64IyEZGIm795Pku3LS26hmH59uWs2rGKaeunUeAKSEpI4qyGZ3F7t9vp1awXqcmp5RqfX6ePiojEjQ51O9Chbocfrc/el83i7MV8vPJjXl/8Ole+fiVVK1Xl3Ebn0qh6IzrU6cB17a+jSnIVkhL8+UpWj0BEpJzkFeTx4fIPGb90PJ+s+oS9h/ay/cB2ANKS0+jaoCt9W/blytZX0qBaAxIsgcP5h1mxfQU5+3M4p9E5x333Ng0NiYhEqZkbZ/Lh8g9Zu2stczfPZd7meQCkV0knwRLYun8r+S4fgFtPv5XH+zx+XO+joSERkSjVtUFXujboWvR85saZfLb6M5ZsXUKiJVK3al3qVa1H9ZTqtK/TPiIxKBGIiESR4omhPETygjIREYkBSgQiInFOiUBEJM4pEYiIxDklAhGROKdEICIS55QIRETinBKBiEici7kpJswsBzjee1XWBraWYTh+qSjtgIrTFrUjulSUdkDZteUU51zIebBjLhGcCDObXdJcG7GkorQDKk5b1I7oUlHaAeXTFg0NiYjEOSUCEZE4F2+J4Bm/AygjFaUdUHHaonZEl4rSDiiHtsTVMQIREfmxeOsRiIhIMUoEIiJxLi4SgZn1MrOlZrbCzO7yO56jMbM1ZrbQzOaZ2WxvXU0zm2hmy72/JweV/7PXtqVmdrF/kYOZPWdm2Wa2KGjdMcduZl28z2CFmT1mZhYF7fi7mW309ss8M+sTA+1oaGaTzWyJmS02s99662Nqn5TSjljcJylmNtPM5ntt+Ye33r994pyr0A8gEVgJNAUqAfOB0/yO6ygxrwFqF1v3EHCXt3wX8KC3fJrXpspAE6+tiT7Gfh7QGVh0IrEDM4EzAQM+BHpHQTv+DtwRomw0t6Me0NlbPglY5sUbU/uklHbE4j4xoKq3nAzMAM7wc5/EQ4+gK7DCObfKOXcIGAv09Tmm49EXeN5bfh7oF7R+rHMu1zm3GlhBoM2+cM5NBbYXW31MsZtZPaCac+5rF/jX/kLQa8pFCe0oSTS343vn3Dfe8h5gCdCAGNsnpbSjJFHZDgAXsNd7muw9HD7uk3hIBA2A9UHPN1D6P6Bo4IBPzGyOmQ3x1tVxzn0Pgf8UQIa3Phbad6yxN/CWi6+PBrea2QJv6Kiw6x4T7TCzxkAnAr9AY3afFGsHxOA+MbNEM5sHZAMTnXO+7pN4SAShxsyi/ZzZs51znYHewK/N7LxSysZi+wqVFHu0tulJ4FSgI/A98B9vfdS3w8yqAm8BtzvndpdWNMS6qGlLiHbE5D5xzuU75zoCmQR+3bctpXjE2xIPiWAD0DDoeSawyadYwuKc2+T9zQbeITDUs8XrCuL9zfaKx0L7jjX2Dd5y8fW+cs5t8f4DFwDP8sMQXFS3w8ySCXx5vuyce9tbHXP7JFQ7YnWfFHLO7QSmAL3wcZ/EQyKYBTQ3syZmVgnoD4z3OaYSmVmamZ1UuAxcBCwiEPNAr9hAYJy3PB7ob2aVzawJ0JzAAaRockyxe93iPWZ2hncWxICg1/im8D+p5woC+wWiuB3e+/4XWOKceyRoU0ztk5LaEaP7JN3ManjLqcBPgO/wc5+U59Fyvx5AHwJnGawE/uJ3PEeJtSmBMwTmA4sL4wVqAZOA5d7fmkGv+YvXtqWU8xkQIeJ/lUAX/TCBXyyDjyd2IIvAf+qVwEi8q+B9bseLwEJggfefs14MtOMcAsMFC4B53qNPrO2TUtoRi/ukPTDXi3kR8DdvvW/7RFNMiIjEuXgYGhIRkVIoEYiIxDklAhGROKdEICIS55QIRETinBKBSBkws8YWNFOpSCxRIhARiXNKBCJlzMyamtlcMzvd71hEwqFEIFKGzKwlgflwbnTOzfI7HpFwJPkdgEgFkk5grpefOucW+x2MSLjUIxApO7sIzBt/tt+BiBwL9QhEys4hAneI+tjM9jrnXvE5HpGwKBGIlCHn3D4zuxSYaGb7nHO+T58tcjSafVREJM7pGIGISJxTIhARiXNKBCIicU6JQEQkzikRiIjEOSUCEZE4p0QgIhLn/h+fV4JqZquklAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot precision for different values of k.\n",
    "xvals = np.linspace(start=5, stop=3030, num=606)\n",
    "precs = []\n",
    "for j in xvals:\n",
    "    k = int(j)\n",
    "    k_preds = preds[0:k]\n",
    "    TP = len([edge for edge in k_preds if (edge in removed_edges)])\n",
    "    FP = len(k_preds) - TP\n",
    "    precs.append(TP/(TP+FP))\n",
    "    \n",
    "fig, axs = plt.subplots()\n",
    "axs.plot(xvals, precs, color=\"g\")\n",
    "axs.set_title(\"Precision for top k predictions.\")\n",
    "axs.set_xlabel(\"k\")\n",
    "axs.set_ylabel(\"Precision\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "id": "b81edea1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the items that took a long time to compute.\n",
    "\n",
    "# G_short.\n",
    "with open(\"test_results/G_short.pk\", \"wb\") as file:\n",
    "    pickle.dump(G_short, file)\n",
    "\n",
    "# Prediction list for G_short.\n",
    "with open(\"test_results/G_short_preds.pk\", \"wb\") as file:\n",
    "    pickle.dump(preds, file)\n",
    "    \n",
    "# Edges that were removed from G_short.\n",
    "with open(\"test_results/G_short_removed_edges.pk\", \"wb\") as file:\n",
    "    pickle.dump(removed_edges, file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}