lab08_temporal_gnn/lab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WrM5oe4yBM3B"
      },
      "source": [
        "# Lab -- Temporal Graph Neural Networks\n",
        "\n",
        "Today we will work with temporal graph networks ([TGN](https://arxiv.org/abs/2006.10637))\n",
        "\n",
        "We will predict the interactions in the common baseline wikipedia dataset.\n",
        "\n",
        "The Wikipedia is a network of links between editors and Wikipedia pages (an edge corresponds to page editing). The text content of the edited page is represented as a vector, which components are attributes of the corresponding edge. In addition, each user is associated with a label of whether he is banned or not.\n",
        "\n",
        "Let us download it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1v-Q_KYcDaF3"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "from torch import nn\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CsdKZ93pBIzA",
        "outputId": "3fc6c277-bbed-4310-99c4-dbf338ec7197"
      },
      "source": [
        "! wget https://www.dropbox.com/s/hkhahv2p595ly32/data.zip\n",
        "! unzip data.zip"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-11-25 18:13:32--  https://www.dropbox.com/s/hkhahv2p595ly32/data.zip\n",
            "Resolving www.dropbox.com (www.dropbox.com)... 162.125.1.18, 2620:100:6016:18::a27d:112\n",
            "Connecting to www.dropbox.com (www.dropbox.com)|162.125.1.18|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: /s/raw/hkhahv2p595ly32/data.zip [following]\n",
            "--2022-11-25 18:13:32--  https://www.dropbox.com/s/raw/hkhahv2p595ly32/data.zip\n",
            "Reusing existing connection to www.dropbox.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://uc90ad2553d40e07ed2e6aa484b0.dl.dropboxusercontent.com/cd/0/inline/Bxc8WIQaqkSiAyyYQm8dMOFL21A7kN2MGcF4BZo7IOyUZPoIFuSKSf4nvNJN1SqJQEvgDfCZYlfwtOcSTp9Eh8WgfZBI3UNu7QIJI1Aq1yD4A15QoSge4IL2CXDdYJg9TSBX6qsF8RKkFvIdyM8_NYr7sClNcBTR8XmDTL7i5qUBRQ/file# [following]\n",
            "--2022-11-25 18:13:33--  https://uc90ad2553d40e07ed2e6aa484b0.dl.dropboxusercontent.com/cd/0/inline/Bxc8WIQaqkSiAyyYQm8dMOFL21A7kN2MGcF4BZo7IOyUZPoIFuSKSf4nvNJN1SqJQEvgDfCZYlfwtOcSTp9Eh8WgfZBI3UNu7QIJI1Aq1yD4A15QoSge4IL2CXDdYJg9TSBX6qsF8RKkFvIdyM8_NYr7sClNcBTR8XmDTL7i5qUBRQ/file\n",
            "Resolving uc90ad2553d40e07ed2e6aa484b0.dl.dropboxusercontent.com (uc90ad2553d40e07ed2e6aa484b0.dl.dropboxusercontent.com)... 162.125.1.15, 2620:100:6016:15::a27d:10f\n",
            "Connecting to uc90ad2553d40e07ed2e6aa484b0.dl.dropboxusercontent.com (uc90ad2553d40e07ed2e6aa484b0.dl.dropboxusercontent.com)|162.125.1.15|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: /cd/0/inline2/BxfOX7BUtk1gBz9dy4-PTsprKEjDNaR3hpQz_PV1gQxIOchHfnQvVJG-pLiwqkZl3Cg-m1AhMd4LIU7B2JKHpq25qFxIceYM1C2uJVFouk9qMt3hwXFyRL_96AD1ABkUPD40XiPBmItfFcRZGchbZUmwmUnGlDT8gHB1CIQdH0CvA9kl1Ei7LAp92R-ozYK2Umkow_OYJMx8pg6ADWQMghjRz6MI9croC4xH3wLJVtgrPBZBGAG_0_cAVTUPLxVku7FbVtbqkJem6t-5pm3ZrgwzyEYMgar0_MRKugGrcZVBG8iQOkzCrb-PP-fG95R4O85VGb_XeD4Lu_wY4lvd781VE-AbmPmOaLCKE80kGYAeYfd2nP49PhI0q5NcPKkcImCnl5X_SLy6S73jr-oKlwyz50dsl9HjopgwP-PsxJWbjQ/file [following]\n",
            "--2022-11-25 18:13:34--  https://uc90ad2553d40e07ed2e6aa484b0.dl.dropboxusercontent.com/cd/0/inline2/BxfOX7BUtk1gBz9dy4-PTsprKEjDNaR3hpQz_PV1gQxIOchHfnQvVJG-pLiwqkZl3Cg-m1AhMd4LIU7B2JKHpq25qFxIceYM1C2uJVFouk9qMt3hwXFyRL_96AD1ABkUPD40XiPBmItfFcRZGchbZUmwmUnGlDT8gHB1CIQdH0CvA9kl1Ei7LAp92R-ozYK2Umkow_OYJMx8pg6ADWQMghjRz6MI9croC4xH3wLJVtgrPBZBGAG_0_cAVTUPLxVku7FbVtbqkJem6t-5pm3ZrgwzyEYMgar0_MRKugGrcZVBG8iQOkzCrb-PP-fG95R4O85VGb_XeD4Lu_wY4lvd781VE-AbmPmOaLCKE80kGYAeYfd2nP49PhI0q5NcPKkcImCnl5X_SLy6S73jr-oKlwyz50dsl9HjopgwP-PsxJWbjQ/file\n",
            "Reusing existing connection to uc90ad2553d40e07ed2e6aa484b0.dl.dropboxusercontent.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 182676257 (174M) [application/zip]\n",
            "Saving to: ‘data.zip’\n",
            "\n",
            "data.zip            100%[===================>] 174.21M  31.6MB/s    in 5.7s    \n",
            "\n",
            "2022-11-25 18:13:40 (30.8 MB/s) - ‘data.zip’ saved [182676257/182676257]\n",
            "\n",
            "Archive:  data.zip\n",
            "  inflating: ml_enron.npy            \n",
            "  inflating: ml_wikipedia.csv        \n",
            "  inflating: ml_uci.csv              \n",
            "  inflating: ml_socialevolve_2weeks.csv  \n",
            "  inflating: ml_socialevolve_1month.csv  \n",
            "  inflating: ml_socialevolve.csv     \n",
            "  inflating: ml_reddit.csv           \n",
            "  inflating: ml_enron.csv            \n",
            "  inflating: ml_wikipedia_node.npy   \n",
            "  inflating: ml_wikipedia.npy        \n",
            "  inflating: ml_uci_node.npy         \n",
            "  inflating: ml_uci.npy              \n",
            "  inflating: ml_socialevolve_node.npy  \n",
            "  inflating: ml_socialevolve_2weeks_node.npy  \n",
            "  inflating: ml_socialevolve_2weeks.npy  \n",
            "  inflating: ml_socialevolve_1month_node.npy  \n",
            "  inflating: ml_socialevolve_1month.npy  \n",
            "  inflating: ml_socialevolve.npy     \n",
            "  inflating: ml_reddit_node.npy      \n",
            "  inflating: ml_reddit.npy           \n",
            "  inflating: ml_enron_node.npy       \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KDoEf_66CjeH"
      },
      "source": [
        "Dataset consist of three parts: edgelist `ml_wikipedia.csv`, edge features `ml_wikipedia.npy` and node features `ml_wikipedia_node.npy`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "btypx_tFCR8C"
      },
      "source": [
        "graph_df = pd.read_csv(\"ml_wikipedia.csv\")\n",
        "graph_df = graph_df.sort_values(\"ts\")\n",
        "edge_features = np.load(\"ml_wikipedia.npy\")\n",
        "node_features = np.load(\"ml_wikipedia_node.npy\")"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8bcieJChWDgz"
      },
      "source": [
        "node_features = torch.from_numpy(node_features).float()\n",
        "edge_features = torch.from_numpy(edge_features).float()"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "up1FyCNkEcJX",
        "outputId": "31f8e4db-6187-4463-c3af-c70d9ac8a599"
      },
      "source": [
        "graph_df.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0  u     i     ts  label  idx\n",
              "0           0  1  8228    0.0      0    1\n",
              "1           1  2  8229   36.0      0    2\n",
              "2           2  2  8229   77.0      0    3\n",
              "3           3  3  8230  131.0      0    4\n",
              "4           4  2  8229  150.0      0    5"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2199e9d4-256f-46c9-8c4a-c07d4e5cab69\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>u</th>\n",
              "      <th>i</th>\n",
              "      <th>ts</th>\n",
              "      <th>label</th>\n",
              "      <th>idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>8228</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>8229</td>\n",
              "      <td>36.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>8229</td>\n",
              "      <td>77.0</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>8230</td>\n",
              "      <td>131.0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>8229</td>\n",
              "      <td>150.0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2199e9d4-256f-46c9-8c4a-c07d4e5cab69')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2199e9d4-256f-46c9-8c4a-c07d4e5cab69 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2199e9d4-256f-46c9-8c4a-c07d4e5cab69');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HAqTEnzEE0x3"
      },
      "source": [
        "The models are inductive, i.e. should work good enough on unseen previously nodes and edges. So let us split our dataset on train, validation and test parts in following fashion:\n",
        "\n",
        "1. Split dataset by some threshold time\n",
        "2. Select part of a nodes that appear on the test time period and remove if from the train dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S2nYcon9GXXE"
      },
      "source": [
        "import random\n",
        "\n",
        "random.seed(0)\n",
        "\n",
        "node_set = set(graph_df.u) | set(graph_df.i)\n",
        "\n",
        "# 70% of time is train, 15% is validation and 15% is test\n",
        "val_time, test_time = list(np.quantile(graph_df.ts, [0.70, 0.85]))\n",
        "\n",
        "test_node_set = set(graph_df[graph_df.ts > val_time].u) | set(graph_df[graph_df.ts > val_time].i)\n",
        "\n",
        "test_node_set = set(random.sample(test_node_set, int(0.1 * len(node_set))))"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0hx5ZGLBI_Jr"
      },
      "source": [
        "test_node_mask = graph_df.u.isin(test_node_set) | graph_df.i.isin(test_node_set)\n",
        "\n",
        "train_mask = (graph_df.ts <= val_time) & (~test_node_mask)  # in order not to use nodes from test\n",
        "val_mask = (graph_df.ts > val_time) & (graph_df.ts <= test_time)\n",
        "test_mask = (graph_df.ts > test_time)\n",
        "\n",
        "new_nodes = node_set - set(graph_df.u[train_mask]) - set(graph_df.i[train_mask])\n",
        "\n",
        "new_node_val_mask = val_mask & (graph_df.u.isin(new_nodes) | graph_df.i.isin(new_nodes))\n",
        "new_node_test_mask = test_mask & (graph_df.u.isin(new_nodes) | graph_df.i.isin(new_nodes))"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y5-0g0BuCLzH"
      },
      "source": [
        "def subgraph_to_dataset(subgraph_df):\n",
        "  return torch.utils.data.TensorDataset(\n",
        "      torch.LongTensor(subgraph_df.u.values), # sources\n",
        "      torch.LongTensor(subgraph_df.i.values), # destinations\n",
        "      torch.FloatTensor(subgraph_df.ts.values), # timestamps\n",
        "      torch.LongTensor(subgraph_df.idx.values), # edge index\n",
        "      torch.LongTensor(subgraph_df.label.values), # label\n",
        "  )"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ol03iAb2CSEG"
      },
      "source": [
        "class NeighborFinder:\n",
        "    \"\"\"Finds neighbors of a given set of nodes prior to a given time\"\"\"\n",
        "    def __init__(self, subgraph_df, uniform=True):\n",
        "        self.prepare_neighbors(subgraph_df)\n",
        "        self.uniform = uniform\n",
        "    \n",
        "    def prepare_neighbors(self, subgraph_df):\n",
        "        tmp_src = subgraph_df.sort_values(\"ts\").groupby(\"u\")[[\"i\", \"idx\", \"ts\"]].agg(list)\n",
        "        tmp_dst = subgraph_df.sort_values(\"ts\").groupby(\"i\")[[\"u\", \"idx\", \"ts\"]].agg(list)\n",
        "        self.node_to_neighbors = tmp_src[\"i\"].map(torch.LongTensor).to_dict()\n",
        "        self.node_to_neighbors.update(\n",
        "            tmp_dst[\"u\"].map(torch.LongTensor).to_dict()\n",
        "        )\n",
        "        self.node_to_edge_idxs = tmp_src[\"idx\"].map(torch.LongTensor).to_dict()\n",
        "        self.node_to_edge_idxs.update(\n",
        "            tmp_dst[\"idx\"].map(torch.LongTensor).to_dict()\n",
        "        )\n",
        "        self.node_to_edge_timestamps = tmp_src[\"ts\"].map(torch.FloatTensor).to_dict()\n",
        "        self.node_to_edge_timestamps.update(\n",
        "            tmp_dst[\"ts\"].map(torch.FloatTensor).to_dict()\n",
        "        )\n",
        "\n",
        "    def find_before(self, src_idx, cut_time):\n",
        "        i = torch.searchsorted(self.node_to_edge_timestamps[src_idx], cut_time)\n",
        "        return self.node_to_neighbors[src_idx][:i], self.node_to_edge_idxs[src_idx][:i], self.node_to_edge_timestamps[src_idx][:i]"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0ukVcJjnCgIV"
      },
      "source": [
        "train_finder = NeighborFinder(graph_df[train_mask])\n",
        "full_finder = NeighborFinder(graph_df)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kk9wbYFu8sFh"
      },
      "source": [
        "class EdgeSampler:\n",
        "    \"\"\"Sample destinations uniformly\"\"\"\n",
        "    def __init__(self, dst_list, seed=0):\n",
        "        self.dst_list = dst_list\n",
        "        self.dst_size = len(self.dst_list)\n",
        "    \n",
        "    def sample(self):\n",
        "        i = random.randint(0, self.dst_size - 1)\n",
        "        return self.dst_list[i]"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h5-Y5QxmAZYi"
      },
      "source": [
        "class TemporalGraphDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, subgraph_df, neighbor_finder, n_neighbors=10):\n",
        "        self.dataset = subgraph_to_dataset(subgraph_df.sort_values(\"ts\"))\n",
        "        self.neighbor_finder = neighbor_finder\n",
        "        self.n_neighbors = n_neighbors\n",
        "        self.random_sampler = EdgeSampler(subgraph_df.i.unique())\n",
        "    \n",
        "    def __len__(self):\n",
        "        return len(self.dataset)\n",
        "    \n",
        "    def get_neighbors(self, source, timestamp):\n",
        "        neighbors, edge_idxs, edge_times = self.neighbor_finder.find_before(source, timestamp)\n",
        "        if neighbors.shape[0] == 0:\n",
        "            neighbors, edge_idxs, edge_times = torch.zeros(self.n_neighbors), torch.zeros(self.n_neighbors), torch.zeros(self.n_neighbors)\n",
        "        else:\n",
        "            ids = np.random.randint(0, neighbors.shape[0], self.n_neighbors)\n",
        "            neighbors, edge_idxs, edge_times = neighbors[ids], edge_idxs[ids], edge_times[ids]\n",
        "            sorted_index = edge_times.argsort()\n",
        "            neighbors, edge_idxs, edge_times = neighbors[sorted_index], edge_idxs[sorted_index], edge_times[sorted_index]\n",
        "        return neighbors.long(), edge_idxs.long(), edge_times\n",
        "\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        source, destination, timestamp, edge_idx, label = self.dataset.__getitem__(idx)\n",
        "        negative = self.random_sampler.sample()\n",
        "        neighbors = {\n",
        "            \"source\": self.get_neighbors(source.item(), timestamp.item()),\n",
        "            \"positive\": self.get_neighbors(destination.item(), timestamp.item()),\n",
        "            \"negative\": self.get_neighbors(negative, timestamp.item()),\n",
        "        }\n",
        "        return source, destination, negative, timestamp, edge_idx, label, neighbors"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6gs05SlaI_Fx"
      },
      "source": [
        "train_ds = TemporalGraphDataset(graph_df[train_mask].sort_values(\"ts\"), train_finder)\n",
        "val_ds = TemporalGraphDataset(graph_df[val_mask].sort_values(\"ts\"), full_finder)\n",
        "new_node_val_ds = TemporalGraphDataset(graph_df[new_node_val_mask].sort_values(\"ts\"), full_finder)\n",
        "test_ds = TemporalGraphDataset(graph_df[test_mask].sort_values(\"ts\"), full_finder)\n",
        "new_node_test_ds = TemporalGraphDataset(graph_df[new_node_test_mask].sort_values(\"ts\"), full_finder)\n",
        "\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(train_ds, batch_size=200, shuffle=False)\n",
        "val_loader = torch.utils.data.DataLoader(val_ds, batch_size=200, shuffle=False)\n",
        "new_node_val_loader = torch.utils.data.DataLoader(new_node_val_ds, batch_size=200, shuffle=False)\n",
        "test_loader = torch.utils.data.DataLoader(test_ds, batch_size=200, shuffle=False)\n",
        "new_node_test_loader = torch.utils.data.DataLoader(new_node_test_ds, batch_size=200, shuffle=False)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FjCRR0jeOOLB"
      },
      "source": [
        "## Model definition\n",
        "\n",
        "![](https://github.com/twitter-research/tgn/blob/master/figures/tgn.png?raw=true)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5vZmQ0x1Wu_f"
      },
      "source": [
        "### Embedding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "opS9bQ__DiDU"
      },
      "source": [
        "class MergeLayer(nn.Module):\n",
        "    \"\"\"Merge node embeddings for getting final edge score\"\"\"\n",
        "    def __init__(self, dim1, dim2, dim3, dim4):\n",
        "        super(MergeLayer, self).__init__()\n",
        "        self.fc1 = torch.nn.Linear(dim1 + dim2, dim3)\n",
        "        self.fc2 = torch.nn.Linear(dim3, dim4)\n",
        "        self.act = torch.nn.ReLU()\n",
        "\n",
        "        torch.nn.init.xavier_normal_(self.fc1.weight)\n",
        "        torch.nn.init.xavier_normal_(self.fc2.weight)\n",
        "\n",
        "    def forward(self, x1, x2):\n",
        "        x = torch.cat([x1, x2], dim=1)\n",
        "        h = self.act(self.fc1(x))\n",
        "        return self.fc2(h)\n",
        "\n",
        "class TimeEncode(nn.Module):\n",
        "    \"\"\"Functional Time encoding from TGAT\"\"\"\n",
        "    def __init__(self, dimension):\n",
        "        super(TimeEncode, self).__init__()\n",
        "        self.dimension = dimension\n",
        "        self.w = torch.nn.Linear(1, dimension)\n",
        "        self.w.weight = torch.nn.Parameter((torch.from_numpy(1 / 10 ** np.linspace(0, 9, dimension)))\n",
        "                                           .float().reshape(dimension, -1))\n",
        "        self.w.bias = torch.nn.Parameter(torch.zeros(dimension).float())\n",
        "\n",
        "    def forward(self, t):\n",
        "        t = t.unsqueeze(dim=2)\n",
        "        output = torch.cos(self.w(t))\n",
        "        return output"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X-VOhVECIb2B"
      },
      "source": [
        "class TemporalAttentionLayer(nn.Module):\n",
        "    \"\"\"Aggregates features of neighbors\"\"\"\n",
        "    def __init__(self, n_node_features, n_neighbors_features, n_edge_features, time_dim, output_dimension):\n",
        "        super(TemporalAttentionLayer, self).__init__()\n",
        "\n",
        "        self.n_head = 2\n",
        "\n",
        "        self.feat_dim = n_node_features\n",
        "        self.time_dim = time_dim\n",
        "\n",
        "        self.query_dim = n_node_features + time_dim\n",
        "        self.key_dim = n_neighbors_features + time_dim + n_edge_features\n",
        "\n",
        "        self.merger = MergeLayer(self.query_dim, n_node_features, n_node_features, output_dimension)\n",
        "\n",
        "        self.multi_head_target = nn.MultiheadAttention(embed_dim=self.query_dim,\n",
        "                                                       kdim=self.key_dim,\n",
        "                                                       vdim=self.key_dim,\n",
        "                                                       num_heads=self.n_head,\n",
        "                                                       dropout=0.1)\n",
        "\n",
        "    def forward(self, src_node_features, src_time_features, neighbors_features,\n",
        "                neighbors_time_features, edge_features, neighbors_padding_mask):\n",
        "\n",
        "        src_node_features_unrolled = torch.unsqueeze(src_node_features, dim=1)\n",
        "\n",
        "        query = torch.cat([src_node_features_unrolled, src_time_features], dim=2)\n",
        "        key = torch.cat([neighbors_features, edge_features, neighbors_time_features], dim=2)\n",
        "        query = query.permute([1, 0, 2]) \n",
        "        key = key.permute([1, 0, 2]) \n",
        "\n",
        "        invalid_neighborhood_mask = neighbors_padding_mask.all(dim=1, keepdim=True)\n",
        "        neighbors_padding_mask[invalid_neighborhood_mask.squeeze(), 0] = False\n",
        "\n",
        "\n",
        "        attn_output, attn_output_weights = self.multi_head_target(query=query, key=key, value=key,\n",
        "                                                                  key_padding_mask=neighbors_padding_mask)\n",
        "\n",
        "        attn_output = attn_output.squeeze()\n",
        "        attn_output_weights = attn_output_weights.squeeze()\n",
        "\n",
        "        attn_output = attn_output.masked_fill(invalid_neighborhood_mask, 0)\n",
        "        attn_output_weights = attn_output_weights.masked_fill(invalid_neighborhood_mask, 0)\n",
        "        attn_output = self.merger(attn_output, src_node_features)\n",
        "\n",
        "        return attn_output"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CVtgNIG2W0iI"
      },
      "source": [
        "### Put all together"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "brE97aRHoBwJ"
      },
      "source": [
        "class TGN(nn.Module):\n",
        "    def __init__(self, node_features, edge_features):\n",
        "        super(TGN, self).__init__()\n",
        "        self.device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
        "        self.n_nodes, self.n_node_features = node_features.shape\n",
        "        self.memory_dim = self.n_node_features\n",
        "        self.n_edge_features = edge_features.shape[1]\n",
        "        self.attention_model = TemporalAttentionLayer(self.n_node_features, self.n_node_features, self.n_edge_features, self.n_node_features, self.n_node_features)\n",
        "        self.affinity = MergeLayer(self.n_node_features, self.n_node_features, self.n_node_features, 1)\n",
        "        self.time_encoder = TimeEncode(self.n_node_features)\n",
        "        self.node_features = node_features.to(self.device)\n",
        "        self.edge_features = edge_features.to(self.device)\n",
        "        self.message_dim = 2 * self.n_node_features + self.n_edge_features + self.time_encoder.dimension\n",
        "\n",
        "        self.__init_memory__()\n",
        "        self.memory_updater = nn.GRUCell(self.message_dim, self.n_node_features)\n",
        "    \n",
        "    def __init_memory__(self):\n",
        "        self.memory = torch.zeros((self.n_nodes, self.n_node_features), device=self.device)\n",
        "        self.last_update = torch.zeros((self.n_nodes, 1), device=self.device)\n",
        "        self.last_message = torch.zeros((self.n_nodes, self.message_dim), device=self.device)\n",
        "        self.last_message_time = torch.zeros((self.n_nodes, 1), device=self.device)\n",
        "        self.node_with_messages = set()\n",
        "    \n",
        "    def forward(self, source, destination, negative, timestamp, edge_idx, neighbors):\n",
        "        # update memory\n",
        "        nodes_with_message = list(self.node_with_messages)\n",
        "        memory = self.memory.clone()\n",
        "        if len(nodes_with_message) > 0:\n",
        "            memory[nodes_with_message] = self.memory_updater(self.last_message[nodes_with_message], self.memory[nodes_with_message])\n",
        "\n",
        "        # calculate embeddings\n",
        "        source_embedding, destination_embedding, negative_embedding = self.compute_embedding(\n",
        "            memory, source, destination, negative, timestamp, edge_idx, neighbors\n",
        "        )\n",
        "\n",
        "        # save actual edges to memory\n",
        "        self.memory[source] = memory[source].detach()\n",
        "        self.memory[destination] = memory[destination].detach()\n",
        "        self.last_update[source] = self.last_message_time[source]\n",
        "        self.last_update[destination] = self.last_message_time[destination]\n",
        "\n",
        "        # preserve messages for future updates\n",
        "        time_delta = timestamp - self.last_update[source]\n",
        "        time_delta_encoding = self.time_encoder(time_delta).reshape(-1, self.n_node_features)\n",
        "        new_source_messages = torch.cat([\n",
        "          source_embedding, destination_embedding, self.edge_features[edge_idx], time_delta_encoding\n",
        "        ], dim=1)\n",
        "        self.last_message[source] = new_source_messages.detach()\n",
        "        self.last_message_time[source] = timestamp.detach()\n",
        "\n",
        "        time_delta = timestamp - self.last_update[destination].reshape(-1, 1)\n",
        "        time_delta_encoding = self.time_encoder(time_delta).reshape(-1, self.n_node_features)\n",
        "        new_destination_messages = torch.cat([\n",
        "          destination_embedding, source_embedding, self.edge_features[edge_idx], time_delta_encoding\n",
        "        ], dim=1)\n",
        "        self.last_message[destination] = new_destination_messages.detach()\n",
        "        self.last_message_time[destination] = timestamp.detach()\n",
        "        self.node_with_messages |= set(source.tolist()) | set(destination.tolist())\n",
        "\n",
        "        # calculate affinity between embeddings\n",
        "        scores = self.affinity(\n",
        "            torch.cat([source_embedding, source_embedding], dim=0),\n",
        "            torch.cat([destination_embedding, negative_embedding], dim=0),\n",
        "        ).sigmoid()\n",
        "        return scores\n",
        "    \n",
        "    def compute_embedding(self, memory, source, destination, negative, timestamp, edge_idx, neighbors):\n",
        "        all_nodes = torch.cat([source, destination, negative])\n",
        "        all_times = torch.cat([timestamp, timestamp, timestamp])\n",
        "        node_features = memory[all_nodes] + self.node_features[all_nodes]\n",
        "        time_embedding = self.time_encoder(torch.zeros_like(all_times))\n",
        "\n",
        "        neighbor_embeddings, edge_time_embeddings, edge_features, mask = self.compute_neighbors(\n",
        "            memory, all_times, neighbors\n",
        "        )\n",
        "        \n",
        "        all_embeddings = self.attention_model(node_features,\n",
        "                                              time_embedding,\n",
        "                                              neighbor_embeddings,\n",
        "                                              edge_time_embeddings,\n",
        "                                              edge_features,\n",
        "                                              mask)\n",
        "        return (\n",
        "            all_embeddings[:source.size(0)],\n",
        "            all_embeddings[source.size(0):2 * source.size(0)],\n",
        "            all_embeddings[2 * source.size(0):]\n",
        "        )\n",
        "\n",
        "    def compute_neighbors(self, memory, timestamp, neighbors):\n",
        "        all_nodes = torch.cat([neighbors[k][0] for k in neighbors], dim=0)\n",
        "        mask = all_nodes == 0\n",
        "        all_nodes = all_nodes.reshape(-1)\n",
        "        all_edges = torch.cat([neighbors[k][1] for k in neighbors], dim=0)\n",
        "        all_times = torch.cat([neighbors[k][2] for k in neighbors], dim=0)\n",
        "        node_features = memory[all_nodes] + self.node_features[all_nodes]\n",
        "        node_features = node_features.reshape(all_edges.shape[0], all_edges.shape[1], -1)\n",
        "        node_features = torch.zeros_like(node_features)\n",
        "\n",
        "        edge_time_embeddings = self.time_encoder(timestamp - all_times)\n",
        "        edge_features = self.edge_features[all_edges]\n",
        "        \n",
        "        return node_features, edge_time_embeddings, edge_features, mask\n"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kQhPym71W7xK"
      },
      "source": [
        "Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B3ndHl5_e3JN"
      },
      "source": [
        "from sklearn.metrics import average_precision_score\n",
        "from matplotlib import pyplot as plt\n",
        "from IPython.display import clear_output\n",
        "from tqdm import tqdm\n",
        "\n",
        "criterion = nn.BCELoss()\n",
        "def train(model, opt, loader):\n",
        "    model.train()\n",
        "    pr_time = None\n",
        "    for source, destination, negative, timestamp, edge_idx, label, neighbors in tqdm(loader):\n",
        "        source = source.to(model.device)\n",
        "        destination = destination.to(model.device)\n",
        "        negative = negative.to(model.device)\n",
        "        timestamp = timestamp.to(model.device)\n",
        "        edge_idx = edge_idx.to(model.device)\n",
        "        neighbors = {k: [j.to(model.device) for j in v] for k, v in neighbors.items()}\n",
        "\n",
        "        scores = model(source, destination, negative, timestamp.reshape(-1, 1), edge_idx, neighbors)\n",
        "        label = torch.cat([torch.ones_like(source, device=model.device), torch.zeros_like(source, device=model.device)]).float()\n",
        "        loss = criterion(scores, label.view(-1, 1))\n",
        "\n",
        "        opt.zero_grad()\n",
        "        loss.backward()\n",
        "        opt.step()\n",
        "\n",
        "def evaluate(model, loader):\n",
        "    model.eval()\n",
        "    pred = []\n",
        "    gt = []\n",
        "    num_batches = 0\n",
        "    for source, destination, negative, timestamp, edge_idx, label, neighbors in tqdm(loader):\n",
        "        source = source.to(model.device)\n",
        "        destination = destination.to(model.device)\n",
        "        negative = negative.to(model.device)\n",
        "        timestamp = timestamp.to(model.device)\n",
        "        edge_idx = edge_idx.to(model.device)\n",
        "        neighbors = {k: [j.to(model.device) for j in v] for k, v in neighbors.items()}\n",
        "\n",
        "        pred.append(model(source, destination, negative, timestamp.reshape(-1, 1), edge_idx, neighbors).detach().cpu().numpy())\n",
        "        num_batches += 1\n",
        "    gt = [np.concatenate([np.ones(200), np.zeros(200)])] * (num_batches - 1)\n",
        "    gt += [np.concatenate([np.ones_like(source.detach().cpu().numpy()), np.zeros_like(source.detach().cpu().numpy())])]\n",
        "    gt = np.concatenate(gt)\n",
        "    pred = np.concatenate(pred)\n",
        "    return average_precision_score(gt, pred)\n",
        "\n",
        "\n",
        "def run(model, opt, train_loader, val_loader, new_node_val_loader, n_epoch):\n",
        "    ap_log = []\n",
        "    new_node_ap_log = []\n",
        "    for epoch in range(n_epoch):\n",
        "\n",
        "        model.__init_memory__()\n",
        "        train(model, opt, train_loader)\n",
        "\n",
        "        train_memory_backup = model.memory.clone()  # memory backup for additional use\n",
        "        train_last_update_backup = model.last_update.clone()\n",
        "        ap_log.append(evaluate(model, val_loader))\n",
        "\n",
        "        val_memory_backup = model.memory.clone()\n",
        "        val_last_update_backup = model.last_update.clone()\n",
        "        model.memory = train_memory_backup\n",
        "        model.last_update = train_last_update_backup\n",
        "        new_node_ap_log.append(evaluate(model, new_node_val_loader))\n",
        "\n",
        "        clear_output()\n",
        "        plt.plot(ap_log, label=f\"all nodes\")\n",
        "        plt.plot(new_node_ap_log, label=f\"new nodes\")\n",
        "        plt.legend()\n",
        "        plt.title(f\"Best score: {max(ap_log):.4f}, {max(new_node_ap_log):.4f} at epoch: {np.argmax(ap_log)}, last score: {ap_log[-1]:.4f}, {new_node_ap_log[-1]:.4f} at epoch {epoch}\")\n",
        "        plt.show()\n",
        "    return val_memory_backup, val_last_update_backup"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HhSmHCFWCRpK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "outputId": "fb89c0a1-cdc5-4a03-e7a7-bf9ffaa182d4"
      },
      "source": [
        "tgn = TGN(node_features, edge_features)\n",
        "tgn.to(tgn.device)\n",
        "opt = torch.optim.Adam(tgn.parameters(), 3e-4,)\n",
        "val_memory_backup, val_last_update_backup = run(tgn, opt, train_loader, val_loader, new_node_val_loader, 10)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAEICAYAAAAeO/7PAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3gVVfrA8e+bkEJISCAJNVRBOlICAeniApaV4qJYUdaCZcVeVtd1Udf6UywIC4pYQQXbKpaVjhQJEnrvTQglCQnpOb8/zgQuIeUm3Jvkwvt5njyZO/WduWfuO3PmzIwYY1BKKaVU+fGr6ACUUkqp840mX6WUUqqcafJVSimlypkmX6WUUqqcafJVSimlypkmX6WUUqqcafJVqhyIyE4RubQClttYRIyIVCnvZSvlq0Rknojc5s1llJh8nR+NdBFJFZFjIvK9iDQ42wVX1I+RJ4lIBxFZISInnP8dihm3lYjMEZFkEdkqIkNdhuX/QKa6/P2jkHnUFJFEEVlU2mkreh1EZKqIZBUY7u8M6yYi/xORo876fSEidd1dhwIx9hWRvWWZtjITkWgR+dTZ9sdE5JNyXr7ur4WP97RT7i916feqiGwRkeMislFEbi4wjRGRNJf94N1SrENjEZnrrMPG4r4TEakvIt84+9VeERntbhxivSQiR5y/l0REXIb7i8hzIrLfWc+VIhLh7noUWB+fPzgUkduccpIqIj+KSL0SJzLGFPsH7AQudbqDgSnA1yVNV5r5VsQfUOUspw8EdgEPAEHAfc7nwMKWBWwGHgT8gUuANOBCZ3hjwJQUEzAZWAAscunn1rQVvQ7AVOC5IoZdBgwHqgMhThn7sYzfS19gb0WVq2LiOqvyDiwEXgPCgQCgo5vTlbl8eDJ+D2y/SrO/uox3AbAG2O+6bYB/AS2xJzdxwDHgYpfhBmhWxvVY4pSDqsDVQBIQXcS4c4FxTnm5CDgK9HMnDuBOYBMQA9QH1gOjXYY/B8wBGgECtAWCy7A+HimfXihv84Db3By3L3AIaOOUswnA/BKnc2PGp+10wOXAZpfPQcCrwG7gIDARqOoMiwK+cwrIUecHxA/4CMgD0oFU4NFCllvotM6wBsCXQCJwBHjb6e8HPOXsVIeAD4HwAl/yX51YFzj9RwEbnB3kJ6CRmxt8ALAPEJd+u4FBhYzb1llP13F/Bp51twACFzs73q14LvmW2zpQTPItZNxOwPFiht/qfGfHge3AnU7/ak6ZynNiTQXqFTJ9cWW2L7AX+DtwGFv+b3CZNtwpV4lOOXsqv1w6w293iW090MllP3oYWA0kA5/h5o+V8z3tBPzL8B2f9r0Ute10f3W/rLv0+xH7e7iTYg5MgG+Bh1w+lyn5AhcCmUCYS7+FuCRFl/6hznKiXfpNAj5yJw5gMXCHy+e/Akud7hrO9rnAzbivAFYCKcAe4JkC34Hh1P7avZDp/YDHgW1O+fkcqFmgnNyBPQg6ADxcYF8f5wzb73QHuQwfDCQ4sW3LLw/Y5Pss8Ct2X/kZiCpi/V4Fxrt8rufEVOz2KdU1XxEJAa4Flrr0fhFbKDoAzbBHSU87wx7C/pBFA7WxP2jGGHOTs9H/bIwJNca8XMjiCp3Wqar8DrvDNnaWN92Z5hbnrx/QFFsA3y4w3z5AK2CgiAx25jvMWc5CYJrL+n4nIo8XsTnaAKuNs7Udq53+7sg/WnS1y6keel9Eolzi8HfW417sl1qYQqctQbmtg+NupwpshYhcXcx8ewPrihl+CLgSe6Z8K/C6iHQyxqRhz6L3O+Uq1Bizv5DpiyuzAHWwyaQ+MBKYJCItnGFvYRNwU2xZutmJAREZDjzj9KsOXIX9sch3DTAIaAK0x5ZVnGmTRKRnEevbDXsW8oFTBbhcRPoUs32KU+i2c4bp/lq008q6811nGmNmFTuRSFWgC2eW5wUi8oeIfCkijd2MoQ2w3Rhz3KXfKgpfBynwP7+74P5aVBxtnHkXtpx2QA7wF2fazSJyTzFxp2H3iQhsIr5LRIY4w3o7/yOcsrWkkOn/BgzBloV62AOv8QXG6Qc0xx5kPeZSHf8kdv/pgD3774o94ENEumIP+B5xYuuNPZDKdz12H6mFPaN9uJh1LLid4cxtfTo3jlp2Yo9IkoBs7NFDO2eYYDfsBS7jdwd2ON1jgW8o5OiKko8WC53WmX8ihZxhAbOBu10+t3BirsKpI6SmLsN/AP5a4AjrBG4cTQP/AKYX6PcJLkd1Lv0DsGcZjzrdA4As4CeXo9RYJ87awIz8Yc7wB4AJTvctnH7mW+y0lWgdOgGRzvDLsUeTPQpZTnvsmVMvd9bBmeZrYIzT3Zdiqp3dKLN9sT8s1VyGf+5sK39nnVu7DLsTmOd0/5QfRxHl/UaXzy8DE91cv0mcOgsMAEZg98dCj8QLTJtf7ouqkXDddrq/ulfWw4AtQOOStg3wAfYM2fUsujf2xzwCe7Cxtqjvp8C8bsI5+3Tp9zwwtYjxF2EPFoOx+99RYJM7cQC5QEuXcZs734dgk5IB3sNWf7d3vuM/uVmexwGvu1M+nXE2AP1dPtctpJy4xvoy8J7TvQ243GXYQGCn0/2f/DgKWeY84CmXz3dTxKUw4FJsLVl7Z3v8B1tTdF1x28HdM98hxpgI50u8F5gvInWwR58hwArnyD0JW9CineleAbYCP4vI9mKOSgtT1LQNgF3GmJxCpqmHPcLOt4tTySDfHpfuRsAbLrEfxRau+m7El4o9e3BVHZtUTmOMycYeuV0B/IE9S/gce6aAMSbVGBNvjMkxxhzEbuMBIhLmXLi/D3sEd4bipq0s6+AM/90Yc8QZPgv7wzfMdRki0gz7AzvGGLOwqKBF5DIRWeqcRSdhk7m7Z/sllVmAY8aeRefbhS1bUdgf44JlLL+8NMDu7EX5w6X7BPaAxR3p2B+M94wx2caY6dhy3MPN6U8qYdvp/krJZR1bu/GRMWZncQsUkVewZz/XGOdX2pn/AmNMljEmCRiDrQlp5cl1cNzgzHsP9jrkxy7rUFIcBZdVHUh11iPd6TfWGJNujFmNrc24vLAgRCTOaSSWKCLJwGjc31/Bfu9fuXzvG7AHB0WVk/z9FQovY/nDPLK/GmN+Af4JzMQeiO3EfifFNvwsVbWzMSbXGPMldsV7YrN9OtDGGBPh/IUbY0Kd8Y8bYx4yxjTFVsE9KCL982dXwrKKmnYP0LCI1nH7sV9UvobYs5iDrrN26d6DveYV4fJX1Riz2I3NsQ5o79oCEHvkU2h1qTFmtTGmjzEm0hgzEFvN9lsR886P0Q9bTVIXWC8ifwBvAF2d6h7/EqatLOtQ1HDX1pONgF+w19U+KipgEQnCFvJXgdrOQeEsl3kVW64oocw6aohINZfPDbFl6zD2iLtgGdvndO/BNsLxtNWcuV4lrecZStp2ur+etj7FlfX+wH3OPvgH9kf8cxF5LH96EfkX9hLIAGNMSgmxnbYvlLAOTQscWF9UzDrsMsZcaYyJNsbEYRNeUftrwTjWOfMubDmrXcankO6CPsVe925gjAnHtrFwd38F+71fVuB7DzbG7HMZx/UOnPz9FQovY/nDPLa/GmPGG2OaG2NqY/exKtiahGInKqmKYCenWjsL9gJ1DvbHC2wy+Byo5XyuDwx0uq/EXlMT7MY5gNPaDnvd+I5illvotNiqv1XYH5Bq2LPxHs40t2Grg5pgj1JmAB+bIqo3gKHOBspfl3BgeEnbxBk3v/XkGOxF/XspovWkM357J9YQ7LWDHTgX/rEtIltgE1UktjHOXGdYEPYaZP7fGGAZUKekaZ3hz+BUi1bUOjjD/+J8J37YarzjQF+XMrMNl4YSxWz3MOzBXx+nbFyGPSp9zhneEptcw4uZR3Flti+2fL/qbJ9e2Grqls7wj4GvnDgaARtxWkViW2zvATo7sTXDqRLlzIaLz+CUTTfWuSb2OtdIbPn/C/asL8qN77gxTrl3Y9vp/upeWY/k9H1yj/PdhzrDn3DWq04h822Dvf7o76zzOOz1/ABn+C041aJFxLXU2ZbBzvYorrVzK+c7DwRuxB48RrsZx2jsGWZ97JniOk5v7bwAW70a5CznEC5VwwXiOASMdLq7Op/zv+cQbJm8sJh1fgBbDZy/L0UDgwuUk0+cebVx5j/AGf4ctvFYNPbgYxGnyntXZ/v1x/4u1efUfj4Pl9bOFLjcVyC+YGwNh2CT+zzg3yWWSTcK7U5OtXI8ji38NxRY8L+x10hSnC/sPpeNthP747UX+IfLdIOxjTiSKORHt4RpG2KvVR1xCtSbTn8/bMOZPdhrEB8DNYramZ3+N2FvF8hviTfFZdgPwN+L2TYdgRXO9vkdl9s/sA1DfnD5/Ar2BzTVmW8zl2HXYXfuNOyP1ocUsuMWVghKmhZ7Xeb5il4HbOOYZGc7rwJGuAz7J6e3eEzFVnEVFfM92LOjJGxL3Om4tKTG3qp0xBleWGvn4spsX6e8PemUrd3ATS7T1nDKVaJTXp7m9NbOo7E/YqnYfaWjy35UZPJ1xi/yOjf2IGCNM16867jFfcec2dq5yG2H7q9ulfUifiNdv1uDbZXsWp7/7gy7xCkfadgk8TXQ3GXafwCfFLOsxtgf93RnPq7LvQFY5/L5fme7pmGTTqzLsJLiEOy106PO38ucft26PvZyTSoFWs0XEvNfsAc6x7GN797m9LI/1okzCehWyPR+2Nu+Njnz2IaT3FzKSX5r5z9waY2P3dffxP4mHXC6g12GD8WeyR/HXjbJPwifh/vJN8KZR5qz/Bdw484EcSZW5ygRScAekR4pcWSFiPTF/jDEVHQs7tLv+NwhIj9j2zxsqOhYfIHTQnsH9oy9sHYFlZZPP1VElcwYU+RTfNS5Qb/jc4cxZkBFx6DKhz7bWSmllCpnWu2slFJKlTM981VKKaXKmV7zLUFUVJRp3LhxRYehlFI+ZcWKFYeNMdElj3l+0uRbgsaNGxMfH1/RYSillE8RkV0lj3X+0mpnpZRSqpxp8lVKKaXKmSZfpZRSqpzpNd8yyM7OZu/evWRkZFR0KOes4OBgYmJiCAgIqOhQlFLK4zT5lsHevXsJCwujcePGnP6SFOUJxhiOHDnC3r17adKkSUWHo5RSHqfVzmWQkZFBZGSkJl4vEREiIyO1ZkEpdc7S5FtGmni9S7evUupcpslXKaUqk52/wvL3IGV/yeMqn6XJ9xzTuHFjDh8+DEBoaKjH579z507atm3r8fkqdd7b8xt8cBVMvRy+fxBeaw1Tr4QVH0D6sYqOTnmYNrhSSlWInNw8Vuw6xuyNh1i24ygBfkJYcBWqVw2genBAEd1VCAsOoHrVKlQPDiA4wL+iV+Ps7V8Jc/8NW36GatEw8N/QtC9s+C+s/hz+ex98/xA0HwDtroYLL4PAkIqOWp0lTb4+asiQIezZs4eMjAzGjBnDHXfc4dZ0O3fu5LLLLqNnz54sXryY+vXr880331C1alUSEhIYPXo0J06c4IILLmDKlCnUqFGDFStWMGrUKAAGDDj1utHc3Fwef/xx5s2bR2ZmJvfccw933nknBw4c4NprryUlJYWcnBwmTJhAr169vLIdlG9JOpHF/M2JzN5wiHmbDmEyUrg5YDb/FxzP+qCOfJl5GQmHa5CSnk1KRg65ecW/dS2wih/Vg20iDnOSc3UnOYcFO5+rOsk7OOCMZF4t0L/i2hccXGeT7sbvIDgCLn0Gut4BgdXs8NptoM9jcCAB1syAtTNh0/cQGAotr4B2w22S9tfb8XyRvlKwBLGxsabgs503bNhAq1atAPjXf9exfn+KR5fZul51/vnnNsWOc/ToUWrWrEl6ejpdunRh/vz5REZGnnwWdVRUFKGhoaSmpp423c6dO2nWrBnx8fF06NCBa665hquuuoobb7yR9u3b89Zbb9GnTx+efvppUlJSGDduHO3bt+ftt9+md+/ePPLII/zwww+sXbuWSZMmcejQIZ566ikyMzPp0aMHX3zxBV9++SUZGRk8+eST5ObmcuLECcLCwkq9HVy3s/JNxhi2JaYye8MhZm88xIpdx8jNM7QISePxGnPomfxfAnJSoXY7OLQOEGh9FXS7G1M/lvScPFLSc0jJyOZ4RvbJ7pSMHCdBZ3P8ZHeOM86p7ozsvGLj8xNOO5MOrxpAnfBg6oVXpW7Eqf91w6tSPbiKZxL14S0w7wVY+yUEhUH3e6DbXRAcXvx0ebmw61dY8wWs/wYykiEkEtoMtYk4piv4VZ4riSKywhgTW9FxVFZ65uuj3nzzTb766isA9uzZw5YtW4iMjHRr2iZNmtChQwcAOnfuzM6dO0lOTiYpKYk+ffoAMHLkSIYPH05SUhJJSUn07t0bgJtuuokffvgBgJ9//pnVq1czY8YMAJKTk9myZQtdunRh1KhRZGdnM2TIkJPLUueHrJw8fttxlNkbDzJn4yF2HTkBQKu61XmiawBDTswkcttM5GgOtB4CPe+HuhdB0m74bTL8/gGs+wqp14mQbncT0nowdcJLf/AGkJmTy/GMHJcEfSpZH89wErlL99G0LJZuO8LB45lnnHVXC/SnXkRV6kZUpV64TciuCbpeeFWqBhZTDX50B8x/GVZPhypVodeD0P1eCKnp3sr4+UOT3vbv8ldh62ybiFd+AsvfhfCG0HaYTcS124DeMVCpafI9SyWdoXrDvHnz+OWXX1iyZAkhISH07du3VPfEBgUFnez29/cnPT29THEYY3jrrbcYOHDgGcMWLFjA999/zy233MKDDz7IzTffXKZlKN9wJDWTuZsSmbPxIAs2HyY1M4fAKn70uCCS23o1ZWCN/dRaNQESvgX/QOh4E1x8L9RsemomEQ1hwLPQ93FYNQ2WToQvb4Ofn4Kut0HnW6FaVKniCqriT1CoP1GhQSWP7CI3z3DoeAb7kzI4kJzOgaQM9jv/DySns35/CodTM8+YLiIkgLrhTnKOCKZeRFUuCEyiw453qbXtC/CrgnS7G3rcD6Fn8ba9KkHQ8nL7l3kcNs6yiXjxW/DrOIhuBe3+Yv9qNC77cpTXaPL1QcnJydSoUYOQkBA2btzI0qVLz3qe4eHh1KhRg4ULF9KrVy8++ugj+vTpQ0REBBERESxatIiePXvyySefnJxm4MCBTJgwgUsuuYSAgAA2b95M/fr1OXz4MDExMdx+++1kZmby+++/a/I9xxhj2PjHceZsPMTsDQdZuScJY6BWWBB/vqgul7SsTY8LahKydyEseg52zIegcHu2FzcaQmsVPfPAatDlNug8CrbNgaXvwJznYP4r0P4aW0Vb27sHvf5+Ys9sw6sCNQodJzMnl4PJmTYpJ6cXSNQZ7Nq1nRuzZ9DXfw6C4YPc/kzIHUxefB3qbdlE3fDdZ1Rt14+oSnRYEP5+pThrDQqDi661f2mHYd1X9hrxnGftX0xXezbcZkjx212VK02+PmjQoEFMnDiRVq1a0aJFC7p16+aR+X7wwQcnG1w1bdqU999/H4D333+fUaNGISKnNbi67bbb2LlzJ506dcIYQ3R0NF9//TXz5s3jlVdeISAggNDQUD788EOPxKcqVkZ2Lku2H2HOhkPM2XiIfUm2xqR9TDhj+jfn0la1aVOvOmLy7DXJqePgwCoIrQN/ehY63wLB1d1foJ8fNL/U/iVugmUTIWEarPzIVr12uxuaD6yw65xBVfxpGBlCw8gCLY/TDtuzz98mY8jmeKsRbLpwNCHZNbjO5Sx6a2IqC7ckkpaVe9rkVfyEtvXDeXV4e5rVKmV1e7Uo6Hq7/UvabRtprZkBPzwCPz4OTfvYRNzyytJ9F8rjtMFVCUpqcKW8R7dzxTuUkmHPbjceYtGWw6Rn51I1wJ+ezaO4tFUt+rWoRa3qwXbk7AxY9Sn8+iYc2wGRzaDHGGh/ra0m9YQTR+H3D+214ZS9UKOJPZPucH3FJ5P0Y7bad+lEyEm3693n0dOr1gswxpCSkXNa1fb+pHSm/7aHE1m5PDekLVd3jjn72A6uh7UzbNV00m6oEgwXDrSJuNmfICD47JdRgDa4Kp4m3xJo8q04up3LnzGGtftSTjaWWr03GYD6EVW5pGUt+reqRbemkaffX5ueBPHv2aSTdgjqd7bXNFteYRsJeUNuDmz8LyydAHuWQWAYdLrJ3qpTs5xfxpGRYuNYMh4yk6HNMOj7BERfWOZZHkzJ4L5pK1m24yjXxMbwr6vaFt+Yy13GwN54m4TXfQlpifZyQOs/Q9u/2BoFD31nmnyLp8m3BJp8K45u5/JxIiuHX7ceYY6TcA+mZCICHRtE0L9Vbfq3qkWL2mFn3maTcsBej41/H7KOwwX9bcvlxr3Kt6XtvhU28a/70t6O0+Jy6Dba+3FkpcFvk+DXN+xZb8srbdKt45knwOXk5vHm7C28NXcrzWuFMv76TjSvXbZW34XKzbHX4tfMsA/0yDoOobXtwUO74VC/01ltP02+xfPJ5Csig4A3AH/gXWPMiwWGNwKmANHAUeBGY8xeZ1gusMYZdbcx5qrilqXJt+LodvasvDzD/uR0dhxOY8fhNLYnprHl0HGW7zxGVk4eoUFV6H1hFP1b1qZvi2gii2ohfHgrLH4DVk2HvBx7n2mPMfZ2oYqUcsCegcdPgRNHoHZb2zir7V88W62anWGXseg1e+bY7E/Q7+82WXnBoi2Huf+zlaRlerAauqDsdPuErTVfwOafIDfLVul3vBF6P1ymWWryLZ7PJV8R8Qc2A38C9gLLgeuMMetdxvkC+M4Y84GIXALcaoy5yRmWaoxx+6HHmnwrjm7nskk6kcW2xDQnyaay/WR3Gpk5px46ERpUhSZR1YhtXINLW9WmS+OaBFYppvHSvhWwaJw9S6oSZH+Yu99b/tW8JclOt2dzSyfYB3eEREHsrbYFdVidss83JwtWfggLXoXjB2wVbb+noGGc52IvwqGUDO6bvpKl24/yl84xjB3chpBAL7WXTU+yT91a84V9iMdfppRpNpp8i+eLybc78IwxZqDz+QkAY8wLLuOsAwYZY/aIrStLNsZUd4Zp8vURup2LlpGdy84jaexITGO7cxa743AqOw6ncexE9snxqvgJDSNDaBpVjabRoTSJqkaTqGo0ja5GdGhQyU9sMsbe7rPoddi50D6Fqcvtzu1CZ3Gfankwxsa8dAJs+gH8qtiHUMSNLt1Zam6Ove94/suQvBsadINLnrTJtxzl5hnemL2Ft+ZsoVl0KO/c4OFq6MLk5Zb5GrAm3+L54q1G9YE9Lp/3AgUPPVcBw7BV00OBMBGJNMYcAYJFJB7IAV40xnxdcAEicgdwB0DDhg09vwZKuSE3z7A/KZ3th9PYkWgTa36i3Z+cjutxc53qwTSJqsZl7eo6ibYaTaJCaVCjKlX8y3ArTm4OrP/aXs/8YzWE1YUBz9nbhYK8/IPvKSKnngh1dDssmwQrP4bVn9kE2m00tPwz+BfxM5iXa2/VmfeCnb5eJ/jz6/badgU8PcrfT3jwTxfStXFN7v9sJVe9/StjB7dheGwD7y3UWw3mlE8mX3c8DLwtIrcAC4B9QP7NdI2MMftEpCkwR0TWGGO2uU5sjJkETAJ75lt+YZ8bpk6dSnx8PG+//XZFh1LpGWM4diKbHYdTT1UVJ6ax/XAqO4+cIMulmjgsqApNo6vRpXENmkQ1cBKs/asW5KFdOTsdEj6xt8wc2wmRzeGqt+3DLTx1u1BFqNkULnvRXptN+MTeM/zFLVA9xt4T2+nmU495zMuDDd/apJu40T53esQ0aHFZpXhkY8/mUcy6rxdjpifwyIzVLN1+lGeHeLEaWnmFL35b+wDXQ70Yp99Jxpj92DNfRCQUuNoYk+QM2+f83y4i84COwGnJVylvys0zvD1nK/M2H2J7YhrJ6aeqiQP8hYY1Q2gaHUq/FrWcKmJbXRwVGui9N/CkJ9nnAy+baBsR1Y+FAc/blsOV6GH9Zy24um2E1fUO27Bo6Tvwyz9h/ktw0Qh7Rrz4LTi4BqJawPCp0GpwpdsGtaoH8/Ftcbw5ewtvztnCqr1JvHNDJy70djW08hhfTL7LgeYi0gSbdEcA17uOICJRwFFjTB7wBLblMyJSAzhhjMl0xukBvFyewXtCca8F3LZtG/fccw+JiYmEhIQwefJkmjdvTrNmzdi+fTvJyclERkYyd+5cevfuTe/evXnvvfdo3rz5yflPnTqVb7/9lhMnTrBt2zaGDh3Kyy/bzTRt2jT+/e9/Y4zhiiuu4KWXXgLsU7BeeOEFIiIiuOiii04+PzoxMZHRo0eze/duAMaNG0ePHj2YP38+Y8aMAUBEWLBgQZnefORrMrJzuX96Aj+u+4MujWtwZfu6NI0OPVlVXD+ijNXEZZWy3+V2oVRodin0fAAa9agUZ3le4+d/6tnIf6yFZRPsCwrip9hWvkMn2eciV+JqV38/4YE/XUjXJjUZMz2Bq95exNjBbRneOabiXpOo3OZzydcYkyMi9wI/YW81mmKMWSciY4F4Y8y3QF/gBREx2Grne5zJWwH/EZE8wA97zXf9GQspjR8ehz/WlDxeadRpZ6vIirFlyxamTZvG5MmTueaaa5g5cyY33ngjd9xxBxMnTqR58+YsW7aMu+++mzlz5tCiRQvWr1/Pjh076NSpEwsXLiQuLo49e/aclnjzJSQksHLlSoKCgmjRogV/+9vf8Pf357HHHmPFihXUqFGDAQMG8PXXXxMXF8c///lPVqxYQXh4OP369aNjx44AjBkzhgceeICePXuye/duBg4cyIYNG3j11VcZP348PXr0IDU1leBgzz9hp7I5lpbF7R/Gs2L3Mf5xZWv+2rMCWglnnYDdS2D7PPv3x2oQP2h7tb1dqE678o+potVpC4PHw6X/svty454+9Y7cHs2imDWmJ/dPT+DRGatZuu0Izw5p67lLEcorfPLbMcbMAmYV6Pe0S/cMYEYh0y0Gzolfl8JeC5iamsrixYsZPnz4yfEyM+2bV3r16sWCBQvYsWMHTzzxBJMnT6ZPnz506dKl0Pn379+f8HD7ftHWrVuza9cujhw5Qt++fYmOtq1cb7jhBhYsWABwWv9rr72WzZs3A/DLL7+wfv2p45uUlBRSU1Pp0aMHDz74IDfccAPDhg0jJjP2WWwAACAASURBVMYL9y5WInuOnmDk+7+x91g6b1/XiSva1y2fBeflwv4E2D7XJts9y+w9nP6B0CAOLnnK3gdb2W4XqgjVouCCfhUdRZnUCgvmo7/G8dacLbwxO78aujMt6pz7tUm+yieTb6VSwhmqtxT2WsC8vDwiIiJISEg4Y/zevXszYcIE9u/fz9ixY3nllVeYN28evXr1cmv+OTk5ZYozLy+PpUuXnnFm+/jjj3PFFVcwa9YsevTowU8//UTLli3LtIzKbu2+ZG6dupzM7Fw+/mscXZu4+f7WsjDGtszNT7Y7FtiXroM9q427E5r2hYbd7duD1DnD30+4/1LbGvq+6QkMHr+IsVe1ZXisVkNXRpWrFYE6K9WrV6dJkyZ88cUXgG1Ju2rVKgC6du3K4sWL8fPzIzg4mA4dOvCf//yH3r3dv1exa9euzJ8/n8OHD5Obm8u0adPo06cPcXFxzJ8/nyNHjpCdnX1y+QADBgzgrbfeOvk5/8Bg27ZttGvXjscee4wuXbqwceNGT2yCSmfB5kSu/c8SAv39mHnXxd5JvKmJ9qES39wD49rBW53g+4fsGW+rq+xDEh7ZBqMX2duFml2qifccdrFTDd25UQ0enbmahz5fRVpm2Q6elffome855pNPPuGuu+7iueeeIzs7mxEjRpxsANWgQYOTrx/s1asX06ZNo10792vh69aty4svvki/fv1ONrgaPHgwAM888wzdu3cnIiLiZHU4wJtvvsk999xD+/btycnJoXfv3kycOJFx48Yxd+5c/Pz8aNOmDZdddplnN0QlMGPFXh6fuZrmtcOYemsXalf30HXtrDTYtcQ5u51vW+YCBEfYe1p7PmDPbms2PbcbTaki1QoL5sNRcYyfu5Vxv2xm1d4kxt/QiZZ19DWClYXPPeGqvOkTriqOr25nYwzj527l1Z8306NZJBNv7ExY8Fk04MnNgf0rTzWS2rMM8rLtdduG3aBpP5ts615UqVvnqoqxeNthxkxPICU9m7GD23BNbINyqYbWJ1wVT898lfKgnNw8nv52HZ8u283QjvV56er2xT8vuTDGwJGtsM25brtzIWSmAAJ120P3u22ybdANAkOKn5c67118gX0oxwOfJfDYzDUs2XaE54e209bQFUy3vlIeciIrh/umreSXDYe4q+8FPDqwhftnGMcP2te75Z/dpjjPjYloaN8adEE/aNwbqkV6K3x1DosOC+KDUV1PVkOv3pfM+Os70aquVkNXFE2+ZWSM0RaEXuRrl0OOpGYy6oN41uxN4tnBbbipe+PiJ8g6Yc9o85PtIed2rKo1oEkfaPqIc91WbwFSnuHvJ9zXvzldGtfkvukrGTL+V565qg0jupRPNbQ6nSbfMggODubIkSNERkZqofUCYwxHjhzxmQdv7DqSxsgpv3EgOYOJN3ZmQJsSXluXkQyTL7FVy/5B0Ki7fXZy035Qp32le5ShOrd0vyCSWff14sHPE3jiyzUs3W6roUO1Grpc6dYug5iYGPbu3UtiYmJFh3LOCg4O9okHbyTsSeKvU5eTZwyf3t6Nzo1qFD+BMfDtfXB0h31u8IWDIKBqucSqVL7osCA+uLUr78zbymv/28yavcm8fX0nWtfTaujyosm3DAICAmjSRKsDz3ezNxzk3k9XEhUWyAe3dqVptBuviV7+rn1V36X/stdylaogfn7CvZc0J7ZxTe6btpIh7/zKM39uw3VdtRq6PGj9llJlMO233dz+YTzNaoXy5V093Eu8+xPgp79D8wFw8X3eD1IpN3RrGsmsMb2Ia1KTv3+1hjHTE0jVh3J4nSZfpUrBGMNr/9vME1+uofeF0Uy/oxvRYW685zYjxb4/NiQKhkzU67qqUokKtdXQjwxswXer9/Pntxaxfn9KRYd1TtNfAKXclJ2bx6MzVvPm7C1cExvD5Jtj3btX0hj4732QtNs+6lFvF1KVkJ+fcE+/Zky7vRsnsnIY8s6vfLJsl8/deeArNPkq5Ya0zBxu+yCeL1bs5f5Lm/PS1e0JcPe9u/HvwbqvoP8/bMtmpSqxuKa2NXS3ppHM3aiNSr1FG1wpVYJDxzMYNXU5Gw4c56Wr23Ftl4buT3xgFfz4d2j2J7h4jPeCVMqDIkODmHpLFzJycrXxlZdo8lWqGNsSUxk55TeOpGbx7s2x9GtZy/2JT17njYShep1X+RY/PyEkUFOEt/jkr4GIDBKRTSKyVUQeL2R4IxGZLSKrRWSeiMQUGF5dRPaKyNvlF7XyNSt2HeXqCYvJyM7lszu7lS7xGgP/HQPHdjnXeaO8F6hSyuf4XPIVEX9gPHAZ0Bq4TkRaFxjtVeBDY0x7YCzwQoHhzwILvB2r8l0/rv2D6ycvo0ZIIF/e1YP2MRGlm0H8FFj3JVzypF7nVUqdweeSL9AV2GqM2W6MyQKmA4MLjNMamON0z3UdLiKdgdrAz+UQq/JBHy7ZyV2frKBV3erMvOtiGkaW8s1BB1bDj0/ABf2hxwNeiVEp5dt8MfnWB/a4fN7r9HO1ChjmdA8FwkQkUkT8gP8DHvZ6lMrn5OUZXvxhI09/s47+LWsz7fZu1KwWWLqZnLzOWxOGTdLrvEqpQp2rvwwPA31EZCXQB9gH5AJ3A7OMMXuLm1hE7hCReBGJ1+c3nx+ycvJ48PMEJs7fxg1xDZl4YyeqBpbyxfTGwHf3w7EdcPV7ep1XKVUkX2zKtg9o4PI5xul3kjFmP86Zr4iEAlcbY5JEpDvQS0TuBkKBQBFJNcY8XmD6ScAkgNjYWL3D/ByXkpHN6I9WsHjbER4Z2IK7+15QttsrVkyFtTPhkn9A4x4ej1Mpde7wxeS7HGguIk2wSXcEcL3rCCISBRw1xuQBTwBTAIwxN7iMcwsQWzDxqvPLH8kZ3PL+b2w9lMpr11zEsE5lfJPSH2vgh8fggkug54OeDVIpdc7xuWpnY0wOcC/wE7AB+NwYs05ExorIVc5ofYFNIrIZ27jq+QoJVlVqmw8eZ9g7v7L3WDrv39ql7Ik38zh8PhKq1oChep1XKVUy0ed2Fi82NtbEx8dXdBjKw5ZuP8IdH8YTHODP+7d2oU298LLNyBiYeZu9rWjkf6FxT88GqpSPEpEVxpjYio6jsvLFamelzsp3q/fz4GeraBgZwtRbuxBTo5S3Ern6/QNYOwMueUoTr1LKbZp81TkpIzuXgykZ/JGcwR8pGU53JvuSTvDTuoN0bVyTSTd3JiKklLcSufpjrb3O27Qf9HzIc8Erpc55mnyVT8nLMxxJyzqZWA8ez+Cgk2D/SMk82Z2cnn3GtCGB/tSpHsz1cQ15+srWBAeU8lYiV5nH4YuREBwBwybrdV6lVKlo8lWVxomsHA6mZNqkmuIkVKfb/mVy6HgG2bmnt1PwE/sy8DrhwTSMDKFrk5rUCQ+mdvVg6lQPpnb1IGqHBxMWVMUzb2gxBr57AI5uh5u/hdDos5+nUuq8oslXlYu89d+SMecV1rV9hI1B7W2SPVkdbBPt8YycM6YLDapC7eo2scY1qUnt8PyEGkwdpzsqNJAq7r5b1xN+/xDWfAH9noQmvcpvuUqpc4YmX+V1OdlZHP3qMWpl76fLvJtYnXMZU/JGUD00jNrhwTSNrsbFF0RSOzyY2mHBp85aw4MJDapkRfTgOvjhUWjaF3rpdV6lVNlUsl82da7JzTN8POVNbsnez48X/ovYKtv56/oPGBW5FRk6EWJ86E6EzFR7P29wuHOd9yyuGSulzmvaSkR5TV6e4bEZq4jb9wHHQhozaMR9RF3zJtz8DZKTAe/9CX75F+RkVnSoJTMGvn8Qjm6Dq9+F0FK821cppQrQ5Ku8whjDP75Zy5GE72jlt5saAx471SK4aV+4azF0uAEWvQaTL7GPZ6zMVn4Mqz+DPo9Dk94VHY1Sysdp8lUeZ4xh7Hfr+WTZLp6t+RMmPAbaDT99pODqMPhtuP5zSEuESf1gwSuQe2ajqwp3cB3Mehia9IHe+jZKpdTZ0+SrPMoYw0s/buL9X3fyTPtkYlJXIxePAf+Awie4cCDcvRRaD4Y5z9mq6MRN5Rt0cTJT7ft5g6rb6ma9zquU8gBNvsqj3pi9hYnzt3Fjt4aMzJ0J1aKh003FTxRSE/7yHgyfCsd2wsResPhtyMstj5CLZgx8/xAc3qLXeZVSHqXJV3nMO/O2Mu6XLQzvHMPYLjnIttnQ7W4IqOreDNoMhXuWQbP+8POTMPVK+yCLipLwCayeDn0fh6Z9Ki4OpdQ5R5Ov8oj3Fu3g5R83MbhDPV68uj1+v75uq2q7/LV0MwqtBSM+hSET7bXWCT1h+Xv2LLQ8HdoA3z9sG1f1fqR8l62UOudp8lVn7eOlu3j2u/Vc1rYO/zf8IvyPboX130LX2+09saUlAh2ug7sXQ4Ou9hafj4ZC8l7PB1+YrDR7P29QGAzT67xKKc/T5KvOyufxe3jq67Vc2qoWb4zoaB/zuGgcVAmCuLvObubhMXDTV3Dl67DnN3inOyR86v2z4O8fhsOb4erJEFbbu8tSSp2XNPmqMvsmYR+PzVxNr+ZRvH19JwKr+EHSHnudtNNIz7xwQARiR8Fdv0LttvD1XTD9Bkg9dPbzLszKT2DVp9DHeYSkUkp5gU8mXxEZJCKbRGSriDxeyPBGIjJbRFaLyDwRiXHp/7uIJIjIOhEZXf7Rnxt+WHOABz9fRVyTmky6KfbU6/mWvG3/X/w3zy6wZhO45TsY8Dxs/QXGx8G6rzy7jEMbbOvmxr2gz2OenbdSSrnwueQrIv7AeOAyoDVwnYi0LjDaq8CHxpj2wFjgBaf/AaC7MaYDEAc8LiL1yifyc8cv6w/yt2kr6dAggvdGdqFqoJN40w7Dig+g/bUQ0cDzC/bzh4vvhdELoUZje//tjFFw4ujZzzsrzbmfN1Tv51VKeZ3PJV+gK7DVGLPdGJMFTAcGFxinNTDH6Z6bP9wYk2WMyX+QcBC+uf4VasHmRO7+5Hfa1KvO+7d2oZrrW4eWToCcDOhxv3eDiG4Bf/0fXPKUbdj1TjfY9OPZzXPWI/bhHsMmQVgdz8SplFJF8MXkUx/Y4/J5r9PP1SpgmNM9FAgTkUgAEWkgIqudebxkjNnv5XjPGUu2HeH2D+NpViuUD0fFUT3Y5alVGSnw22Ro9WeIvtD7wfhXsbcA3T7HPshj2rXwzT2QkVz6eSV8au/p7f0IXHCJ52NVSqkCfDH5uuNhoI+IrAT6APuAXABjzB6nOroZMFJEzmjOKiJ3iEi8iMQnJiaWZ9yV1opdR/nrB8tpWDOEj/7alfCQAo+LjH8PMpOh14PlG1jd9jYB93rIJtF3Lobt89yf/tBGe523UU/7MA2llCoHvph89wGuFxRjnH4nGWP2G2OGGWM6Ak86/ZIKjgOsBXoVXIAxZpIxJtYYExsd7YEWuz5u1Z4kbpmynDrVg/nk9jgiQ4NOHyE7HZaMt2eN9TqWf4BVgqD/07YqOiAYPhxsbxfKSit+uqwT9jpvQIhe51VKlStfTL7LgeYi0kREAoERwLeuI4hIlIjkr9sTwBSnf4yIVHW6awA9gUr0FP/KZ93+ZG6e8hsR1QL45PY4aoUFnznSyo/tm4l6PVT+AbqKiYU7F9pHWi6fDBN6wO6lRY//wyOQuNFe561et/ziVEqd93wu+RpjcoB7gZ+ADcDnxph1IjJWRK5yRusLbBKRzUBt4HmnfytgmYisAuYDrxpjKvmLZCvO5oPHuem936gW6M+nt3Wjbnghz2jOzYZf34SYrtCoR/kHWVBgCAx6AUZ+ByYXpgyCn/8B2Rmnj5cwzR409HrIPktaKaXKkZjyfmauj4mNjTXx8fEVHUa5256YyjX/WYqfwOd3dqdxVLXCR0yYBl+Phus+gxaDyjfIkmQet4l3xfsQ3RKGTrTV4ombYFJf233zt7bxllLKo0RkhTEmtqLjqKx87sxXed/uIye4fvIywPDp7d2KTrx5ebDodajVxr6Xt7IJCoM/j4MbZtpW0JP7w5znXa7zvqeJVylVIfSXR51mX1I617+7lIycXKbf0Y1mtUKLHnnT93B4k01iIuUXZGk1vxTuXgI/PAYLXrb9bpyp13mVUhVGk6866WBKBjdMXkpyejaf3taNlnWqFz2yMbDwNfukqdZDyi3GMqtawzasans1ZKVCs0srOiKl1HlMk68C4HBqJtdPXkri8Uw+ui2OdjElvApw+zzY/ztcOc63qm4rY/W4Uuq840O/mspbjqVlceO7y9iflMEHo7rSqWGNkida9BqE1oEO13s/QKWUOsdog6vzXHJ6NjdNWcb2w2m8OzKWrk1qljzRnuWwY4F9yUGVoJLHV0opdRpNvuex1MwcRk75jU1/HOc/N3WmR7Mo9yZc9BoER0DnW70boFJKnaM0+Z6nTmTlMOr95azdl8z46zvRr0Ut9yY8uB42zYK40fb1e0oppUpNk+95KCM7l9s/jCd+11HGjejAgDaleIXeotchoBrE3em9AJVS6hynyfc8k5mTy10fr2DxtiO8Ovwirmxfz/2Jj+6AtTMh9lYIcePasFJKqUJp8j2PZOfm8bdPVzJ3UyL/HtqOYZ1iSjeDxW/aN/90v8c7ASql1HlCk+95Iic3j/s/S+Dn9Qf511VtuK5rw9LN4PgfsPITuOg6qF6Ks2WllFJn0OR7HsjLMzw6YzXfrz7Ak5e3YuTFjUs/kyXjIS8beozxeHxKKXW+0eR7jjPG8OTXa/hy5T4eHnAht/duWvqZpB+D+CnQZihEXuD5IJVS6jyjyfcc9+7CHUz7bQ/39mvGvZc0L9tMfptsn4fc80HPBqeUUucpTb7nsLX7knn5p40MalOHhwZcWLaZZKXB0glw4SCo09azASql1HlKk+85Kj0rlzHTV1KzWiAvDGuHlPWVfys+gPSjetarlFIe5JPJV0QGicgmEdkqIo8XMryRiMwWkdUiMk9EYpz+HURkiYisc4ZdW/7Rl4/nZ61nW2Iar13TgRrVAss2k5xMWPwWNOoJDeM8G6BSSp3HfC75iog/MB64DGgNXCcirQuM9irwoTGmPTAWeMHpfwK42RjTBhgEjBORiPKJvPz8sv4gHy/dzR29m7r/vObCrP4Mju+HXg94LjillFK+l3yBrsBWY8x2Y0wWMB0YXGCc1sAcp3tu/nBjzGZjzBanez9wCIgul6jLyaHjGTw6czVt6lUv+3VegLxcWDQO6l4EF/T3XIBKKaV8MvnWB/a4fN7r9HO1ChjmdA8FwkQk0nUEEekKBALbvBRnucvLMzz8xWpOZOXwxogOBFXxL/vM1n8DR7fZa71lvV6slFKqUL6YfN3xMNBHRFYCfYB9QG7+QBGpC3wE3GqMySs4sYjcISLxIhKfmJhYXjGftamLd7JgcyJPXdGaZrXCyj4jY2DhaxDZHFr92XMBKqWUAnwz+e4DGrh8jnH6nWSM2W+MGWaM6Qg86fRLAhCR6sD3wJPGmKWFLcAYM8kYE2uMiY2O9o1a6Q0HUnjxh41c2qoWN8SV8tGRBW35HxxcAz3vt89yVkop5VG+mHyXA81FpImIBAIjgG9dRxCRKBHJX7cngClO/0DgK2xjrBnlGLNXZWTncv/0BKpXDeClq9uX/baifIteg+ox0O4azwSolFLqND6XfI0xOcC9wE/ABuBzY8w6ERkrIlc5o/UFNonIZqA28LzT/xqgN3CLiCQ4fx3Kdw0878UfNrLp4HH+75qLiAwNOruZ7VoMu5dAj/ugShlvUVJKKVWsKhUdQFkYY2YBswr0e9qlewZwxpmtMeZj4GOvB1iO5m46xNTFOxnVowl9LvRAFfnC1yAkCjredPbzUkopVSifO/NVpxxOzeSRL1bTsk4Yjw5qcfYzPLAKtv4Put0FgSFnPz+llFKF8skzX2XfVvTojNWkZGTzyW1xBAd4oGHUotchMAy63Hb281JKKVUkPfP1UR8v3cWcjYf4+2UtaVHnLG4rynd4K6z7GrreBlXPuYd+KaVUpaLJ1wdtOXic577fQJ8Loxl5cWPPzPTX16FKEHS72zPzU0opVSRNvj4mMyeX+6YnEBpUhVeGe+C2IoDkvbDqM9vIKrTW2c9PKaVUsfSar4955cdNbDiQwnsjY6kVFuyZmS5+G0weXPw3z8xPKaVUsfTM14cs3JLIu4t2cHP3RvRvVdszM007DL9/AO2vgRqNPDNPpZRSxdLk6yOOpmXx0OeraFYrlL9f3spzM142EbLToae+NlAppcqLJl8fYIzhsZmrSTqRzRsjOnjmtiKAjBT4bRK0vAKiPXCfsFJKKbdo8vUB05fv4X/rD/LooBa0qRfuuRnHT4GMZOj1oOfmqZRSqkSafCu5bYmpjP3veno2i2JUjyaem3F2BiwZD037Qv3OnpuvUkqpEmlr50osKyeP+6cnEBzgx/9dcxF+fh58qX3Cx5B2CHq957l5KqWUcosm30rstf9tZs2+ZP5zU2dqV/fQbUUAuTnw6xtQPxYa9/LcfJVSSrlFq50rqcXbDvOfBdu4rmtDBrap49mZr50JSbuh10PgiYd0KKWUKhVNvpVQ0oksHvxsFU0iq/GPKz14WxFAXh4seg2iW8GFgzw7b6WUUm7R5FvJGGP4+1drOJyayRsjOhIS6OErA5t/gMSNtoWzn379SilVEfTXt5L5YsVeZq35g4cGtKBdjAdvKwIwBhb+H0Q0gjbDPDtvpZRSbvPJ5Csig0Rkk4hsFZHHCxneSERmi8hqEZknIjEuw34UkSQR+a58oy7ZzsNpPPPtOro1rckdvZt6fgE7FsC+FdBjDPhrWzullKooPpd8RcQfGA9cBrQGrhOR1gVGexX40BjTHhgLvOAy7BXgpvKItTSyc/MY81kCAf5+vHZNB/w9eVtRvoX/B6G1ocMNnp+3Ukopt/lc8gW6AluNMduNMVnAdGBwgXFaA3Oc7rmuw40xs4Hj5RFoabw5ewur9iTx76HtqBdR1fML2LsCdsyH7vdAgAdvW1JKKVVqvph86wN7XD7vdfq5WgXkX9QcCoSJSKS7CxCRO0QkXkTiExMTzypYd/y24yjj525leOcYrmhf1zsLWfQaBIdD7CjvzF8ppZTbfDH5uuNhoI+IrAT6APuAXHcnNsZMMsbEGmNio6OjvRUjAMnp2TzwWQINaobwz6vaeGchhzbAxu+g650QFOadZSillHKbL7a62Qc0cPkc4/Q7yRizH+fMV0RCgauNMUnlFqGbjDE89fVa/kjJYMbo7oQGeenrWDQOAkIgbrR35q+UUqpUfPHMdznQXESaiEggMAL41nUEEYkSkfx1ewKYUs4xuuXrhH38d9V+7u/fnI4Na3hnIcd2wZovoPMtUM3tmnellFJe5HPJ1xiTA9wL/ARsAD43xqwTkbEicpUzWl9gk4hsBmoDz+dPLyILgS+A/iKyV0QGlusKOPYcPcE/vl5Hl8Y1uLtfM+8taNlE+wjJ7vd4bxlKKaVKxRernTHGzAJmFej3tEv3DGBGEdNW+JsEcnLzuP+zBAR4/Vov3VYEkJECv38ErYdAeEzJ4yullCoXPpl8fd34udtYsesYb4zoQEyNEO8taOXHkHUcut/tvWUopZQqNZ+rdvZ1K3Yd4805WxjasT6DOxS8Q8qD8nJtlXODblC/s/eWo5RSqtQ0+Zaj4xnZ3P/ZSuqGB/OvwV66rSjfplmQtAu63eXd5SillCo1rXYuR//8dh37jqXz+Z3dqR4c4N2FLXkHwhtCyyu9uxyllFKlpme+5eTbVfv58vd93HtJc2Ib1/TuwvavhN2LIe5OfYGCUkpVQpp8y8G+pHSe/GoNHRtGcN8lXrytKN/SCRAYCp0q3fsjlFJKocnX63LzDA9MTyAvz/DGtR2p4u/lTZ5yANbOhI432mc5K6WUqnS0TtLLJs7fxm87j/J/wy+iYaQXbyvKt/xd29I57k7vL0sppVSZ6JmvFyXsSeL1/23myvZ1GdbJi7cV5ctOh/gp0OJyqNnU+8tTSilVJpp8vSQtM4f7p6+kVlgQzw9ph4iXnmLlatV0SD+qD9VQSqlKTqudveRoWhZVA6vw4tWtCQ/x8m1FAMbYhlZ12kGjHt5fnlJKqTLT5OslDWqG8P3feuLnrec2F7RtNhzeBEOcFykopZSqtLTa2YvKLfGCfahGaG1oO6z8lqmUUqpMNPmeCw5ttGe+XW6DKkEVHY1SSqkSaPI9FyybAP5BEDuqoiNRSinlBk2+vu7EUdvK+aJroVpURUejlFLKDZp8fV38FMjJgDh9e5FSSvkKn0y+IjJIRDaJyFYRebyQ4Y1EZLaIrBaReSIS4zJspIhscf5Glm/kHpaTBb9Nhqb9oHbrio5GKaWUm3wu+YqIPzAeuAxoDVwnIgUzz6vAh8aY9sBY4AVn2prAP4E4oCvwTxGpUV6xe9z6ryH1D+h+T0VHopRSqhR8Lvlik+ZWY8x2Y0wWMB0YXGCc1sAcp3uuy/CBwP+MMUeNMceA/wGDyiFmzzMGloyHyOZwQf+KjkYppVQp+GLyrQ/scfm81+nnahWQf8PrUCBMRCLdnBYRuUNE4kUkPjEx0WOBe9TuJXAgAbrdBX6++DUqpdT561z91X4Y6CMiK4E+wD4g192JjTGTjDGxxpjY6Ohob8V4dpa+A8ERcNF1FR2JUkqpUvLF5LsPaODyOcbpd5IxZr8xZpgxpiPwpNMvyZ1pfcKxnbDxe4i9FQLL4TWFSimlPMoXk+9yoLmINBGRQGAE8K3rCCISJSL56/YEMMXp/gkYICI1nIZWA5x+vmXZJBA/6HJ7RUeilFKqDHwu+RpjcoB7sUlzA/C5MWadiIwVkauc0foCm0RkM1AbeN6Z9ijwLDaBLwfGOv18R0YK/P4htB4C4eXwjmCllFIe55NvNTLGzAJmFej3tEv3DGBGEdNO4dSZsO9Z+TFkHdd39iqllA/zuTPf81peLiybCA26Qf3OFR2NUkqpIW2HdAAACd5JREFUMtLk60s2zYKkXfb2IqWUUj5Lk68vWfIOhDeElldWdCRKKaXOgiZfX7F/JexeDHF3gr9PXqpXSinl0OTrK5ZOgMBQ6HRTRUeilFLqLGny9QUpB2DtTOh4IwSHV3Q0SimlzpImX1+w/F3b0jnuzoqORCmllAdo8q3sstMhfgq0uBxqNq3oaJRSSnmAJt/KbvVnkH5UH6qhlFLnEE2+lZkxtqFVnfbQqEdFR6OUUspDNPlWZttmQ+JG6HY3iFR0NEoppTxEk29ltnQChNaGtldXdCRKKaU8SJNvZZW4Cbb+Yl8bWCWwoqNRSinlQZp8K6ul74B/EMTeWtGRKKWU8jBNvpXRiaOwajpcdC1Ui6roaJRSSnmYJt/KKH4K5GRAnL69SCmlzkWafCubnCz7RKum/aB264qORimllBf4ZPIVkUEisklEtorI44UMbygic0VkpYisFpHLnf6BIvK+iKwRkVUi0rfcgy/J+q/h+AHofk9FR6KUUur/27vf2KrqO47j7w+ttVIDbgO3QFGrYYyqI2iHdSZLlJkILhKzLNIFE40Rp+IcIVl0MYvxsVvcg2KCDp3OjBlmFjKJLAs+Ukq4AqL8WzoGpcDCXdRhnK5WvntwjvG2zAzRc3497ef16N5fD+d8+JX2wzn3d+4tSOXKV1IT0AssAjqBHkmjTxEfAp6PiPnAUmB1Pn4nQERcDlwP/ELS2JmDCNjSC1+ZDZcsTJ3GzMwKMnaK5/QtAPoj4kBEDAHrgCWjtglgSv54KnA0f9wJbAaIiOPAO0BX4YlP10AfHNsJ3XfDpCp+a8zM7HRU8Tf8TOBww/PBfKzRw8AySYPARuC+fPx14CZJzZI6gCuBWaMPIGm5pJqkWr1e/6Lzf7q+Xmg9D+b1lHdMMzMrXRXL93T0AE9HRDuwGHg2v7y8lqysa8BjwKvAR6P/cESsiYiuiOiaPn16OYnfPgj7Xszu622ZXM4xzcwsiebUAc7AEUaerbbnY43uAG4AiIgtklqBafml5pUfbyTpVeCvxcY9TVvXgCbBguWpk5iZWcGqeOa7DZgtqUNSC9mCqg2jthkAFgJImgu0AnVJkyW15ePXA8MRsae86J/igxOw/Rm49GaYMiN1GjMzK1jlznwjYljSCmAT0ASsjYjdkh4BahGxAVgFPCFpJdniq9siIiSdD2ySdJLsbPnWRH+NkXY+B0PvZgutzMxs3Ktc+QJExEayhVSNYz9veLwHOOUDcCPiIDCn6HyfycmPsk8vmtUNM69MncbMzEpQxcvO48v+jfDOIbj6ntRJzMysJC7f1Poeh6kXwJwbUycxM7OSuHxTOroTDr0CV90FTZV8BcDMzM6AyzelvtXQci5cMTbWfZmZWTlcvqmcOAZvvgDzl0Hr1NRpzMysRC7fVLY9CSeHs0vOZmY2obh8U/jwfaithTmL4csXp05jZmYlc/mmsOv38P5bvr3IzGyCcvmWLSK7vehr34QLT3kfEDMzmwBcvmX722ao74Pue0BKncbMzBJw+ZatbzWc+1W47Pupk5iZWSIu3zLV90P/X+Bbd0JzS+o0ZmaWiMu3TH2PQ9PZ0HV76iRmZpaQy7cs/34LXl8H826Btmmp05iZWUIu37LU1sLw+9lCKzMzm9BcvmUYHsre0eria+H8uanTmJlZYi7fMuz5I7x7DK6+N3USMzMbAypZvpJukLRfUr+kB/7H1y+Q9LKkHZJ2SVqcj58l6TeS3pC0V9KDhYeNyG4vmvZ1uGRh4YczM7Oxr3LlK6kJ6AUWAZ1Aj6TOUZs9BDwfEfOBpcDqfPwHwNkRcTlwJXCXpIsKDTzQB0d3wFU/gkmVm24zMytAFdtgAdAfEQciYghYBywZtU0AU/LHU4GjDeNtkpqBc4Ah4EShaft6ofU8mNdT6GHMzKw6qli+M4HDDc8H87FGDwPLJA0CG4H78vH1wHvAMWAAeDQi3hp9AEnLJdUk1er1+pknffsg7Hsxu6+3ZfKZ78fMzMaVKpbv6egBno6IdmAx8KykSWRnzR8BM4AOYJWkUz7TLyLWRERXRHRNnz79zFNsXQOaBAuWn/k+zMxs3Kli+R4BZjU8b8/HGt0BPA8QEVuAVmAa8EPgpYj4MCKOA68AXYWk/OAEbH8GLr0Zpswo5BBmZlZNVSzfbcBsSR2SWsgWVG0Ytc0AsBBA0lyy8q3n49fl421AN7CvkJT/OQGXXAvddxeyezMzq67m1AE+q4gYlrQC2AQ0AWsjYrekR4BaRGwAVgFPSFpJtsjqtogISb3AU5J2AwKeiohdhQSd2g63PFvIrs3MrNoUEakzjGldXV1Rq9VSxzAzqxRJr0VEMS/rjQNVvOxsZmZWaS5fMzOzkrl8zczMSubyNTMzK5nL18zMrGQuXzMzs5K5fM3MzErm+3z/D0l14NDn2MU04J9fUJyq81yM5PkYyfPxifEwFxdGxOd4c/zxzeVbMEk132ie8VyM5PkYyfPxCc/F+OfLzmZmZiVz+ZqZmZXM5Vu8NakDjCGei5E8HyN5Pj7huRjn/JqvmZlZyXzma2ZmVjKXr5mZWclcvgWRdIOk/ZL6JT2QOk9KkmZJelnSHkm7Jd2fOlNqkpok7ZD0p9RZUpN0nqT1kvZJ2ivp6tSZUpK0Mv85eVPS7yS1ps5kXzyXbwEkNQG9wCKgE+iR1Jk2VVLDwKqI6AS6gXsn+HwA3A/sTR1ijPgV8FJEfAOYxwSeF0kzgR8DXRFxGdAELE2byorg8i3GAqA/Ig5ExBCwDliSOFMyEXEsIrbnj98l++U6M22qdCS1AzcCT6bOkpqkqcB3gF8DRMRQRLyTNlVyzcA5kpqBycDRxHmsAC7fYswEDjc8H2QCl00jSRcB84GtaZMk9RjwU+Bk6iBjQAdQB57KL8M/KaktdahUIuII8CgwABwD/hURf06byorg8rXSSDoX+APwk4g4kTpPCpK+BxyPiNdSZxkjmoErgMcjYj7wHjBh10hI+hLZVbIOYAbQJmlZ2lRWBJdvMY4Asxqet+djE5aks8iK97mIeCF1noSuAW6SdJDs5YjrJP02baSkBoHBiPj4Ssh6sjKeqL4L/D0i6hHxIfAC8O3EmawALt9ibANmS+qQ1EK2YGJD4kzJSBLZa3p7I+KXqfOkFBEPRkR7RFxE9u9ic0RM2DObiPgHcFjSnHxoIbAnYaTUBoBuSZPzn5uFTOAFaONZc+oA41FEDEtaAWwiW624NiJ2J46V0jXArcAbknbmYz+LiI0JM9nYcR/wXP4f1QPA7YnzJBMRWyWtB7aT3SWwA7/V5Ljkt5c0MzMrmS87m5mZlczla2ZmVjKXr5mZWclcvmZmZiVz+ZqZmZXM5WtmZlYyl6+ZmVnJ/gsKvodg5HvjkgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SRn02DmJ1pw-",
        "outputId": "3f521a77-bc27-4c15-b0da-b57fc2f3d0cf"
      },
      "source": [
        "ap = evaluate(tgn, test_loader)\n",
        "print(\"\")\n",
        "print(f\"All nodes AP: {ap}\")\n",
        "tgn.memory = val_memory_backup\n",
        "tgn.last_update = val_last_update_backup\n",
        "ap = evaluate(tgn, new_node_test_loader)\n",
        "print(\"\")\n",
        "print(f\"New nodes AP: {ap}\")"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 119/119 [00:09<00:00, 12.77it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "All nodes AP: 0.9511688553005719\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 62/62 [00:04<00:00, 12.79it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "New nodes AP: 0.9606566312891041\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sBTfG0xk-ZQJ"
      },
      "source": [],
      "execution_count": null,
      "outputs": []
    }
  ]
}