5-Transformer-NER-CONLL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7MqJXdKydP2w"
   },
   "source": [
    "# Transformer\n",
    "\n",
    "\n",
    "Dans ce notebook, nous allons remplacer le LSTM par le transformer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 256
    },
    "colab_type": "code",
    "id": "MNKbpqYslGAU",
    "outputId": "823b325e-906d-4b1a-bd3e-237a27e746ab"
   },
   "outputs": [],
   "source": [
    "#!pip install torchtext==0.6.0\n",
    "#!pip install pytorch-crf\n",
    "\n",
    "import time\n",
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "from torch import nn\n",
    "from torch.optim import Adam\n",
    "from torchtext.data import Field, NestedField, BucketIterator\n",
    "from torchtext.datasets import SequenceTaggingDataset\n",
    "from torchtext.vocab import Vocab\n",
    "from torchcrf import CRF\n",
    "from collections import Counter\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchtext import data\n",
    "from torchtext import datasets\n",
    "\n",
    "import spacy\n",
    "from torchcrf import CRF\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import time\n",
    "import math\n",
    "import random\n",
    "import string\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vj6lBUSJmp2K"
   },
   "source": [
    "# Préparation des données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2QdrcnlymJYp"
   },
   "outputs": [],
   "source": [
    "SEED = 1234\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "TEXT = data.Field(lower = False) \n",
    "TAG = data.Field(unk_token = None) \n",
    "\n",
    "my_path = \"data_ner/\"\n",
    "train_data, valid_data, test_data = data.TabularDataset.splits(\n",
    "        path= my_path ,\n",
    "        train=\"train.csv\",\n",
    "        validation=\"valid.csv\",\n",
    "        test=\"test.csv\", format='csv', skip_header=True,\n",
    "        fields=((\"text\", TEXT), (\"tag\", TAG))\n",
    "    )\n",
    "\n",
    "MIN_FREQ = 1\n",
    "\n",
    "TEXT.build_vocab(train_data, \n",
    "                 min_freq = MIN_FREQ, # les mots qui apparaissent moins que MIN_FREQ fois seront ignorés du vocabulaire\n",
    "                 vectors = \"glove.6B.300d\",\n",
    "                 unk_init = torch.Tensor.normal_)\n",
    "\n",
    "\n",
    "TAG.build_vocab(train_data)\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "#device = 'cpu'\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE,\n",
    "    device = device, sort=False)\n",
    "\n",
    "# padding index\n",
    "TEXT_PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
    "TAG_PAD_IDX = TAG.vocab.stoi[TAG.pad_token]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training examples: 14986\n",
      "Number of validation examples: 3465\n",
      "Number of testing examples: 3683\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of training examples: {len(train_data)}\")\n",
    "print(f\"Number of validation examples: {len(valid_data)}\")\n",
    "print(f\"Number of testing examples: {len(test_data)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MKP8oaESnWiP"
   },
   "source": [
    "# Construire le modèle\n",
    "\n",
    "Pour le modèle transformer, on ajoute une couche fully-connected avec une activation GELU et une autre couche fully-connected qui transforme la dimension de la sortie en nombre correcte de tags possible pour la couche CRF.\n",
    "\n",
    "Comme nous n'utilisons pas le Transformer complet Encoder-Decoder décrit par [Vaswani et al., 2017](https://arxiv.org/pdf/1706.03762.pdf). Nous n'avons besoin que des encoders, pour \"encoder\" l'information de chaque token et déterminer s'il s'agit d'une entité. \n",
    "On utilise alors les deux modules de PyTorch `TransformerEncoder` and `TransformerEncoderLayer`.\n",
    "\n",
    "De plus, la classe `PositionalEncoding` provient de la page de tutoriels de PyTorch.\n",
    "Elle est mentionnée dans l'article de [Devlin et al., 2019](https://arxiv.org/pdf/1810.04805.pdf), on somme alors les embeddings positionnels au embeddings des tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6brXA7dknKso"
   },
   "outputs": [],
   "source": [
    "class PositionalEncoding(nn.Module):\n",
    "\n",
    "    def __init__(self, d_model, dropout=0.6, max_len=700):\n",
    "        super(PositionalEncoding, self).__init__()\n",
    "        self.dropout = nn.Dropout(p=dropout)\n",
    "        pe = torch.zeros(max_len, d_model)\n",
    "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
    "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
    "        pe[:, 0::2] = torch.sin(position * div_term)\n",
    "        pe[:, 1::2] = torch.cos(position * div_term)\n",
    "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
    "        self.register_buffer('pe', pe)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.pe[:x.size(0), :]\n",
    "        return self.dropout(x)\n",
    "\n",
    "class Transformer(nn.Module):\n",
    "\n",
    "    def __init__(self,\n",
    "                 input_dim,\n",
    "                 embedding_dim,\n",
    "                 attn_heads,\n",
    "                 fc_hidden,\n",
    "                 trf_layers,\n",
    "                 output_dim,\n",
    "                 emb_dropout,\n",
    "                 trf_dropout,\n",
    "                 fc_dropout,\n",
    "                 word_pad_idx,\n",
    "                 tag_pad_idx):  \n",
    "        super().__init__()\n",
    "        \n",
    "        self.word_pad_idx = word_pad_idx\n",
    "        self.tag_pad_idx = tag_pad_idx\n",
    "\n",
    "        # LAYER 1: Word Embedding\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.embedding = nn.Embedding(\n",
    "            num_embeddings=input_dim,\n",
    "            embedding_dim= embedding_dim,\n",
    "            padding_idx=word_pad_idx\n",
    "        )\n",
    "\n",
    "        self.emb_dropout = nn.Dropout(emb_dropout)\n",
    "        \n",
    "        # LAYER 2: Transformer\n",
    "        all_emb_size = embedding_dim \n",
    "        self.position_encoder = PositionalEncoding(\n",
    "            d_model=all_emb_size\n",
    "        )\n",
    "        encoder_layers = nn.TransformerEncoderLayer(\n",
    "            d_model=all_emb_size,\n",
    "            nhead=attn_heads,\n",
    "            activation=\"relu\",\n",
    "            dropout=trf_dropout\n",
    "        )\n",
    "        self.encoder = nn.TransformerEncoder(\n",
    "            encoder_layer=encoder_layers,\n",
    "            num_layers=trf_layers\n",
    "        )\n",
    "        \n",
    "        # LAYER 3: 2-couches fully-connected avec GELU activation \n",
    "        self.fc1 = nn.Linear(\n",
    "            in_features=all_emb_size,\n",
    "            out_features=fc_hidden\n",
    "        )\n",
    "        self.fc1_gelu = nn.GELU()\n",
    "        self.fc1_norm = nn.LayerNorm(fc_hidden)\n",
    "        self.fc2_dropout = nn.Dropout(fc_dropout)\n",
    "        self.fc2 = nn.Linear(\n",
    "            in_features=fc_hidden,\n",
    "            out_features=output_dim\n",
    "        )\n",
    "\n",
    "        # LAYER 4: CRF\n",
    "        self.crf = CRF(num_tags=output_dim)\n",
    "        \n",
    "        # init poids distribution normale \n",
    "        for name, param in self.named_parameters():\n",
    "            nn.init.normal_(param.data, mean=0, std=0.1)\n",
    "\n",
    "    def forward(self, words, tags=None):\n",
    "        # words = [sentence length, batch size]\n",
    "        # chars = [batch size, sentence length, word length)\n",
    "        # tags = [sentence length, batch size]\n",
    "\n",
    "        # embedding_out = [sentence length, batch size, embedding dim]\n",
    "        embedding_out = self.emb_dropout(self.embedding(words))\n",
    "\n",
    "        # Transformer\n",
    "        key_padding_mask = torch.as_tensor(words == self.word_pad_idx).permute(1, 0)\n",
    "        \n",
    "        # pos_out = [sentence length, batch size, embedding dim + char emb dim * num filter]\n",
    "        pos_out = self.position_encoder(embedding_out)\n",
    "        # enc_out = [sentence length, batch size, embedding dim + char emb dim * num filter]\n",
    "        enc_out = self.encoder(pos_out, src_key_padding_mask=key_padding_mask)\n",
    "        # Fully-connected\n",
    "        # fc1_out = [sentence length, batch size, fc hidden]\n",
    "        fc1_out = self.fc1_norm(self.fc1_gelu(self.fc1(enc_out)))\n",
    "        # fc2_out = [sentence length, batch size, output dim]\n",
    "        fc2_out = self.fc2(self.fc2_dropout(fc1_out))\n",
    "     \n",
    "        # CRF    \n",
    "        crf_mask = words != self.word_pad_idx\n",
    "        crf_out = self.crf.decode(fc2_out, mask=crf_mask)\n",
    "        crf_loss = -self.crf(fc2_out, tags=tags, mask=crf_mask) if tags is not None else None\n",
    "        return crf_out, crf_loss\n",
    "        \n",
    "    \n",
    "    def init_crf_transitions(self, tag_names, imp_value=-100):\n",
    "        num_tags = len(tag_names)\n",
    "        for i in range(num_tags):\n",
    "            tag_name = tag_names[i]\n",
    "            # I and <pad> impossible as a start\n",
    "            if tag_name[0] == \"I\" or tag_name[0] == \"<pad>\":\n",
    "                torch.nn.init.constant_(self.crf.start_transitions[i], imp_value)\n",
    "        # init impossible transitions between positions\n",
    "        tag_is = {}\n",
    "        for tag_position in (\"B\", \"I\", \"O\"):\n",
    "            tag_is[tag_position] = [i for i, tag in enumerate(tag_names) if tag[0] == tag_position]\n",
    "        impossible_transitions_position = {\n",
    "            \"O\": \"I\"\n",
    "        }\n",
    "        for from_tag, to_tag_list in impossible_transitions_position.items():\n",
    "            to_tags = list(to_tag_list)\n",
    "            \n",
    "            for from_tag_i in tag_is[from_tag]:\n",
    "                for to_tag in to_tags:\n",
    "                    for to_tag_i in tag_is[to_tag]:\n",
    "                    \n",
    "                        torch.nn.init.constant_(\n",
    "                            self.crf.transitions[from_tag_i, to_tag_i], imp_value\n",
    "                        )\n",
    "        # init impossible B and I transitions to different entity types\n",
    "        impossible_transitions_tags = {\n",
    "            \"B\": \"I\",\n",
    "            \"I\": \"I\"\n",
    "        }\n",
    "        for from_tag, to_tag_list in impossible_transitions_tags.items():\n",
    "            to_tags = list(to_tag_list)\n",
    "            for from_tag_i in tag_is[from_tag]:\n",
    "                for to_tag in to_tags:\n",
    "                    for to_tag_i in tag_is[to_tag]:\n",
    "                        if tag_names[from_tag_i].split(\"-\")[1] != tag_names[to_tag_i].split(\"-\")[1]:\n",
    "                            torch.nn.init.constant_(\n",
    "                                self.crf.transitions[from_tag_i, to_tag_i], imp_value\n",
    "                            )\n",
    "\n",
    "        \n",
    "    def count_parameters(self):\n",
    "        return sum(p.numel() for p in self.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 623
    },
    "colab_type": "code",
    "id": "FUL7Z0eynp1k",
    "outputId": "24be05dd-9acc-48f6-f333-8474bdc82bec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Le modèle a 8,701,510 paramètres à entraîner.\n"
     ]
    }
   ],
   "source": [
    "model = Transformer(\n",
    "    input_dim = len(TEXT.vocab),\n",
    "    embedding_dim = 300,\n",
    "    attn_heads = 4,  \n",
    "    fc_hidden = 64,  \n",
    "    trf_layers = 1,\n",
    "    output_dim = len(TAG.vocab),\n",
    "    emb_dropout = 0.2,\n",
    "    trf_dropout = 0.2,\n",
    "    fc_dropout = 0.5,\n",
    "    word_pad_idx=TEXT_PAD_IDX,\n",
    "    tag_pad_idx=TAG_PAD_IDX,\n",
    ")\n",
    "def init_weights(m):\n",
    "    for name, param in m.named_parameters():\n",
    "        nn.init.normal_(param.data, mean = 0, std = 0.1)\n",
    "        \n",
    "model.apply(init_weights)\n",
    "\n",
    "pretrained_embeddings = TEXT.vocab.vectors\n",
    "model.embedding.weight.data.copy_(pretrained_embeddings)\n",
    "model.embedding.weight.data[TAG_PAD_IDX] = torch.zeros(300)\n",
    "\n",
    "# CRF transitions initialisation\n",
    "model.init_crf_transitions(\n",
    "    tag_names = TAG.vocab.itos\n",
    ")\n",
    "print(f\"Le modèle a {model.count_parameters():,} paramètres à entraîner.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oLGiZih_nyeL"
   },
   "source": [
    "# Entraînement\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimiseur(model, lr=1e-4, eps=1e-6, weight_decay_rate=0.001, second_weight_decay_rate=0.0):\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "            'weight_decay_rate': weight_decay_rate},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "            'weight_decay_rate': second_weight_decay_rate}]\n",
    "    return optim.Adam(\n",
    "        optimizer_grouped_parameters,\n",
    "        lr=lr,\n",
    "        eps=eps\n",
    "    )\n",
    "\n",
    "#optimizer = optimiseur(model, lr=1e-5, eps=1e-7, weight_decay_rate=0.001, second_weight_decay_rate=0.0)\n",
    "#scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)\n",
    "optimizer = optim.Adam(model.parameters())\n",
    "criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)\n",
    "\n",
    "model = model.to(device)\n",
    "criterion = criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score, classification_report\n",
    "\n",
    "def f1_loss( preds, y, tag_pad_idx, full_report=False):\n",
    "    index_o = TAG.vocab.stoi[\"O\"]\n",
    "    positive_labels = [i for i in range(len(TAG.vocab.itos))\n",
    "                       if i not in (tag_pad_idx, index_o)]\n",
    "\n",
    "    flatten_preds = [pred for sent_pred in preds for pred in sent_pred]\n",
    "\n",
    "    positive_preds = [pred for pred in flatten_preds\n",
    "                      if pred not in (tag_pad_idx, index_o)]\n",
    "\n",
    "    flatten_y = [tag for sent_tag in y for tag in sent_tag]\n",
    "    if full_report:\n",
    "      \n",
    "        positive_names = [TAG.vocab.itos[i]\n",
    "                              for i in range(len(TAG.vocab.itos))\n",
    "                              if i not in (tag_pad_idx, index_o)]\n",
    "        print(classification_report(\n",
    "                y_true=flatten_y,\n",
    "                y_pred=flatten_preds,\n",
    "                labels=positive_labels,\n",
    "                target_names=positive_names\n",
    "            ))\n",
    "\n",
    "    return f1_score(\n",
    "            y_true=flatten_y,\n",
    "            y_pred=flatten_preds,\n",
    "            labels=positive_labels,\n",
    "            average=\"micro\"\n",
    "        ), flatten_preds, flatten_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, tag_pad_idx):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_f1 = 0    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in iterator:\n",
    "        \n",
    "        text = batch.text\n",
    "        tags = batch.tag\n",
    "      \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        pred_tags_list, batch_loss = model(text, tags)\n",
    "        \n",
    "        # pour calculer la loss et le score f1, on flatten true tags\n",
    "        true_tags_list = [\n",
    "                [tag for tag in sent_tag if tag != TAG_PAD_IDX]\n",
    "                for sent_tag in tags.permute(1, 0).tolist()\n",
    "            ]\n",
    "        f1,_,_ = f1_loss(pred_tags_list, true_tags_list, tag_pad_idx)\n",
    "\n",
    "        batch_loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        epoch_loss += batch_loss.item()\n",
    "        epoch_f1 += f1\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_f1 / len(iterator)\n",
    "\n",
    "def evaluate(model, iterator, tag_pad_idx,full_report):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_f1 = 0\n",
    "    \n",
    "    model.eval()\n",
    "    preds = []\n",
    "    labels = []\n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "\n",
    "            text = batch.text\n",
    "            tags = batch.tag\n",
    "        \n",
    "            pred_tags_list, batch_loss = model(text, tags)\n",
    "            true_tags_list = [\n",
    "                [tag for tag in sent_tag if tag != TAG_PAD_IDX]\n",
    "                for sent_tag in tags.permute(1, 0).tolist()\n",
    "                ]\n",
    "            \n",
    "            f1, pred, lab = f1_loss(pred_tags_list, true_tags_list, tag_pad_idx, full_report)\n",
    "            preds.append(pred)\n",
    "            labels.append(lab)\n",
    "            epoch_loss += batch_loss.item()\n",
    "            epoch_f1 += f1\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_f1 / len(iterator),preds, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 0m 21s\n",
      "\tTrain Loss: 654.996 | Train F1 score: 0.24%\n",
      "\t Val. Loss: 640.762 |  Val. F1 score: 0.00%\n",
      "Epoch: 11 | Epoch Time: 0m 21s\n",
      "\tTrain Loss: 196.292 | Train F1 score: 46.19%\n",
      "\t Val. Loss: 889.743 |  Val. F1 score: 20.38%\n",
      "Epoch: 21 | Epoch Time: 0m 21s\n",
      "\tTrain Loss: 134.153 | Train F1 score: 64.02%\n",
      "\t Val. Loss: 757.669 |  Val. F1 score: 42.27%\n",
      "Epoch: 31 | Epoch Time: 0m 21s\n",
      "\tTrain Loss: 93.506 | Train F1 score: 75.90%\n",
      "\t Val. Loss: 699.157 |  Val. F1 score: 52.66%\n",
      "Epoch: 41 | Epoch Time: 0m 21s\n",
      "\tTrain Loss: 68.017 | Train F1 score: 82.73%\n",
      "\t Val. Loss: 630.050 |  Val. F1 score: 60.84%\n"
     ]
    }
   ],
   "source": [
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs\n",
    "\n",
    "N_EPOCHS = 50\n",
    "\n",
    "t_loss = []\n",
    "t_f1 = []\n",
    "v_loss = []\n",
    "v_f1 = []\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "   \n",
    "    \n",
    "    train_loss, train_f1 = train(model, train_iterator, optimizer, TAG_PAD_IDX)\n",
    "    t_loss.append(train_loss)\n",
    "    t_f1.append(train_f1) \n",
    "    \n",
    "    valid_loss, valid_f1,_,_ = evaluate(model, valid_iterator, TAG_PAD_IDX, full_report= False)\n",
    "    v_loss.append(valid_loss)\n",
    "    v_f1.append(valid_f1)\n",
    "    \n",
    "    #scheduler.step()\n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), 'tut5-model_conll.pt')\n",
    "    \n",
    "\n",
    "    if epoch%10 == 0: \n",
    "        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "        print(f'\\tTrain Loss: {train_loss:.3f} | Train F1 score: {train_f1*100:.2f}%')\n",
    "        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. F1 score: {valid_f1*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f3b315cf950>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAELCAYAAADURYGZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xW9f3//8cZ18zeg5UwDShDkOmEWpRta6WitrW2tj+0tbZ+lNZW+nH0a7St9eOs/VhrK1oHHwcRBSsuZIgKyB5JgEBC9s61z/n9cYUokoRMMs7rfrvllnCd65zr/SbjeZ33VEzTNBFCCGFZak8XQAghRM+SIBBCCIuTIBBCCIuTIBBCCIuTIBBCCIuTIBBCCIuTIBBCCIuTIBCiFTNnzmTDhg09XQwhupUEgRBCWJwEgRDt5Pf7ue+++zj//PM5//zzue+++/D7/QBUVFTwk5/8hEmTJjF58mSWLFmCYRgAPPXUU1xwwQVMmDCB2bNns3Hjxp6shhBN9J4ugBB9zRNPPMH27dt5/fXXURSFpUuX8vjjj/OLX/yCZ555hpSUlKY/8tu3b0dRFPLy8lixYgWvvPIKKSkpHD16tCkghOhpckcgRDutWrWKm266iYSEBOLj47npppt44403ANB1ndLSUgoLC7HZbEyaNAlFUdA0Db/fT25uLoFAgIEDBzJ48OAerokQYRIEQrRTSUkJ6enpTf9OT0+npKQEgBtuuIEhQ4bwwx/+kFmzZvHUU08BMGTIEH7zm9/wyCOPMH36dG699VaKi4t7pPxCfJ0EgRDtlJycTGFhYdO/i4qKSE5OBiAyMpJly5bx7rvv8uSTT/LMM880NRPNnz+fF154gffeew9FUfjjH//YI+UX4uskCIQ4jUAggM/na/qYO3cuTzzxBBUVFVRUVPDYY48xf/58AN577z0OHz6MaZpERkaiaRqqqpKXl8fGjRvx+/3Y7XYcDgeapvVwzYQIk85iIU7jxhtvPOnfP/zhDzn77LNZsGABAJdddhlLly4F4PDhw9xzzz1UVFQQHR3N1VdfzZQpU9i7dy9/+tOfyM3NxWazMWHCBO6+++4zXhchmqPIxjRCCGFt0jQkhBAWJ0EghBAWJ0EghBAWJ0EghBAWJ0EghBAWJ0EghBAWd9p5BNnZ2axZs4Zjx46xatUqRo4cCUB+fj7Lli2jqqqK2NhYsrOzycjI6NSx9qisrMcw2j/yNSEhkvLyunaf15dJna3BanW2Wn2hc3VWVYW4uIhmj502CGbNmsX3vvc9rrnmmpMeX758OUuWLGHhwoW8/vrr3HXXXfzzn//s1LH2MAyzQ0Fw4lyrkTpbg9XqbLX6QvfU+bRNQ5MmTSItLe2kx8rLy9m9ezfz5s0DYN68eezevZuKiooOHxNCCNEzOrTERFFRESkpKU1rpWiaRnJyMkVFRZim2aFj8fHxXVQlIYQQ7dFn1xpKSIjs8LlJSVFdWJK+QepsDb25zoZhUFBQQH19PV2xsE3jyt+Wcro6KwpEREQwaNAgVLXtY4E6FARpaWkUFxcTCoXQNI1QKERJSQlpaWmYptmhY+1VXl7XobaypKQoSktr231eXyZ1tobeXufa2iqCwRCJiQNQlM4PWNR1lWDQWru8na7OpmlQVVXGoUPHiIqKPemYqiotvoHu0HcjISGBrKwscnJyAMjJySErK4v4+PgOHxNC9G8eTx1RUbFdEgKieYqiEhUVh8fTvpFFp1199N5772Xt2rWUlZURFxdHbGwsb775Jrm5uSxbtoyamhqio6PJzs5m6NChAB0+1h5yR9B2Umdr6O11Pn78MCkpg1EUpUuuJ3cEzQu3vBwhNXXISY+3dkfQZ5ehtmoQGFXHaci5H+clN6IPGN2mc/p6nTtC6tz7HD9++JQ/Tp0hQdCy5v6vWwuCPttZbEWmaeLd8BxmQxWBvR+2OQiEECf78Y+/TyAQIBgMUFBwhMzMYQCMHDmK3/xmeZuu8dprr+Dz+Vi8+JrTP/krrrxyPg888BBDhw5vd7m7iwRBB5ihAP7tq1F0J0p0ImpUMmpUIord1a2vGzz8OaGjO1FcMQQPb8UM+lF0e7e+phD90d/+9iwARUWF/OhH1/GPfzx/ynOCwSC63vKfyEWLruy28p1pEgQdENj/Mf5PXz3lccUZhRKVhGPiQvTB47r0Nc2gD9+G51HjBuKYchWet/9MsGAHtsyJXfo6QljZlVfOZ968hXz22RbS0wdw441L+f3v76S+vh6/38/06TNYuvQWAJ5++q94PB5uvvkXrF69infeeZuoqGjy8nKJiork3nsfICEhsdXXO3q0gAcf/ANVVZVomsaNN97E1KnT8Xq93Hvvcg4dykPTdAYPHsI999zP4cOHuPvu5Xi9XgwjxOWXz2fJkus6XW8JgnYyTZPArndR4wfhnncHRm0ZRm0JRk0ZZm0JwcK9eP7zGO6Fv0NLGNRlr+vf9iZmXTnO+b9GSxmO4ogkmLdFgkD0SR/vKGL9F0Wduoai0Ox8hPPHpjHjnPYPST+hrKyMRx75KwA+n4/s7Idwu90Eg0F++cub2bRpA1OnTj/lvD17dvPssy+QkpJKdva9vPLKi/zkJze1+lr//d+/ZeHCK5g3bxH5+XncfPOPee65V/jii23U1tby3HMvA1BTUwPAypUvM23aDH7wgx+d9HhnSRC0U+j4foyKAhwXXo/ijERzRqIlZTQdNxqqaPi/3+NZ+zDuK5ajOjs/wceoKcG/fTX68KnoaaMA0DMnEji4SZqHhOhil102t+lrwzB4/PGH2bHjC8CkvLycAwf2NxsEY8eOIyUlFYAxY85my5bNrb5OQ0M9Bw/uZ86cBQBkZg5l+PBR7Nq1g+HDR3DkyCH+9KdsJkyYyPTp5wMwfvy5PPLIQwQCAc49dxLnnjupS+osQdBOgV3vgt2NbfjUZo+r7lhc3/w5Dav+gPc/j+Oa8ysUteX/ZtNXj9FQhRY3oMXneDesAFXHMWVx02P60PMI7P1AmodEnzTjnM69a4fuGzXkdn/Z1/fiiyuora3hqaf+gcPhIDv7Pvx+X7Pn2e1fviFT1fCE2da0NGBTURQGDBjIihUv8+mnW9i06WOeeuoxnn3238ycOYvRo8/mk0828dxz/+DNN9/grrvu6UAtTyYzO9rBqK8kmP8ZtrMuRNEdLT5PSx6K84LrCRXuwbfx3y0+L3hoK/Uv/YaGl+/E8/7fMBqqT33O4W2EjmzHce5C1Ii4L18jPaupeUgI0T1qa2tJSEjE4XBQWlrC+vUfdNm1IyIiGT58JG+9FZ5ge/jwIXJz9zN69NmUlBSjqhoXXngxP//5r6iqqqS2toaCgiPExycwZ858rr/+x+zevatLyiJ3BO0Q2PM+mAb20TNP+1zbyBmEyo8Q2LEGNWEQ9rMuajpm+urxbnie4IGPUeMHoQ+bTGD3OoL5n+OYdAW2MbNQVA0z6Me7YQVqbBq2sy896fqKqoWbh3I3S/OQEN3kO9/5Lr/73R1cf/0SkpNTmDjxvC69/vLl9/Lgg3/gpZeeR9M0fvvbu4mLi2Pjxo958slHATCMENde+wMSE5N47rlnePvtt7DZdBRF4ZZbftUl5ZAJZW1khoLUP/9L1KRM3Jfd2rZzjBCet/5MqGgv7sZO3uCRL/B++HdMTw32CfOwT1iAoukYVUV4N6wgdHRneGTQjGsJHd+H/9NXcc35L/SBY065fvDoTjyr/4jz0p+12jzU2ycadQepc+8jE8o6TyaU9bBg/qfhP95jZrX5HEXVcM36/6h/7W48ax9BG3Q2wf0fo8YNwDX7Fyd1Mquxabgu/xXBw5/j2/A8npz7QVHRMyc1GwJwcvOQ9BMIITpKgqCN/Lv+gxKdgjbw7HadpzgjcX3zFhpev4fggQ3Yx8/DPnEhimY79bmKgi1jIvrAs/FvW03wyDYc065u+dqqhp55LoHcT6R5SAjRYRIEbRAqO4RRfBDHtKs7tHKiFj8A94I7w1+3YW6BojtwTLoCx6QrTvtcfehkAns/lNFDQogOk1FDbRDY9S7odmwjz+/wNbSEQV06wazpujJ6SAjRSRIEp2F66wgc3IRtxHQUR0RPF+cUJ5qHgke2YQb9PV0cIUQfJEFwGoF9H0IogK0dncRnmj50MgS8BI/u6OmiCCH6IAmCVpiGgX/3OrS0UWjxXd+s01W09LPCzUO57WseMoN+TL+nm0olhOgrJAhaESrYjllbhm3MN3q6KK1SVL1NzUOmaRAqO4xv22oa3nyQumeXUvevWwgWfHEGSytEz/vlL3/Ga6+tPOkx0zT5zncWsG3b5y2ed999v2flyheB8H4EL764otnnrV69it/+9vZmj1155Xzy8g52sOTdQ0YNtcK/5wMUVwx6xoSeLsppnRg95P/sNZToZDCCEAphGkEqnRqeo3mECvdgesMTjtS4gdhGzyJUuBfPmodxzloqo46EZcydu4AXX1zBokXfbnps69bP0DSN8ePPbdM1ZD8CCzDqKwkVbMc+bm6ri8b1Flr6WSjuWPzbV59yzA8orhi0QeegDxiDNnAMqjsWCC930fDWn/H+5zG45MfYhk87wyUXVhTY/3G4/60TFEVpduE226gLsY2c0eq5F154MX/+8/3k5+eRmRneM/3NN99gzpz55OYe5E9/uh+v14Pf72fBgiu46qolp1zjq/sRBAIBHnroAbZu/YykpGQGD85oUx1kP4JeLrB/PZgmtlEX9HRR2kRRdSK+cx+mrx5UHTQdRdVA00lKiaO0rL7ZTcMVRwTuObfhWfMw3nVPYQb9J62LJER/ZLPZuPTSy3jrrVUsXXoLDQ31fPTRB/z0pzcTERHBX/7yOHa7nYaGBm688ftMnjyNjIzMFq/3+usrKSoq5F//eolgMMhNN/2YtLTTr64q+xH0YqZpENj7IVp6FmpMSk8Xp80UR0SzQ1wVVWs2BJqO2124Lr8Vz9pH8H34DAT92L+2yJ0QXck2csZp37WfTmfXGpo7dyG33fYzbrzxJt599x3Gjh1HUlIyFRXlPPro/Rw8uB9FUSkrK+Xgwf2tBsHnn3/G5ZfPQ9d1dF1n9uzL+eKLba2+fm/aj0A6i5sRKtyLWVuK7awLe7ooZ4yiO3DNvgU941x8G1bg25bTra8XKj5IYN9HLa7JLkR3GzFiJAkJiWzevJHVq99g7tzwH+S//vUx4uMT+PvfV/Dssy+QlTUGv7/1OTod+Tluy34E5503hU8/3cwPfnA1Pp+PmTNn8cQTTzNgwECee+4f3HPPXe1+3eZIEDQjsPdDsLvRM6zVeapoNpzfWIo+bCr+T14hcKjl0RMdYZomwcK9NLz5IA2v34v3g6cxyg936WsI0R5z5y7g739/ioKCI5x/frhJtK6uluTkFHRdJy/vINu3t/7OHmDSpPN4++3VBINBfD4v77zz9mnPkf0IejHTW0cw/1NsWRdZchE3RdVxXvIjGsqP4Nv4Avqgc5pdIK89TNMkdHQn/q2rCB3fj+KKxj7pW/g/f4PAgY1oiRldU3gh2unSSy/nscf+h4ULv4XNFv45//73b+Cee+5i7dq3GDBgAOPHn37U4IIF3+LgwYNcd91VJCenMH78RIqKjp32PNmPoJO6az8C/8538G1Ygfvbd6MlDO5MEXuNjqxTf2KvA/vkK3GMn9fh1w4e3Ylvy0qM0nyUiHjs4+Y07vBmx7P2fwgV5xJxzUMoatfenPb2tfm7Q2+vs+xH0HmyH8EZYJomgT0foCZl9psQ6Ch94NnoQybg/3wVthEzTtomsy2MunJ8G54neOgzlKgkHBdej23EDBTtyx85ffg0goc+J1S4G72dy3sLIbqO9BF8hVGaj1F5FNso63QSt8Yx7WowQvg+eaXN55ihIL5tOdS/9GuCBTuwn3clEVf9AftZF50UAgD64HFgdxE4sLGriy6EaAe5I/iKwN4PwstND5/a00XpFdToZOxjZ+Pf9iahMTPRkoe1+vzg0V34Pv4XRvVx9IyJOKZdjRqV2OLzFd2Obeh54Y11At9DsTm6ugqilzFNs9WhzKLzOtLaL3cEjcyAl0DuZvShk1Hsrp4uTq9hHz8PxR2L9+PnMM3m2yYNby2ed5/As/pBTNPAddkvcX3zZ62GwAn68OnhlVMPb+3qooteRlU1QqFgTxej3wuFgqiq1q5zJAgaBXM/gYAXm8yqPYlid+GY/B2M0nyC+z8+5Xjw0FYaXr6TYP6n2CcuIuLKe9EHj23z9bW0kSgR8QQObOjKYoteyOWKpLa2qsU3FKLzTNOgtrYSl6v5TuGWSNNQI/++D1Fj09BShvd0UXodfcQ01N3r8H3yMnrmJBS7C9PfgHfD8wT3r0dNGIRrzn91aAc2RVGxjZiGf/tbGJ4aVFd0N9RA9AaRkTFUVpZSXHwU6PxgRVVVMQxrhcrp66xgtzuJjIxp13UlCIBQxbHwnsRTF0v7ZTMURcU5/RoaXrsb/9ZVaAPG4P3gacyGSuwT5mM/d+EpHcHtoQ+fjn/bmwRzN8vSFv2YoijExyd32fV6+3DZ7tBddZYgAIIHPgZVQx/RubVP+jMteSj6yPPxf/E2bF+NGpOKa+FvT9uB3KZrxw9ATRhE4OBGCQIheoAEARAqzUdNHCLNEqfhmHwlRkke2sCzcUy+sktnXtuGT8e3+UWM6uOoMalddl0hxOl1urP4vffeY9GiRSxcuJD58+ezdu1aAPLz81m8eDGzZ89m8eLFHDp0qOmc1o71BKOyEDU2vUfL0Beo7lgirvoDzulLunz5DX34VECROQVC9IBOBYFpmtx+++088MADvP766zz44IPccccdGIbB8uXLWbJkCWvWrGHJkiXcddeXq+S1duxMM331mJ5qCYIepkbEoQ3IInBwo6xIKsQZ1uk7AlVVqa0Nd17U1taSnJxMZWUlu3fvZt688Bo18+bNY/fu3VRUVFBeXt7isZ5gVBUBoMWdfhMJ0b1sw6dh1pRglOT2dFGEsJRO9REoisJf/vIXli5ditvtpr6+nr/+9a8UFRWRkpKCpoUnNWiaRnJyMkVFRZim2eKx+Pj4zteonU4EgRorQdDT9MxJsP6f4RVJZRivEGdMp4IgGAzy17/+lccff5yJEyfy2Wefceutt/LAAw90Vfla1NIqem2RlBTV9HX5F2V4NZ3koUPDWzv2U1+tc+8VhTlqMp78T0hccGOnhqRCX6lz17Jana1WX+ieOnfqN23Pnj2UlJQwcWJ4A5eJEyficrlwOBwUFxcTCoXQNI1QKERJSQlpaWmYptnisfboqmWoGwoPoUanUlbe0O5r9RV9aby1Mfg8jN0fU/DSH3FeeD2K3d2h6/SlOncVq9XZavWFztW5tWWoO9VHkJqayvHjx8nLywMgNzeXsrIyhgwZQlZWFjk54Z13cnJyyMrKIj4+noSEhBaP9QSjqghV+gd6DW3QOOyTrySY/xn1K5cTKj3U00USot/r9MY0b7zxBn/729+aZuT+/Oc/5xvf+Aa5ubksW7aMmpoaoqOjyc7OZujQoQCtHmurrrgjMIN+6p75CfYJC3BMuqLd1+or+uI7p+DxA3jffQLTU4Nj6mJsY77RrlnffbHOnWW1OlutvtB9dwSW3qEsVF5Aw8rf4Zz503699HRf/YUxvXV43v8boSPb0TMm4rzohyiOiDad21fr3BlWq7PV6gvdFwSWnllsVBUCoMbJHILeSHFG4pr9CwI71uDb/DL1K+/ClnUxakxq40cyii57GAjRWdYOgspCQJElDXoxRVGwj70MLXUE3g+exr9l5cnHI+JRY1NR4wehpY1CTx2J4uz4iDIhrMjaQVBVhBKV2OXLJYiupyUPI+I7f8AMeDGqixs/joc/qo4T2P0ugR1rAFDjBqJkjiEQm4mWfhaqO7aHSy9E72bxICiUZqE+RrE50RKHoCUOOelxM+gnVJpPqGgfoeP7qd35Aab/RDCkow0Ygz5gNFraWbIDnRBfY9kgMA0Do/o4toFn93RRRBdQdDt62ij0tFEAJCa4Kd67i1DhXoLHdhHY8wGBne+AoqElD0UbMg772MtQVMv+CgjRxLK/BWZtKYSCaLLYXL+kqBpaUiZaUib2cZdjhgKEig8SOrqL4LFd+D95hdDxg7i+sbRNTYPBwj2ocQP65VLlpmHg//T/UNwx6IPHo0Yn9XSRxBlm2SCQEUPWomg29PQs9PQsHFyJf/c6fOv/iWfNX3B98+coNmez55lBP74NKwjs/QA1Nh33wjvbPIS1rwjseQ//tvAET9+GFahx6eiDx6MNmYCWPAxFla3N+zvLfodDlbLYnJXZR8/EefGPCRXuoWH1HzF99ac8x6gpoeH1+wjs/QB95AyMmhI8a/8HMxTogRJ3D9Nbh+/T/0NLzyJicTaOaVejuGLwf7EGzxv3Uf+vn+Pb/BJmwNvTRRXdyNJ3BIorpt+9uxNtZxs5A3Q73nVP0pDzAK45v2pq+gke2orn/b8B4Jp9C/qQCQQGnoN33ZN43/sbzlk/RVH6/vso35aV4PfgmH4NakwK9nNmYz9nNqavnuDRXQTzt+DfvprAwU04pl+DnnGu7OvdD1k6CKRZSNiGnodic+BZ+wieVffjmvMrArvexb99NWriEFzfuLmpzdw2fCpGXQX+T17CF5mAc+riHi5954TKDhPY8z62s7+BFj/wpGOKIwLbsMnYhk0mePwAvvXP4n3nEbTB43BOv1b6EfqZvv+WpgNM08SoLJJmIQGAPmgsrjm3YdRXUP/v2/FvX40t62LcC+485Q+efdzl2EbPIvDFW/h3/qeHStx5pmni27ACxRmJY+KiVp+rp47A/a3f45i6mFDhXupfvhPf1lWYoeAZKq3obtYMgoYqCHgkCEQTPW0U7rm3oyYMxnnxj3Fe8INmRxMpihJuIhkyIdyJnP9ZD5S284K5mwgd34998pVtah5VVB372MuJuOoP6IPOwb9lJQ2r/h+mIWHQH1gyCL7clUyahsSXtOShRFyxPNx30ApFVXHO+ilqcibedU8SOPQZZtB3hkrZeWbAi2/Ti6iJGdhGXtCuc9XIBFzf/BnOi3+EUZKLf/tb3VRKCOR/SrBwb7ddX3zJkn0E4TWGZOio6DhFd+Ca/QsaXr8P79pHQNFQEwahpQxDSxmBljIMJTKxV3as+rfmYDZU4br05g4PDbWNPJ/gke34P3sdPWMiWhf/LgWPfIH3ncdQkzLQr1jepdcWp7JmEFQVgs2JImvQiE5QXdFEfOv34WUtig8SKj5IYN9HBHa9Gz6eMBj33Nt71SJ4RnUx/i/eRh8xo9P7QjumX0vo2B68HzyNe8GdXTbfwKgqwrPuCcDEKC/ADAU7vW2paJ0l/3eNqiLU2PRe+W5N9C2K3YU+ZDz6kPEAmEYIo+IooaK9+Da/jOedR3HNve2MLmURKjtMYP96FLsbxRUVHibtikZ1RePd9G/QdBxTvtPp11HdMTimL8H73lMEdr6DfezsTl/T9NXTsObhcJ/ElMX4Nr+IUVGAlpTZ6WuLllk2CLSBY3q6GKIfUlStaVE8xRGJ9/2/4Vv/HI4Lvn9G3niEyg7TkJMNQT8YIeDUzZscUxZ32Yqs+vBpaLmb8W1ZiZ4xATU6ucPXMg0Dz7onMWtLcc29HTUiHt/mFwmV5ksQdDPLBYHhrcdsqJKOYtHtbCNnYFQV4t/2Jmr8AOxnX9qtr+cvLcCz+o8odhfuK+9BccdhemsxPTWNH9UQCqKPnN5lr6koCs4LfkD9S7/B+8Hfcc27vcMT7XyfvESoYAeOC36AnjYK0zRRHJGESvJhdJcVuVsYnhoCX7yN/dwFLS5X0ptZLgj85ccAZLE5cUbYz/s2RlURvo3Po8akog86p1tex6gupujN+0HVwsNgIxMAUNwx4I7pltc8QY2IwzHtu/g+fIbAng+wj76k3dcI7P+YwBdvYxs9E3vWxUA4ZNTkTIyy/C4ucdcL7F6Hf/tqUBQckzvf7HamWW74aKDsKABqnMwhEN1PUVScl9yIGj8Qz7uPE2ocsdaVjNoyGt58ANMI4Zr7X6gxKV3+GqdjG3Uh2oDR4Tb9uvJ2nRsqycP70TNoaWfhmL7kpGNaUiZG5THMQO8dnmuaJoGDGwHwf/E2RvXxHi5R+1kuCPxlR0HVUaJkirw4MxSbE9fsX6BoNjxr/oLpreuyaxv1leEQ8HtIu/p3aHEDuuza7RFuIroeTAPvh8+0eaJZqPQQnjV/QXHH4rz0plM61bWkTDBNQuWHO1W+UMVRTNPo1DVaYpTmY1YXYz/v26DZ8G54vltepztZrmkoUHYUNSYVRdV6uijCQsITsX5Ow6r7aXjrz2gpwzD9XvA3YAa8mP4GCPoa92BOQ41JDX+OTUNxxzZ1NJumCUYQgn5MT214NVRPDe45t+FIHQqltT1Xx+gkHFO/i2/9P2lYdT+uWUtRI+NbfH4g7xO87/0vijMS12W3ojqjTr1mYyexUZIPqSM7VK7AwU141z2Jffw8HJOv7NA1Wr/+RlB17KNnomg6vk0vEjyyDX3w+C5/re5iuSDwlx9DjRvU08UQFqSlDMd58Q14P3o2vF+23RXeNtPuQnFGomjxGPUVBPZ9BF9d9tnmRNHtmEE/BH1gfmUkkGbDdfmvOj0noKvYR89EsbvxfvQPGlbehXPmjeiDxp70HNM08X/+Ov7PXkNNGY7r0p+httCPobpjUSLiCZV2rJ/A8NTg27ACFA3/9tXoQyehJWZ06FrNMY0QwdzN6IPHhRfqG3Mpgb0f4t3wAhEDxqBoti57re5kqSAwg36CVSXYM6f0dFGERdmGT8M2fFqrzzFNE7OhCqOqCKOqMLwkihEC3RFe/0i3N352hGcyx/euNza24VPREofg+c9jeN76M/bx87BPugJF1TCDPrzvP00w7xP0ETNwXviD0/6x1JIyOxwEvg0rMP0NuObfgfedx/B+8HfcV9zVZfM6QoV7MD016CPC31NF03FMvwbP6j/i37EGx/h5XfI63c1SQRCqPA6mIYvNiV5NURSUiDjUiDgY0MvHTbZAjU3DvegufBuew78th1DxARxTv4t3/T8xSg9hn3wV9nGXt2luhZqUSfDQZ5i++nbtHxI8tIdQI/cAAB7QSURBVJVg7mbsE69ATx2J4/zv4X3nEfzb38IxYX5nqtckcGAj2F0n3fXoA89GzzgX/+ersI2YEf4+9nKW6izOeTvcsy9rDAnR/RTdjvPCH4Z3givNp+HV/8aoKsL1zZ/jGD+nzRPsTkwma89dgemrx7v+WdT4gdjHzwXAljkRPXMS/s9eJ1TV+dFbZtBH8NBn2DLPO2WlWsfU74IZwrf5xfZfNxTE9Hs6Xb72sFQQRAXLMQE1JrWniyKEZdhGzsB9xXL0kefjXngnesaEdp2vJWUA7QsC36YXMT3VOC+64aR1ihwzrgObA+8Hf8c0OjeKKHh4GwS86MOnnnJMjU7GPm4OwYObCBbta/M1TdPEs/pB6l/9/Rnd78FSQZCiVlNNVLPrzAshuo8WNwDXxT/qUH+G4ohAiU7BaGMQBI/uIrDvQ+xjLz9laQrVHYNz2hKM4oMEdr/b7rKc9DoHN6G4Y9HSzmr2uH38XJSIeHwbnmtz6ARzNxMq2odZXUxg//pOla89LBUEUdRTGOz97XVCiJO1tcPYDHjxfvQMSkwq9hZ2XtNHTEcbdA6+T17GqC3tUHlMbx3Bgi/Qh09tcdVVRXfgmPZdjPICAjvXtqHsPnybX0JNHIKaNBT/GdwFzlJBkDd4Pi/VTcLnD/V0UYQQ7aAlZWLWV2I0VLX6PN+WlZi15Tgv+mGLd/4n1kdCUfF++I/w3Ix2CuRtASN02hFgeuZ54d3sPnmFUFnrk+L821dj1lfgmH4NjkmLMOvKz9hdgaWCwJ44mEojkur63jtdXQhxKjX5KxPLWhAqySWw8z/YxsxEP83kMzUyAceUqwgd24X33ScIHNiAUV/Z5vIED25EjU1HTRjc6vMURcFx0Q9RnJF41z3Z4k52Rl154zyHyeipI9EGnoOafObuCiwVBDER4XcI1fX+Hi6JEKI9tIQhoCiESvNafI5vy0oUVxSO89o2e9iWdTG20TMJHtuF972nqF9xK3UvLsP70T8I5G5ucSkQo7aM0PH94Wahtgx/dUbhvORGjKrj+Da+0HzZN78EgGPqYqAxQCZeEb4r2Pdhm+rTGZaaRxB9IgjqJAiE6EsUmwM1bkCL/QTBon2Eju3GMe3q8GzttlxTUXGe/z0cM67FKC8gVLiHYOEeAgc3EdjzPugObKMvwT728pNmPgdyNwGctlnoq/QBo7GPuxz/9tVoA8/BljnxpLIHczdjP3dh06qxANrAs1FThuPfmoNt1AXdOkvZUkEQG+kA5I5AiL5IS8okcOjz8D4FX3sn7v/0VRR3LLas9i+BrShq02ZC9rGXhXeZK83Hv+tdAjvWENi1DlvWxdjHXY4aEUfwwCbUlOGo0e1buNI+6VsEC/fg/fDvaEmZqJHxmKaBb+PzKBHx2MfP+Vq5FBwTF+FZ/UcC+z7CPnpmu+vWVp1uGvL5fCxfvpxvfvObzJ8/n9/97ncA5Ofns3jxYmbPns3ixYs5dOhQ0zmtHetOkW4bqqpIH4EQfZCalAm+esyvjfQJFu4hVLQ3PFyzC4aGK6qGljIc18yfEPGd/4c+bDKBXf+h/t//hWfdkxiVR9t1N9B0XU3HNfMnEAriff9vmIZBcN96jLLDOKZchaI7TjlHGzCm6a7ADAU6XbeWdDoIHnzwQRwOB2vWrGHVqlXccsstACxfvpwlS5awZs0alixZwl133dV0TmvHupOqKMRGOqiSpiEh+hwtaSgQXrr6BNM0w3cDEXHYzrqoy19TjU3FdfGPiFh8P7YRMwjmbQFVQx82uWPXi0nFOeNaQoV78H/6f/i2vIKaMhx9WPPrnzX1FdRXENj7QWeq0nq5OnNyfX09r732GrfcckvTrVpiYiLl5eXs3r2befPCCy7NmzeP3bt3U1FR0eqxMyEu2kGNNA0J0eeo8QNB1U/qMA4d20Xo+H7sE+Z360RRNToZ54XXE/HdB3Av/F2zS2a3lT7yfPShk/Fvy8H01OCcfk2rnc7agNFoqSPxb3sTI9g9f7s61UdQUFBAbGwsjz76KJs3byYiIoJbbrkFp9NJSkoKmhZe81/TNJKTkykqKsI0zRaPxce3vHb51yUkRHaozHFRTiprvSQldfwb2RdZrb4gde6PAqmZKFUFJCVFYZomxvZVaNGJpM+Yg6KfgSWfu+j/N3TFzRQ++xtcGeeQOHrsaZ/vmXk1Rc//N7Vb/0PSeXNO+/z26lQQBINBCgoKGD16NHfccQfbt2/npz/9KQ8//HBXla9F5eV1GEb7J4LERTk4UFBJaQ9u4HGmJSVFWaq+IHXur4zYwQQOfExJcTWRdXn4ju3DccEPKKv0At7Tnt+bOBbdjaEobfqemREZaKkjqd68Cn/GBR16PVVVWnwD3akgSE9PR9f1pmaecePGERcXh9PppLi4mFAohKZphEIhSkpKSEtLwzTNFo+dCXHRTmrrAxiGiaq2bfVDIUTvoCVnEtj9LkZ1EZXr/40SlYht5Pk9XawOaWlpimafqyg4L7weR9luuqPLuFN9BPHx8UyZMoWPP/4YCI8GKi8vJyMjg6ysLHJycgDIyckhKyuL+Ph4EhISWjx2JsRHOTBMkzpP9/XACyG6x4mtK/2fvoqv6CCOCQtOWl20P1Nj04id1vz6SZ2lmB1ZaOMrCgoK+M1vfkNVVRW6rvOLX/yCiy66iNzcXJYtW0ZNTQ3R0dFkZ2czdGi417+1Y23V0aah/UW13P/sFn5//XkMTunf7aknWKHJ4Oukzv2TaRjUPbs0vPxzbArOK+/rst3G+oLOfI+7rWkIYNCgQfzrX/865fFhw4bx8ssvN3tOa8e6W1xUeKyujBwSou9R1PDkr1DRPuIu+A5eC4VAd7Lc/2J8tBNA5hII0UfpmZPANIk8+0K85Q09XZx+wXJB8OUyEzK7WIi+yH72pdjPvhRF1Xq6KP2GpVYfBXA6dJx2TdYbEkKIRpYLAoCYSIesQCqEEI2sGQQRdrkjEEKIRpYMgthIO9V10kcghBBg0SCIljsCIYRoYskgiImw4/WHZBN7IYTAokEgQ0iFEOJLlgwC2cReCCG+ZMkgkE3shRDiS5YMAtnEXgghvmTJIIh021AV2cReCCHAokGgKgrRETZZeE4IIbBoEADERMgm9kIIAVYOgki7dBYLIQRWDoIIO1XSRyCEEBYOgkh70yb2QghhZdYNgojwJva1som9EMLiLBwEJyaVSfOQEMLarBsEkeEgkJFDQgirs3AQhGcXy1wCIYTVWTcI3CcWnpOmISGEtVk2CBx2TTaxF0IILBwEIJvYCyEEWD0IZMtKIYSwdhDIJvZCCGHxIJBN7IUQwuJBIJvYCyGExYNANrEXQgiLB4FsYi+EEBYPAtnEXgghLB4Esom9EEJYPAhkE3shhOjCIHj00UcZNWoU+/fvByA/P5/Fixcze/ZsFi9ezKFDh5qe29qxM0k2sRdCiC4Kgl27drFt2zbS09ObHlu+fDlLlixhzZo1LFmyhLvuuqtNx8402cReCGF1nQ4Cv9/P3XffzfLly1EUBYDy8nJ2797NvHnzAJg3bx67d++moqKi1WM9QTaxF0JYnd7ZCzz88MMsWLCAQYMGNT1WVFRESkoKmqYBoGkaycnJFBUVYZpmi8fi4+Pb/LoJCZEdLnNSUlTT1ykJERwtrTvpsf6ov9evOVLn/s9q9YXuqXOngmDr1q3s2LGD2267ravK02bl5XUd2ng+KSmK0tLapn87dIWqWj/FxTWoqtKVRew1vl5nK5A6939Wqy90rs6qqrT4BrpTTUNbtmwhLy+PWbNmMXPmTI4fP84NN9zAkSNHKC4uJhQKL90QCoUoKSkhLS2NtLS0Fo/1BNnEXghhdZ0KghtvvJH169ezbt061q1bR2pqKk8//TRz5swhKyuLnJwcAHJycsjKyiI+Pp6EhIQWj/UE2cReCGF1ne4jaMnvf/97li1bxuOPP050dDTZ2dltOnamySb2Qgir69IgWLduXdPXw4YN4+WXX272ea0dO9NkE3shhNVZemYxyCb2Qghh+SBo2sRe7giEEBZl+SCAxk3spY9ACGFREgTIJvZCCGuTICC8iX15tQfTbP8ENSGE6OskCICzMxMor/HxRW55TxdFCCHOOAkCYOqYFBJjnLzx8SG5KxBCWI4EAaBrKnOnDSG/qIZd+T2zCqoQQvQUCYJGM85JIz7aIXcFQgjLkSBopGsqc6cO4eCxavYeruzp4gghxBkjQfAV549NJy4qfFcghBBWIUHwFTZd5bIpg9lXUMW+I3JXIISwBgmCr7loXDrREXa5KxBCWIYEwdfYbRqXTxnMnsOVHDha1dPFEUKIbidB0IyLxw8gym1jldwVCCEsQIKgGQ67xmWTB7Mzv4K8wpqeLo4QQnQrCYIWXHLuACJdNlZ9nN/TRRFCiG4lQdACp11n9uRBbM8t582Nh3q6OEII0W26bc/i/mD25MEcK6tn5Qd5eHwhvn3RUBRF6eliCSFEl5IgaIWuqfxo3micNo3Vmw7j8Qe55tKRqBIGQoh+RILgNFRF4brZo3A6dN7efASvL8QP556FpkqrmhCif5AgaANFUfjOxcNwOXRe/TAPXyDETxaMwaZLGAgh+j75S9ZGiqIwf3oGV39jBJ/vL+V/XtlOvTfQ08USQohOkyBop0snDeL6OWex+3Alv/7rJt7begzDkGWrhRB9lzQNdcAFY9MZkhLFC/85wL/W7OO9z4+x5BsjOGtIXE8XTQgh2k3uCDpocEoUty+ZwP+36Gw8vgAPvLCVx1/dQVm1p6eLJoQQ7SJ3BJ2gKArnnZXMuGEJvL35CKs3HWZ7bjkzzknjgrFpZKRGybwDIUSvJ0HQBew2jQXnZ3L+2DRe/SiPj3cU8f7WYwxMiuD8selMG5NClNve08UUQohmSRB0ofhoJzfMHc3Vs0aweU8J678o5N/vHuDl9w4yYUQiU8ekMiYjHodd6+miCiFEEwmCbuB22rhkwgAumTCAoyV1rN9RxIadx/l0Xym6ppI1JI5xwxMYNyyRhBhnTxdXCGFxEgTdbGByJN+dNYIrLx7GgYIqth0sZ/vBMp5bW85z7GdgUiTjRyQyaVQSg5IjpU9BCHHGSRCcIbqmkpURT1ZGPN+dNZzjFQ1sP1jOtoNlvLnxEDkbDpES52LSWclMGpXM4BQJBSHEmSFB0AMURSEtIYK0hAgumzKYmno/nx8o5dO9Jby16QhvbjxMcqyLiaOSGDc8kWEDomVtIyFEt5Eg6AWiI+xcPH4AF48fQG2Dn60Hyvh0bwlrtxTw1uYjuBw6YzLiOGdYAucMTSA20tHTRRZC9COdCoLKykpuv/12jhw5gt1uZ8iQIdx9993Ex8eTn5/PsmXLqKqqIjY2luzsbDIyMgBaPWZ1UW47F45L58Jx6TR4g+w+VMGOvHJ25JXz6b5SAAYnR3LWkDhGDIxh+MBYYiJkaKoQouMU0zQ7vFBOVVUV+/btY8qUKQBkZ2dTXV3NH/7wB773ve/x7W9/m4ULF/L666+zcuVK/vnPfwK0eqytysvrOrTGT1JSFKWlte0+r6eZpklBSR078srZmVdBXlENgaABQEqcixEDYxkxMIZRg2NJinWd1L/QV+vcGVLn/s9q9YXO1VlVFRISIps91qkg+Lo1a9bwwgsv8Kc//YnZs2ezefNmNE0jFAoxZcoU1q5di2maLR6Lj49v82tZLQi+LhgyOHS8lgNHqzhQUM2Bo1XUe4MAxEU5OGtwLKMGx3HW4FhGj0imrKyuh0t8ZvWX73N7WK3OVqsvdF8QdFkfgWEYvPDCC8ycOZOioiJSUlLQtPDEKU3TSE5OpqioCNM0WzzWniBoqUJtkZQU1eFze5O01BimjR8IgGGYHC2pZVdeOV8cLGNnbjkbdxUDkBjr4pxhCZw9LJFzhiWSmuC2xIik/vJ9bg+r1dlq9YXuqXOXBcE999yD2+3m2muvZffu3V112RZZ/Y6gOS5NYdKIRCaNSMQ0TYrKG9h3pJK84jo+21PMe58dBcJ3DKMGx3LW4DhGDoolJc7V74KhP3+fW2K1OlutvtDL7wiys7M5fPgwTz75JKqqkpaWRnFxMaFQqKn5p6SkhLS0NEzTbPGY6DqKopCeGEF6YgRXJUVRUlLTFAz7CqrYfaiSTY13DJEuG8PSoxk2IIZhA2LITIvCaZcBZUJYRad/2x966CF27tzJU089hd0eHr2SkJBAVlYWOTk5LFy4kJycHLKyspqaflo7JrrHV4PhknMHYpomxysa2F9QRe6xGnILq9meW974XBiUFBnugB4Uw8hBsTJkVYh+rFOdxQcOHGDevHlkZGTgdIbXzBk4cCCPPfYYubm5LFu2jJqaGqKjo8nOzmbo0KEArR5rK2kaaru21rnOEyCvsIbcY9UcPFZNbmE1/kB4ZFJSrJORA2MZMSg8Oik1vnf3M8j3uf+zWn2hj4waOpMkCNquo3UOhgwKSurYX1DF/oIqDhytps4T3qc5wqkzbEAMwxs/MtOie9WqqvJ97v+sVl/o5X0Eon/SNZXMtGgy06KZPXlwU3PSwaPhO4aDx6r5orE5SVUUBiZFkJEWTWZaFJlp0aQnRqBrsjSGEL2dBIFos6+ukXTBuHTgRHNSOBTyC2v4bF8JH24vBMCmqwxOjiQjLZqhadEMTY8muR+OUBKir5MgEJ0S6bIxdlgiY4clAuEZ0CVVHg4V1ZJfVMOhoho++qKQdxuHrrodOpnp4buMzNQo0pMiSIpxoaoSDkL0FAkC0aUURSElzk1KnJspo1MACBkGRWUN5BXVkF9UQ15hDas3HsZo7J6y6SppCW7SEyMY0DiyadiAGKJle08hzggJAtHtNFVlYHIkA5MjubCxScnnD3G0tI5jZfUUNn7sO1LVNLcBID0xgpGDYhk1KJaRg2KJi5IhrEJ0BwkC0SMcdq1pAttXeXxBCkrqOHC0iv0F1WzadZz3tx4DIDnWxeCUSFITIkhPcJOWEEFqvLtXjVYSoi+SIBC9isuhM7LxDmDutHCz0tGSevYdqWT/0WoKSuv5fH9ZU7MSQEK0k8EpkU3DWTNSrbf+jBCdIUEgejVNVRmSGsWQ1Ci+OTn8WCBoUFLZQFF5A0Xl9RSVN5B/vJatB8oaz1EYNjCGwcmRZKZGkxzvIjnWRaTLJiOWhGiGBIHoc2y6yoCkSAYknTw5pqbBT96xGg4eq+ZIaR0fbivkP8GjTcddDp3kuHAopMS7GZISyZDUKBKinRIQwtIkCES/Ee22M35EIuNHJJKUFEXR8WqKKz2UVnooqfJQUtlASaWHw8W1fLavtKl5KdJlY0hqFBmpUQxJiSI9MYLkOJdMhhOWIUEg+i1dUxnQOCT16/yBEEdL6zl8vIZDx2s5fLyWtzcfIdS4bImqKCTGOkmLd5P6lY7ptAQ3UTKsVfQzEgTCkuw2jaHp4dnOJwSCIY6V1Tf2PTRwvKKB4+X17D5c2bQtKITvINISwqGQGh9BaoKb5FgXSbFObLqMYBJ9jwSBEI1sukZGajQZqdEnPW6YJuXVXo5XNFBUVk9R4+etB8qobShqep4CxEc7SI5zkxTrIiXORVJs+CM5zoXLIb9uoneSn0whTkNVlKY/6OcMTTjpWJ0nQHFFuO/hq/0Qn+8vbVqp9YRIl63xOk6SYl0kxjhJjHWRFOMkPtopfRKix0gQCNEJkS4bkc1MjANo8AYprfI0fZQ0fs4rrOHTvaUnzYVQFIiPcpAQ7SQ+xhn+HO0kIdpBfLSTuCgHbocuo5tEt5AgEKKbuJ160xyIrwsZBpW1PsqqvJRWeyir8lJW7aGixsfBo9VsqS1p6rg+QddUYiPtxETaiY10EBvhID7a0XS3EhHlPFNVE/2MBIEQPUBTVRJjXCTGuDiLuFOOG4ZJdb2f8hovFTVeqmp9VNX5qar3UV3np7Csnj2HKmnwBU8670TzU2KMsykwYiLC4RET4SA20i4T68QpJAiE6IVUVSEuyhFeaK+ZZqcTPL5w81NJpQdP0CD/aBWlVR6OFNdSnefH6w+dco7ddiKEnCTFuEiIcZIU6yQ5zk1ynAuHTUY+WY0EgRB9mMuhMzglisEpUc1uY+j1B6mu91Nd56e63k9VrY+y6nAzVFm1lwNHq/D4Tg6LuCgHqfFuUuLCM7Cj3DZ0TcWmqei62vS13aYS6bIR4bJJePRxEgRC9GNOu47TrpMS5272uGmaNHzlrqK4ooHjFeHRT1v2llDvDTZ73tfZ9MZQcNqIdOlEuu1EuW1EuWxEfe3rSLeNSJdNRkn1IhIEQliYoihEOG1EpNpOmT8B4eGxDd4AgZBJMGgQCBkEgwbBkIHXH6LeG6DOE6DeGwx/9oT/fbSkrunfZjOvC+Hd6iLdNqLcNuKinCQ2jpZKjGn8OsaJ0y5/os4E+V8WQrQo0hV+995RIcOg3hOktsFPbUM4JGob/NR6AtQ2BJoeLyipY9uBMoIh46TznXbtpDuNiMbyuJ02EuPchAJBnA4dp13DaddxOTTcThsRTh2XQ0eVTvE2kSAQQnQbTVWJjrATHXH69ZkM06Sm3t/Uh1Fe7aW63t94lxG+4yit9lLvCdDgDbZ4p3GCQrgPxe1sDJDGz+GvGz+7dOy6hqoqqIqCqipoqoKqgMOuh5u03OE+kP480kqCQAjRK6iKEp4fEelgeCsjpSAcGtExbo4WVuH1h/D6g3h9ITz+IA3eIPXeIA3eAPWeIPW+xs/eAGXV3sbmrtMHyVfZdZUot41Itx23Q8dsnAxomIBpYhJuZotrnBSY8JVmroRoJ3ab2quDRIJACNHnqIqCy6ETG9mxfawNI9xJXu8JEAgZGIZJyDAxTDP8dcjEGwg1NV2d+FzTEB6SqxD+w68AiqqgKAqhkEHusWo+3XvqZEBVUbDZVOy6il3XsNvCn3VdQVdVdE1B01Q0VQmPytLVpuau8Ofw1+eMChHj6PoRWhIEQgjLUVWl0/0fLTEMk6o6H+U1XsqrvZTXePEFQvgDBv6ggT8QavocChkEG0Mn6A0SChmEDBN/INR4pxM6KVTi1+fzx6XTu7zMEgRCCNGFVFUhvnGtqBEDO3+9QNDA6w/i84cYNCAWT72v8xf9GgkCIYToxWy6ik23E+WGSLe9W4JAZnQIIYTFSRAIIYTFSRAIIYTFSRAIIYTFSRAIIYTFSRAIIYTF9dnho6ra8enanTm3r5I6W4PV6my1+kLH69zaeYppmu1ZckMIIUQ/I01DQghhcRIEQghhcRIEQghhcRIEQghhcRIEQghhcRIEQghhcRIEQghhcRIEQghhcRIEQghhcZYJgvz8fBYvXszs2bNZvHgxhw4d6ukidbns7GxmzpzJqFGj2L9/f9Pj/bXulZWV/PjHP2b27NnMnz+fm2++mYqKCqD/1hlg6dKlLFiwgEWLFrFkyRL27NkD9O86n/Doo4+e9PPdn+s8c+ZMLrvsMhYuXMjChQv56KOPgG6qs2kR1113nfnaa6+Zpmmar732mnndddf1cIm63pYtW8zCwkLzkksuMfft29f0eH+te2Vlpblp06amf99///3mr3/9a9M0+2+dTdM0a2pqmr5+5513zEWLFpmm2b/rbJqmuXPnTvOGG24wL7744qaf7/5c56//Hp/QHXW2RBCUlZWZEydONIPBoGmaphkMBs2JEyea5eXlPVyy7vHVHyAr1f3tt982v//971uqzq+++qp5xRVX9Ps6+3w+86qrrjKPHDnS9PPd3+vcXBB0V5377Oqj7VFUVERKSgqapgGgaRrJyckUFRURHx/fw6XrXlapu2EYvPDCC8ycOdMSdb7zzjv5+OOPMU2T//3f/+33dX744YdZsGABgwYNanqsv9cZ4LbbbsM0TSZOnMgvf/nLbquzZfoIRP92zz334Ha7ufbaa3u6KGfEfffdx/vvv8+tt97KAw880NPF6VZbt25lx44dLFmypKeLckatWLGCN954g5UrV2KaJnfffXe3vZYlgiAtLY3i4mJCoRAAoVCIkpIS0tLSerhk3c8Kdc/Ozubw4cP85S9/QVVVS9T5hEWLFrF582ZSU1P7bZ23bNlCXl4es2bNYubMmRw/fpwbbriBI0eO9Ns6A031sNvtLFmyhM8//7zbfrYtEQQJCQlkZWWRk5MDQE5ODllZWf3m9rE1/b3uDz30EDt37uSxxx7DbrcD/bvO9fX1FBUVNf173bp1xMTE9Os633jjjaxfv55169axbt06UlNTefrpp5kzZ06/rXNDQwO1tbUAmKbJ6tWrycrK6rbvs2U2psnNzWXZsmXU1NQQHR1NdnY2Q4cO7elidal7772XtWvXUlZWRlxcHLGxsbz55pv9tu4HDhxg3rx5ZGRk4HQ6ARg4cCCPPfZYv61zWVkZS5cuxePxoKoqMTEx3HHHHYwZM6bf1vnrZs6cyZNPPsnIkSP7bZ0LCgr42c9+RigUwjAMhg0bxm9/+1uSk5O7pc6WCQIhhBDNs0TTkBBCiJZJEAghhMVJEAghhMVJEAghhMVJEAghhMVJEAghhMVJEAghhMVJEAghhMX9/7X5jtemjb/IAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt \n",
    "import numpy as np \n",
    "sns.set()\n",
    "\n",
    "x = np.linspace(0, N_EPOCHS,N_EPOCHS)\n",
    "\n",
    "plt.plot(x,t_loss)\n",
    "plt.plot(x,v_loss)\n",
    "plt.title(\"Loss\")\n",
    "plt.legend([\"Train loss\", \"Valid loss\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f3b3152ded0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAELCAYAAADTK53JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b3/8dc5Z2ayh5B9whZ2ArIjyo4RZBEJ1rVYW5ei1u2q9V6praLXbnh/te5al1ottnWtKCKCG4jIIrKHNSQEwmQnZJv9fH9/BKlIIAkkmcnM5/l48EhIzpn5fJnJm5Pv+S6aUkohhBAi5OiBLkAIIUTbkIAXQogQJQEvhBAhSgJeCCFClAS8EEKEKAl4IYQIURLwQggRoiyBLkCIs5WdnU15eTmGYRz/2rJly0hLS+OBBx5g/fr1HDhwgN///vf86Ec/CmClQrQvCXgREp5//nnGjh170tcHDBjAzJkz+b//+78AVHUin8+HxSI/cqL9SBeNCGnXXHMNY8aMISIiosljV65cycyZMxk+fDgTJkzg5ZdfPv69Tz75hJycHEaMGMGUKVNYtWoVACUlJdxyyy2MHj2aqVOn8uabbx4/56mnnuLOO+/k3nvvZcSIEfz73/+mpqaG+++/n/HjxzNhwgT+/Oc/4/f7W7/hQiBX8EIc9+tf/5rHH3+cUaNGcfToUQ4dOgTA1q1bue+++3jyyScZM2YMZWVl1NbWAvDLX/6SPn368OWXX7J//36uv/56unXrxpgxYwD49NNPeeKJJ3j00UfxeDzcc889JCcns3z5cpxOJzfffDN2u52rr746YO0WoUuu4EVIuO222xg1ahSjRo3i1ltvPaPHsFgs7Nu3j9raWjp16sSgQYMAePvtt7nssssYN24cuq6TlpZG7969cTgcbNy4kXvvvZeIiAiysrK44oorWLx48fHHHDZsGFOmTEHXdWpra1m1ahX3338/0dHRJCUlcd111/Hhhx+2yr+BED8kV/AiJDzzzDON9sG3xJNPPslzzz3Hn/70J/r3788vf/lLhg8fjsPhYNKkSScdX1paSqdOnYiNjT3+tYyMDLZv33787+np6cc/P3z4MD6fj/Hjxx//mmma2O32s6pbiFORgBfimCFDhvDcc8/h9Xp5/fXXueuuu1i5ciV2u53CwsKTjk9NTeXo0aPU1tYeD3mHw0FaWtrxYzRNO/55eno6NpuNtWvXys1W0S6ki0aENI/Hg9vtRimFz+fD7XZjmmajx73//vvU1NRgtVqJiYk5Puzy8ssv59133+Xrr7/GNE1KSkrIy8vDbrczfPhwHnvsMdxuN7t27eLtt9/mkksuabSW1NRUxo0bxx//+Edqa2sxTZPCwkLWr1/fpv8GInxJwIuQduONNzJkyBA2bdrEAw88wJAhQ9iwYUOjxy5evJjs7GxGjBjBv/71Lx599FGg4cr+D3/4A7///e8ZOXIkP/nJTzh8+DAAjz32GEVFRUyYMIHbb7+dO+64g3Hjxp2ynkcffRSv18vMmTM599xzufPOOykrK2v9hgsBaLLhhxBChCa5ghdCiBAlAS+EECFKAl4IIUKUBLwQQoQoCXghhAhREvBCCBGigmo63ZEjdZhmy0dtJiXFUlFR2wYVBS9pc3iQNoeHM22zrmt07hxzyu8HVcCbpjqjgP/u3HAjbQ4P0ubw0BZtli4aIYQIURLwQggRooKqi6YxSimOHCnD43EBjf8KU1qqN7qAVChrSZsNw0JsbAJRUafuqxNChJ6gD/ja2qNomkZaWlc0rfFfOCwWHZ8vvAK+uW1WSuH1eqiqaljQSkJeiPAR9F00TmctcXEJpwx3cXqapmGzRZCQkEJtbVWgyxFCtKOgT03T9GMYQf+LRtCzWm34/b5AlyGEaEcdIjm/vyuOODPybyhE4NW7vBwsraWwpLbhY2kNFUddPPjz80mJtbX683WIgA8W8+b9DK/Xi8/n5eDBQnr27A1Av379uf/+Bc16jPfeexu3281VV13Toue+/fabKCkpISamoQ+9R48ePPzwH9i7dzePPbaQPXt2M2bMOH7720db1ighRIt5fSYllfWUV7uorHZRcdRFRXXDn9p6b6PneHwmR2rcx/8eH2Oje2osg3slkWmPp67G1ep1SsC3wIsvvgqAw3GYn//8Wv72t3+cdIzP5zvtfptz5lx+xs9/1133Mm7cBOA/N1k7d07k9tvvYe/e3XzzzbozfmwhRNMOldWyavNh1mwvpt79ny5Pi6GRGBdJUqdIeqRHNvobs6FrZCTH0D01lm6psXSKjTj+vehIqwR8sLr88kuYNSuHjRs3kJHRhZtuupWHHvo1dXV1eDwexo4dx623/hcAL7/8F5xOJ7fffhdLl37AihXLiIuLZ//+POLiYvntbx8lKSm52c+dnJxCcnIKBw7kt1XzhAhrbo+f9btKWLXlMHlF1VgMjRH9UhjeN4XkhEiS4iOJj7GhB2E3aIcK+K+2OVi91XHS1zUNznbjwfFD7IwbbD/j88vLy3nqqb8A4Ha7Wbjwz0RHR+Pz+bjnnttZu3YN558/9qTzdu7M5dVX/0laWjoLF/6Wt99+g5tvvq3R53j88f/Hiy8+B8DVV/+Y6dMb39xZCNE0t8dPcWU9xZX1lFY58Xj9+PwmPr/Cf+yjy+tnR34FTrcfe1I0V2X3Yew56cRFt35/eVvoUAEfzKZPv/j456Zp8uyzT7Bt21ZAUVFRwd69exoN+CFDhpKWlg7AoEHnsGHDqbtZGuuiEUKcqLTKyf7DR/F4Tbw+E4/Pj9dr4vb5cbr9lBwL9e/3h0NDF4phaFh0HYuhYRgNH4f1SWbSsC707dqpww1W6FABP25w41fZwRB20dFRxz9/443Xqamp5oUX/kZERAQLF/4Oj8fd6Hk223+uBHTdwO/3t3mtQoQan99k095yVm4uIrfgSKPHGLpGpM0gtXM0A7p3Jj0xivSkGNITo0ntHEWE1Wjnqttehwr4jqKmpoakpGQiIiIoKytl9eqVzJlzWaDLEiLklFTWs2rLYVZvc1BT7yUpPoJLJ/RkeL8UomwWrFYdm0XHZjHQ9Y519d0aJODbwBVXXM0DD9zH9dfPJTU1jZEjz22z53I4DnPrrT/H5XLh8bi59NKZ3HjjTcyaNafNnlOItuD2+vH6TIxqF+VHnfj96nifeHW9hyM17mN/XByp8VBZ7aKovA5d0xjWN5lJwzIYlJkYlkF+KppSZ3t7svVUVNSetCZycfEB0tN7nPa8YOiiaW9n0ubm/FsGs5SUOMrKagJdRrsK1TbXOr0cKKnhQHENBcU1HCiupqyq6WGCGhAXY6NzXASdYyPolRHP+CF2Er435LAjOtPXWdc1kpJiT/l9uYIXQrQppRTFlfXsOVjFnoNV7D10lPKj/wnz5E6R9EiPY9xgO9ERFhISonHWu7EYOhZDx9A14qNtJMTZSIiNwGIE/QorQUMCXgjR6kqrnGzLq2B34RH2HKyi+tjszvgYG327dmLSsAwy0+PpkR5HbJT1hHND9beWQGhWwOfn5zN//nyqqqpISEhg4cKFZGZmnnBMRUUFv/rVr3A4HHi9Xs4//3x+85vfnHZWpxAidNS7fGzYVcKa7cXsPXQUgKT4CAb1TKJ/9wT6dUsgrXNUhxtq2JE1K30XLFjA3LlzycnJYfHixTz44IO89tprJxzz/PPP07t3b1544QW8Xi9z585l+fLlzJw5s00KF0IEnt802ZF/hDXbHWzaW47XZ2JPiuaySb0YnZVGSkJU0w8i2kyTAV9RUUFubi6vvPIKALNmzeKRRx6hsrKSxMTE48dpmkZdXR2maeLxePB6vaSlpbVd5UKIgKisdrEjv5IdBZXkFhyh1uklJtLChGOzwTPT4+QqPUg0GfAOh4O0tDQMo2ESgGEYpKam4nA4Tgj4W2+9lTvuuIPx48fjdDq55pprGDlyZNtVLoRoF16fn50HqtieX8GO/EocFfUAdIq1MbR3EsP6JjOkdzJWi9z8DDat1kG+bNky+vfvz6uvvkpdXR3z5s1j2bJlTJ8+vdmP0dhwn9JSHUsz3jjNOSbUtLTNuq6TkhLXRtW0j45e/5kIRJuP1rrZkFvC+txiNu0uxeXxY7PonNM7mZnjejK8Xyrd2/BKXV7n1tFkwNvtdkpKSvD7/RhGw1T60tJS7PYTlwxYtGgRv//979F1nbi4OLKzs1m3bl2LAr6xcfCmaTY53ru9xsHfc88dTJw4+YRZqUoprrwyh1//+iGGDRvR6Hm/+91DDBiQxWWXXXXa9eCXLv2ANWu+bHRN99ZYD940zQ49OiEcR1e0RZsPFNewdX8FGg3jqHVNQ9NA1zRcXj/b9leQd+goCugcF8GYQekM7ZPMgO4J2L43nb+8vLZV6/qOvM7Nd9bj4JOSksjKymLJkiXk5OSwZMkSsrKyTuieAejatSurVq1iyJAheDwevv76a6ZOndrigoPZxRfP5o03Xj8h4Ddt2ohhGKcM9x+S9eBFoBwqreW91fl8u6fstMd1T4vlknGZDO+bQve0WOlP78Ca1UXz0EMPMX/+fJ599lni4+NZuHAhAPPmzePOO+9k8ODB3H///SxYsIBLLrkEv9/Peeedx5VXXtmqxXr3fIV396qTvq5pGmc7IdfafyLWfuNOe8zEiZN57LE/kp+/n549ewHw4YfvM3PmJeTl7eNPf/ojLpcTj8fD7NmXcuWVc096jO+vB+/1evnznx9l06aNpKSk0r17ZovrlvXgRVMcFXUsXp3Php2lREYY5IzvyZRRXYmwGpimwlQK0wRTKXRNIzpShjaHima9kr179+att9466esvvvji8c+7d+9+fKRNqLJarUydOp2PPvqAW2/9L+rr6/jyy5XccsvtxMTE8Pjjz2Kz2aivr+emm37G6NFjyMzsecrHW7z4HRyOw/z972/i8/m47bZ5J3V9fZ+sBy9aoriyng/XFLBmRzE2i8HMMT2YNrr7iROLQm8BRfE9Heq/amu/cY1eZbfnWjQXX5zDvffewU033cann65gyJChpKSkUllZwdNP/5F9+/agaTrl5WXs27fntAH/7bcbmTFjFhaLBYvFwrRpM9i6dfMpj5f14EVTvD6Tb/eUsXJzEbsKq7BadC46txszzutBfEzH2KRCtJ4OFfDBoG/ffiQlJbNu3dcsXfr+8W6Yv/zlGRITk/jrX1/HYrFw99234fF4TvtYQbTOm+jgiivrWbW5YdncWqeX5E6RXDapF+MH20/Y+1OEFwn4M3DxxbP5619foKTEwfjxkwCora2hd+++WCwW9u/fx5Ytm5k69fQjiEaNOpdly5aSnT0Vv9/HihXLju/uJMSp+E0TR3n9sVUYa9jvqCbfUY2h/2fZ3IGZiUG5R6hoXxLwZ2Dq1Bk888yT5OT8CKu1oT/zZz+7kUceeZDlyz+iS5cuDBs2vMnHmT37R+zbt49rr72S1NQ0hg0bicNR1KJaZD340KeUYr+jmnW5JeQfruZgaS2eY91zETaDHqmxcrUuGiXrwXdQsh586KuqdbM1/wgfry3AUVGPzaKTaY8nMz2OHulxZKbHkZYYHXJX6uH2OoOsBy9EWPB4/WzNq2D1Ngfb91diKkWfLp24bsYAzh2QSlSE/MiK5pN3ixABVuv0smVfOd/uKWNHfiUen0nnuAhmnN+dSyb1wUbQ/JItOpgOEfBKKZlNd5aCqCdO0LDWy7qdpWzaU8aeQ1Uo1bAswPghdob3SyGre2d0XSMlJTbsuitE6wn6gNd1A7/fh8VibfpgcUperwfDCPqXO+QdKq3l4w2FrMstwedXdE2J4eIxmYzol0yPNFlmV7SuoP+Jj4qKpaamioSEJDQt/FaMPFtKKbxeD1VVZcTFdQ50OWFJKcX2/EqWry9kR8ERbFadiUMzuHBkV+xJMYEuT4SwoA/42NhOHDlSRknJIThFX6Su65hmeI2iaUmbDcNCXFxnoqIkTNpTVa2bjbvL+HxTEYfL60iItXHZpF5MGtblpH1IhWgLQR/wmqaRmJh62mNkWJUIFpXVLjbuKWPjrlL2Hltyt3tqLD+flcXorDQshvwWKtpP0Ae8EMGuus7D2twSNuwqIa+oGoAuKTHMHt+TUf1T6JJy6nHKQrQlCXghzoDPb7I1r4KvtjnYmleB31R0S43l0om9GNU/RfrWRVCQgBeiBQ6V1rJ6m4OvdxRTU+8lPsbG1FHdGDc4Xa7URdCRgBeiCU63j/U7S1i1xXHCol7jBtsZ3CsRQ5d+dRGcJOCFaIRSiv2Hq1m15TDrd5bi9vrpkhzD1Rf2ZcygNOKiZW11Efwk4IX4HqUUG3aV8sGaAorK6oiwGozOSmXi0Ax6ZcTLRCTRoUjAC3FMYUkN//hkL3sOVtElJYafTe/P6Kw0WeBLdFjyzhVhr7rew79X7WfV5sPERFn56bT+TByaga7L1bro2CTgRdjy+U0++7aIxavz8Xj9TBnVjdnjM4mJlFmmIjRIwIuwYpqKPQerWL+zhG92l1Hr9HJOz0SuvrAvGckydl2EFgl4EfKUUuQ7alh3bLZpVa0Hm1VneN8Uxg1OZ1Bmotw8FSFJAl6ELK/PZO2OYpatL8RRUY/F0BjcK4nzBqYxtHcyETYj0CUK0aYk4EXIqXf5WLmliBUbDlJV66F7WizXzxzAyH6pREfKW16ED3m3i5BRVetmxTcH+WJTEU63n4GZnbnx4oEMzOwsXTAiLEnAi5Dw1TYHf1++G6/PZFT/VGac353M9PhAlyVEQEnAiw7N7fGzaPluvtpeTL9uCVw/cwBpnaMDXZYQzaJ8btwb3sU7MQdo/fetBLzosA6V1fLce9sprqjnkrGZzB6fKQt/iQ7F/fU/8e5ciTl6ChgS8EKglOLLrQ7+sWIPkREW7rl6GIMyEwNdlhAt4i3YiHfnF1iHzCAivRe0wQ5tEvCiQ6mqdfPm5/tYu6OErB6duemSgXSKjQh0WUK0iFl3BPfKV9CTexBx7mVt9jwS8KJDKK9y8vaq/SxfV4jfNJkzoSezxmTKejEiqCivC/w+tMhTb/6ilInrixdRfg/R2begGW0XwxLwIqg5KupY+vUB1uaWoGkw9px0ZpzfQ26kioBTpolZVYS/dD9maV7DxyNFoBnYRuZgGzoTTT95Mp1368f4i3KJmHAdeoK9TWuUgBdByVFRx7+/zGfjrlKsFp0Lhndh7oyB4PMFujQhcH+7GM/mpeBzN3whIgYjpSe2zJGYR4rwbHgHX8G3RE76OUZil+Pn+csLcG94G0vmSKwDJrV5nRLwIqi4vX4+/LqAj9YWYrXozBzTg6mjuhEfYyOlcxRlbXAjSoQWs7YSX+FmQANdR9N00HTQddAtaNEJ6NEJaNGd0Cwt35nLrHLg2fgeRpdBWPuOxUjthRafdsJkOu/+9bhX/536dxdgGzkH29AZ4Pfh/PR5tKh4Iide3y6T7yTgRdDYsq+c11fsofyoizGD0rkyuw+dYmRrPNF8Zv1R6j/4PaqmvHknRMSgR3dGi0smcsyP0TulNXmK+5t/gyWCyAtuQo9qfDKdtddoDPsA3Ktfw7PhbXwFG9Fjk1BHS4ia9T+n7aNvTRLwIuAqjrr4xyd72LS3HHtSNP/z4+EM6NE50GWJdqT8Pry7v0SPTcQXdQ7Q8oXglM+Dc/kTqPpqombNbwhrZYJpNnxUJsrnQTmPouqPYtYdQdVXoeqr8BXtxPnFi0TPvr/hiv8U/OUH8O1fj23E7FOG+3f0qHiipt6ON2897tWv4SvLxzZ0JpaMrBa37Uw1K+Dz8/OZP38+VVVVJCQksHDhQjIzM086bunSpTz33HMopdA0jVdeeYXk5OTWrlmECKUUn31bxFtf7AMFl0/uzUXndsNiyGSlcOP55l08W5YCULgMtJhEjJSe6CmZGCk9Mez90YxTb8SilInr8xcwS/OJnHo7lowBLXp+756vcH3xIt7cz7ANmnLK49wb3oGIGGxDpjf7sa29R2NkDMBfuAVLnzEtqutsNSvgFyxYwNy5c8nJyWHx4sU8+OCDvPbaayccs23bNp5++mleffVVUlJSqKmpwWaTX69F47w+k78v383qrQ7O6ZXIT6f1J7lTVKDLEgHgK8rFs+UjrP0nYuk3jiing+qCXfjLCvAVbARAT7ATOfEGjPS+jT6GZ8M7+PK/IeL8q7D2HNniGix9x2Ls+xr3+rex9BiOHpt0cp3Fe/Af3Ipt9JVotpaN4tKj4tH7T2hxXWeryUuliooKcnNzmTVrFgCzZs0iNzeXysrKE47729/+xg033EBKSgoAcXFxRETIBBRxsqO1bh7957es3upg1thM7rpiqIR7mFKuWlxfvIjeKY2Isddgsfcn4bxLiMq+hdir/kjsdc8SOeU2lM9D/fu/x/XVooax5t/j2bUSz+YPsWZdgHVw86+sv0/TNCLH/wyUiWv1ayilTqxTKTzr30aL6oTtnAvPuL3trckreIfDQVpaGobR0CdmGAapqak4HA4SE/8zPTwvL4+uXbtyzTXXUF9fz9SpU/nFL37RojvFSUlnfuMhJSXujM/tqDpim/cePMLv/r6RWqeX+346ivFDuzR90vd0xDafrVBts1KK0nefRzlryLjufiLs/+nO/U+b46BLGuawMVR+8TrV3yxDHdpC8sxbiO41jPr8LdR8+SpRvYaRPucXjY47b7aUOKom/5jKT14lunwbsQPHHf9Wfd4maov3kDRtHp3sbdPt3Bavc6vdZPX7/ezevZtXXnkFj8fDz3/+czIyMpgzZ06zH6OiohbTVE0f+AMpKXFhN3yuI7Z57Y5iXvloF/HRVn51zQi6p7WsDR2xzWcrlNvs3bUK16612EZfSbUl5fhaLKds84iriMoYjnvlXyn+5yNYep+H7+BW9IQMjIk3U15Rf9Y1qcyJ6MkrKVv2EvVxvdAiY1HKpP6Tv6PFJePuel6bvB5n+jrrunbaC+Mmu2jsdjslJSX4/X6gIchLS0ux20+cgZWRkcH06dOx2WzExsZy4YUXsnXr1hYXLEJPrdPLP1bs4YUPculpj+eB686le1poXpWK5jGPFuNa8zpGRha2oc3vVrGk9yP6sv/FNmwWvv0b0AwbUTPuRrO1ThefphtETrwe5arFve4NAHz5GzHLDxAx8tI2XVagLTRZbVJSEllZWSxZsoScnByWLFlCVlbWCd0z0NA3v3LlSnJycvD5fKxdu5Zp06a1WeEi+DndPj755iDL1h/E5faRPaILV1/YV0bJhDll+nB+9hcwLEROnnfaYYmN0Sw2IkZfjrXfeLBYG70hejaM5B7YhkzHs2Uplt7n4fnmXfSEjHYfAdMamvXf0UMPPcT8+fN59tlniY+PZ+HChQDMmzePO++8k8GDB3PxxRezfft2Zs6cia7rjB8/nssvv7xNixfByeP18/mmIj78+gC1Ti/D+yZz6YRedE1tn8kdov2oY2PMW3Jl69m4GLMsn8gpt6HHnvkyz3pC+hmf2xTbyDl487/BufxJ8HmInHo7Wgfca0BTP7xdHEDSB998wdhmUym+3HKY978q4EiNm0GZnbl0Ym96ZbTO1nnB2Oa2Fsxt9hXvwbXiaZTpx9p3LNYBkzASuzZ6rFIKs/IQvrx1DSNe+o8nctKNjR4bLG32FeXi/PBR9ORMoi9d0KZLC7RVH3zH6lASQetonYeXl+SyPb+SPl06MW/WQJmNGsI8uZ/jXrMILS4ZS1IPvLmf492+Aj21F9YBk7D2Pg8sEZgVB/Dt/wZv/gbU0RLQNIzuQ4gYe02gm9AkS5eBRF50B0Zitw67absEvDhr2/dX8NKSXJweP9dO68/kYRkd9gdCnJ7y+3CveR3vzs8xug0mKvsWtIgYTFcNvr1r8O5aiXvVK7jX/AMtKq5hTRhNx8jIwjJkBpbMEU1O8Q8m1syWT5oKJhLw4oz5/CbvrtzPsvWFdEmO4d4fD6JrivSzhyrTWY1rxdP4i/dgGzoT27mXH++X1iPjsA2ehvWcizBL8/DuWoVy1WAZPhtL5oh2W1xLnEgCXpyRkiP1/GXxDgqKa5g8vAtXZfchwnoWk0xEUPOXF+D8+EmUq4bI7Fuw9jm/0eM0TcNI64OR1qedKxSNkYAXLfb19mJeW74bi65x26WDGdk/JdAliTbkLfgW16fPo0XGEp3za4zkzECXJJpJAl40m9Pt4+/Ld7N2Rwn9unZi3iWDSOoUGeiyRBvy7PgE95rX0ZN7EjXtv9CjOwW6JNECEvCiWfKKjvKX93dQWe2WDa/DgFImnvVvN0z26TGcyAtvQbPI4oEdjQS8OC3TVCxde4D3vsync1wE868ZQZ+uchUXypTfi+uLl/DlrcM6MJuIsT/pkJN8hAS8OI2Koy5e/jCXXYVVjM5K5afT+hMdeepNF0THp9x1OJc/id+xG9voK7ENnSFDXjswCXhxEp/f5OP1hXywpgANjRtmZjFucLr8oAcR5fdhVpdgVhVjHi1Bj0/G0n3YGW0irZRCHS3Gd3gn3u2fYFaXnnakjOg4JODFCXYUVPL68j0UV9Yzol8KV1/YRzbjCALm0RK8u7/EX3kQs6oYVVPWsM/o91kjsfQYjrXP+RhdB6Hpjf94K6VQNWX4Du/Ef3gn/sO7UPVVAGhxyUTN/GW77hsq2o4EvACgstrFvz7bxze7SklNiOKuK4YypHfrrtInWkYphbNgG/Vfvoe/cAtoOnpnO0ZSN/Teo9ET7A1/4lMbNoPOW4s3fyO+fV+jRcRi6TUKPbEbqu4IZl0lqray4WNdJfh9AGhRnTAysjAyBmDJyEKLT5Xf1EKIBLzgi01FvPHZPkylmDOhJzPO647VIpOWmkuZfjxbl2FWHkJ56sFdj/I4UZ56lKcePS4F66ALsfYd06yRKMrnwbvva7zbV1BbeQgtMg7biEuwDsxGj05o9BxLl4FYugwkYtxP8R/ahnffOrx714DPA5qBFpOAfmwjay1zJHp8CoZ9AHqCXQI9hEnAhzGlFB+sKeC9L/MZ1LNh4+uUBOmOaQnlceL85Bn8h7ajxSWjRcSg2aLRO6WBLRrNFonfsQf3l3/Dvf4trP0nYBt0IXrcfyaHNfSBl+Bz7MLv2IX/4C0m0ioAABytSURBVHaUuxY9sRsps27DmTq02X3rmmHB0mM4lh7DUT43yl2PFtVJRsGEKQn4MKWU4q3P81i2vpAxg9K54eIBGBICLWLWlONc9jhm1WEiJlyHLWtyo8cppfCX7MO7fQXebcvxbvsYS/dhGBlZ+Ev24XfsRjmPAqBFxWN0PQdr1iQM+wDiUuNxneHSuZolQsauhzkJ+DBkmoq/L9/Nys2HyR7RhblT+6HLr+kt4i/dj/Pjx1F+L1Ezfoml66BTHqtpGpb0vljS+2LWVuLd+TnenV/gO7AJLaYzRpcsDPsALPb+aJ1ktJJoPRLwYcbnN3lpSS7rd5Zy8Zge/GhiLwmUFvLu34Dr8xfQojsRPes+jM5dmn2uHptIxLmXYRt+CcpVgxaTKP/+os1IwIcRj9fPc+9tZ0teBVdM7s2M83sEuqQORSmFZ8tHeNa/iZ7au2FtljNc21yz2NBaeS9RIX5IAj4MKKXYeeAI767aT/7haq6d1p8Lhjf/qlM08O78As/6N7H0Gk3k5J+f0aQiIdqTBHwIM03FN7tL+WhdIQeKa+gUY+PmnEGMzkoLdGkdjr90P+41r2N0G9yw8JYmN6RF8JOAD0Eer5+vtjlYtr6QsioXaYnRXDdjAGMGpcn49jNgumpwrngaLboTURfcLOEuOgwJ+BCzYVcpr6/YQ3Wdh14Z8Vx5QV+G902WpX3PkDJNXJ8+j3JVEz37N7L1nOhQJOBDRHW9h0XL9/DNrlJ6pMfxi5xB9OuWICM0zpJn47/xF+0gYuL1GCmZgS5HiBaRgA8BG3aV8vePd+N0+/jRxF7MOL+7TFpqBb4Dm/Fs+qBh9umASYEuR4gWk4DvwKrrPby+fA8bjl2133hxFl1TpAuhNZjVpTg/fwE9qQcR464NdDlCnBEJ+A5q3XYHT7yxiXqXXLW3NuVz41zxNGgaUVNvl+GQosOSgO9gPF4/b36+j8++LaJ7Wiz//ePhctXeivzlBbg+fwHziIOo6Xehx6c0fZIQQUoCvgMpKqvl+fd3UFRWx5xJvZlxbjesFrlqbw3K9OPZ/CGejYvRouKImnEPlm6DA12WEGdFAr4DUErxxebD/OvTvUTZDO6+cijZ52VSdoarDIoTmVXFOL94AbN0P5be5xE57loZDilCggR8kKt1enll6U427S3nnJ6J3DhrIJ1ipE+4NSil8OZ+invtm2Cxyj6kIuRIwAex3IJKXlqSS029l6uy+zD13G6yrO8ZUkqhasvxlx/ALD9w7GMBylndsPzAxBvQYzoHukwhWpUEfBDy+kzeXZXHx+sPYk+K5r8uH0qP9LhAl9XhmDXl+Aq34Cvcgr80D9x1Dd/QdPTOXTC6DcHSdRCW3ufLhDARkiTgg0xReR0vvL+Dg6W1XDCiC1de0IcIq6wf0xzK9OMvzcN/YDO+wq2YRw4BoMWnYe05Cj05EyO5B3piVxn6KMKCBHyQUErx2bdFvPn5PiJtBndePoRhfZIDXVbQUj43ZmUR/opCzIqDmJUH8VcUgtcFmoFh70dE/6uxdB+GnpAe6HKFCAgJ+CBQWe3ib8t2sX1/JYN7JXHDxVlyI7URpteNZ/sneHd+gVlVBEo1fMMaiZ7YFWvfsRgZA7B0PQfNFh3YYoUIAhLwAWQqxarNh3nz832YSnHN1H5kj+gi/cE/oDxOPLmfcnDHCvx1R9HT+mAbPhs9qRtGUne0uGRZwleIRjQr4PPz85k/fz5VVVUkJCSwcOFCMjMzGz12//79XHrppcydO5f77ruvNWsNKaVVTv62dCe7CqvI6tGZ62YMICUhKtBlBRXTVYN323I8Oz4Bj5OoXsNg0Aws9v6BLk2IDqFZAb9gwQLmzp1LTk4Oixcv5sEHH+S111476Ti/38+CBQuYMmVKqxcaKkxT8enGQ7yzKg9D17huxgAmDLHLVfsxylOP7+A2fAc24Sv4FnweLJkjsQ2fRfrAITK5S4gWaDLgKyoqyM3N5ZVXXgFg1qxZPPLII1RWVpKYmHjCsS+88AKTJ0+mvr6e+vr6tqm4AyuurOevH+5kX9FRhvZO4tpp/UmMjwx0WQFn1pThO7AZ34HN+A/vAuVHi4zD2vt8rEOmYXSW/WOFOBNNBrzD4SAtLQ3DaBiqZxgGqampOByOEwJ+165drF69mtdee41nn3227SrugEyl+PSbQ7yzMg+rRWferIGcPygt7K/ald+Ha+VL+PatBUBPsGMbMg2jx3CM1N5osjqmEGelVW6yer1eHnjgAf7whz8c/4/gTCQlnfn6HykpwTkRqLiijsf/tYkd+ys4d2Aat18xrNWu2oO1zc1h+jyUvvMUvn0bSRh7KbFDsrElZTR5Xkdu85mSNoeHtmhzkwFvt9spKSnB7/djGAZ+v5/S0lLsdvvxY8rKyigsLOSmm24CoLq6GqUUtbW1PPLII80upqKiFtNULW5ESkpc0PXNmkqxclMRb36eh67DDTOzGDc4Hb/bS1mZ96wfPxjb3FzK68L58RP4D+8iYsJ1+LMmc9QEmmhPR27zmZI2h4czbbOua6e9MG4y4JOSksjKymLJkiXk5OSwZMkSsrKyTuieycjIYN26dcf//tRTT1FfXx+2o2hKjtTz2rLd7DxwhEE9E7l+xgDpaz9Geeqp/+gxzNI8Ii+Yh7Xv2ECXJETIalYXzUMPPcT8+fN59tlniY+PZ+HChQDMmzePO++8k8GDZd1saNiMY+naAyxdW4jF0Pjp9P5MGpoR9n3t3zFdNTiX/j/MykNEXngr1l7nBrokIUKappRqeZ9IG+nIXTSb95XzjxV7KD/q4vyBaVxxQR86x0W02fMFQ5tbwqyvwvnh/2FWlxA19Q4s3Ye2+DE6Wptbg7Q5PASsi0acXlmVk39+spfN+8qxJ0Xz3z8eTlYPWXb2+5Tpw7n0T5g15URNvwdLl4GBLkmIsCABfxZWbi7iH5/sRdc0rrigN1NHdcNiyNC+H/Lu+Ayz8iCRU++QcBeiHUnAnwGf3+SNT/fx6beH5CZqE0xnNe6N/8boeg6WzBGBLkeIsCIB30K1Ti/PvbednQeOMG10N66Y3Addl5uop+LZ8A54PUSMmSs3m4VoZxLwLVBUVsuT72zlSI2bGy/OYtxge9MnhTF/WQHeXauwDr4Io3PTk5iEEK1LAr6ZNu8t5y8f7CDSanDf3BH07tIp0CUFNaUUrjWL0KLiiBiZE+hyhAhLEvDN8Pm3h1i0fA890uO447IhbTr8MVT49n2NWbKPiInXy+YbQgSIBHwTduRXsmjFHob2SeaWnEHYZH/UJimvC/e6N9FTemLtPyHQ5QgRtmRM32mUHKnn+cXb6ZIcw02zB0q4N5Nn0xJUfRWRY6+RnZaECCD56TsFp9vHU+9sA+COy4YQaZNfdprDrC7Fs3UZlr5jMdL6BLocIcKaBHwjTKV4aUkuxRX1/GLOObKVXjMprxvXV4vAsBAx+opAlyNE2JPL0ka8vzqfTXvL+fGUvgzMTGz6hDCmlMIsP4B310q8+9aC10nE+Vejx8hyDUIEmgT8D3yzq5T3vypg/GA7U0Z2DXQ5QUu56/Du+xrvrlWYFYVg2LD0OhfrgIkY6f0CXZ4QAgn4ExwsreXlD3fSKyOea6f1l5mXP6B8HnwHt+LLW4fvwGbwe9GTehAx/qdYe5+HFhET6BKFEN8jAX+MaSr+8v4OIiMMbrt0MFaL3J6AhpUg/UW5ePetw1ewEbwutKh4rP0nYh0wASM5M9AlCiFOQQL+mA27SjlcXscv5pwjE5loWCTMu2053l0rUa4asEVh6Xku1j7nYWRkoekyZFSIYCcBT8PV+/tf5dMlOYaR/VMCXU5AmbUVeLZ8hHfXSvD7sGSOwNJvLJZuQ9AMa6DLE0K0gAQ88M3uUhwV9dySMwg9TPvdzapiPFs+xLt3DSiw9B1LxLCZ6AmyoJoQHVXYB7ypFB98VUBGcgyj+qcGupx2p5TCve4NvNs+Bt2CNWsytiEz0OOSA12aEOIshX3Ab9xdRlF5HTfPHhSW67p7vnkX79ZlWAdMxDbqMvRoWSVTiFAR1gFvKsX7q/OxJ0Vz7oDwu3r3bFuOZ9MHWAdMJGLC9TIsVIgQE9ZjAb89dvV+ydjMsLt69+5dg/vrf2DJHEnE+Osk3IUIQWEb8KZqGDmTnhjN6Ky0QJfTrnyFm3F98TJGRhaR2Tej6WH7NhAipIXtT/amPWUcKgu/q3df8V6cK55FT+5O1EV3ollsgS5JCNFGwjLgTaVYvLqAtMRoRg8Mn753f8VBnMv+jB6bSNT0u9FsskqmEKEsLAN+055yDpXVcsnYHhhh0j2hXLU4lz2GZo0k6uL/Ro+KD3RJQog2Fh7p9j1KKT74Kp/UzlGcNzB8+t5dX/0dVV9N1LQ70WOTAl2OEKIdhF3Al1U5KSytZcrIrmFz9e7NW48vbx22kTmyOJgQYSQ8Eu578oqqAejfPTw2pDDrq3CtfhU9pRe2YRcHuhwhRDsKu4Dfd/gokTaDLsmhv3a5UgrXyr+Cz0PUBfNkBUghwkzYBXxe0VF62uPDYmikd/cq/Ae3EnHelbJomBBhKKwC3uXxcbC0lt5dQn+9FbO6DPfX/8TIyMI66MJAlyOECICwCvh8Rw1KQZ8uoT1EUCkT18qXAIicdCOaFlYvsxDimLD6yc8rOgpAr4zQvoL3bluO37GbyLHXyLK/QoSxsAv49MRoYqNCd2cib/43uDe8jaXHcCz9xge6HCFEAIXNcsFKKfIOVzO0T2hO8lFK4dn0AZ5v3kVP7UXEpBtkhUghwlzYBHzpESe1Tm9I3mBVPg+ulS/jy1uHpc8YIideL4uICSGaF/D5+fnMnz+fqqoqEhISWLhwIZmZmScc88wzz7B06VIMw8BisXD33XczYcKEtqj5jOw71v/eJ8T63826IziXP4lZVoBt9OXYhl4sV+5CCKCZAb9gwQLmzp1LTk4Oixcv5sEHH+S111474ZghQ4Zwww03EBUVxa5du/jJT37C6tWriYyMbJPCWyrvcDWRNoOMEJrg5C/Lx/nxEyiPk8iL7sCaOSLQJQkhgkiTAV9RUUFubi6vvPIKALNmzeKRRx6hsrKSxMTE48d9/2q9f//+KKWoqqoiPT29Dcpuubyio/TK6LgTnJRSqPoqzPIC/GUFFFcfoj5/K1pUPNE5v8FI6hboEoUQQabJgHc4HKSlpWEYDdPcDcMgNTUVh8NxQsB/33vvvUf37t2DJtydbh+HymqZNSYz0KW0mL9kH+5v38csL0A5G9bRQdNQSV2w9h2D7dzLZelfIUSjWv0m6/r163niiSf461//2uJzk5Jiz/h5U1LiTvm9LXvLUApGDEw/7XHBxld7hEMrnkTTdGL6jCDC3ouI9N7Y0jLRbcHR9dXeOtLr11qkzeGhLdrcZMDb7XZKSkrw+/0YhoHf76e0tBS7/eS1TTZt2sR///d/8+yzz9KrV68WF1NRUYtpqhafl5ISR1lZzSm/vzG3GIDkWOtpjwsmSpk4P3oc0+0k+kcPoXXuggfwABz1kpIS2WHa0lqaep1DkbQ5PJxpm3VdO+2FcZMTnZKSksjKymLJkiUALFmyhKysrJO6Z7Zu3crdd9/Nk08+yaBBg1pcaFvKKzqKPSmamMiOM8HJu+1j/Ie2EzHmxxiduwS6HCFEB9SsmawPPfQQixYtYtq0aSxatIiHH34YgHnz5rFt2zYAHn74YVwuFw8++CA5OTnk5OSwe/futqu8mZRS7D9c3aHGv/vLC3CvfxtL5gisWRcEuhwhRAfVrD743r1789Zbb5309RdffPH45++8807rVdWKSo5NcOrTQQJeeV04P30eLSqeyIkyG1UIceZCfi2a7xYY653RMUaauNe8jjpaQuQFN6FFnvlNZyGECIuAj4qwYO8AE5y8eevx7v4S2/BZWDKyAl2OEKKDC/mA3/fdBKcg7+owa8pwffkKempvbCNzAl2OECIEhPRiY063j6KyOkb0Swl0KaeklIlv/ze4178FCqKyb0HTQ/plEUK0k5BOkv2OahQE7Q1W36HtuNe/jVlegN65K5HTf44eH7z/GQkhOpaQDvj/7OAUXDdY/aX7cW94G39RLlpsEpGT52HpMwZND/keMyFEOwrxgK8mIzmG6CCZ4KTMhr1SfXvXoEXGETH2GqxZk9GM4KhPCBFaQjbgTaXYf/hoUPW/ezZ9gG/vGmxDZ2IbfgmaLSrQJQkhQljIBnxJZT11Ll/QzGD1Fe/F8+17WPqcj230FTKBSQjR5kK20zff0bC0bjD0vyt3Ha7PnkeLTSZy/M8k3IUQ7SJkA77AUYPNqpORFNgJTkopXF++iqo7QlT2zdItI4RoN6Eb8CU1dE+LC/gOTr49q/HtX49t1KUYaX0CWosQIryEZMCbpqKwpIbMtMBuGmBWFeP6ahGGfQC2oRcHtBYhRPgJyZusjoo6PF6TTHvbBrx3/wa821egJ/fA0nUwRkZ/NEsEAMrvxfnZc2BYiMy+Wca4CyHaXUgGfEFxw84oPdLb7garN28drs/+ghbTGX9ZPt7tK8CwYKT3x9JtMGZVMWb5ASIvuhM9pnOb1SGEEKcSsgEfYTWwJ0a3yeN7963F9flfMNL6EjXjHtB0/I7d+A5tx39oG+61/wLAOjAba+aINqlBCCGaEqIBX033tNg2ucHq3fc1rs9fwEjvR9T0u9GsDZtfW7oNxtJtMPBjzNoK/GUFWLoPafXnF0KI5gq5gPebJgdLapk0rPX3MfXuXYPrixcx0vsfC/eIRo/TY5PQY5Na/fmFEKIlQi7gHeX1eHytf4PVu+crXCtfwrAPIGraXacMdyGECBYhF/Df3WDNTG+9gPfuWoVr1SsYGQOImn7X8ZEyQggRzEIw4KuJsBmktcINVqUUno3v4fl2MUbXc4i66A4JdyFEhxFyAX+guIYeaXFnvUWf8vtwffkKvj1fYek3nsiJ18lOS0KIDiWkEstvmhSW1nLB8LO7warcdThXPI3/8E5soy7FNny2LBAmhOhwQirgD5fX4/WZZ9X/btZW4PzoMcyqYiInz8Pab1wrViiEEO0npAK+4NgSwZn2M5vB6i8/gHPZn1E+N1Ezf4mly8DWLE8IIdpVaAV8cQ2RNoPUzi1fktd3YBPOT59Hi4wlevZvMBJbfxy9EEK0p5AL+Mz0lt1gVUrh3b4c99f/Qk/uQdT0u9CjE9qwSiGEaB8hE/A+v8nB0louHNn8K29l+nGveR1v7mdYMkcSecFNMoFJCBEyQibgD5fX4fObZDZzBUnlceL89Fn8B7c1bII9+nI0TZb0FUKEjpAJ+OMzWJuxRIFZU45z2eOYVQ4iJlyHLWtyG1cnhBDtL6QCPirCQmrC6W+wms5q6hf/tmGkzIx7sHQd1E4VCiFE+wqdgHdUk5ke1+SEJPea11GuGqLnPIiR3KOdqhNCiPYXEp3OXp/JobJaejQxwclXsAlf3jpsw2dLuAshQl5IBPyB4mp8fnXaGazKXYdr9avoiV2xDZMNsIUQoS8kAj7vUBVw+iWC3eveQDmPEjnpRjQjZHqmhBDilEIi4PcerCI6wkLKKW6w+opy8e5ahXXwdIyUnu1cnRBCBEZIBHzeoSp6nOIGq/K6ca16Ba1TGhGjLg1AdUIIERjNCvj8/Hyuuuoqpk2bxlVXXUVBQcFJx/j9fh5++GGmTJnC1KlTeeutt1q71kZ5fWbDCJpTjH93b3gHVVNG5MQb0Cy2dqlJCCGCQbMCfsGCBcydO5ePP/6YuXPn8uCDD550zAcffEBhYSHLly/njTfe4KmnnuLQoUOtXvAPFZXX4vMrejYyg9Vfsg/v9hVYB2Zjsfdv81qEECKYNBnwFRUV5ObmMmvWLABmzZpFbm4ulZWVJxy3dOlSrrjiCnRdJzExkSlTprBs2bK2qfp7DhVXMcRaSE/3Trx7VuPZtRJP7ud4dnyCa+XLaDGdiRh9RZvXIYQQwabJ4SQOh4O0tDQMwwDAMAxSU1NxOBwkJiaecFxGRsbxv9vtdoqLi1tUTFJSbIuOBzivczlD476AdV/g+sH3NIuNtMv/h+guqS1+3I4gJaX1NhbvKKTN4UHa3DqCarxgRUUtpqladI61xwhSfvE0lRW1oOugG6A1fNQsEdRZI6grq2mjigMnJSWOshBs1+lIm8ODtLn5dF077YVxkwFvt9spKSnB7/djGAZ+v5/S0lLsdvtJxx0+fJghQ4YAJ1/RtyVroh3dH15vCCGEaEqTffBJSUlkZWWxZMkSAJYsWUJWVtYJ3TMA06dP56233sI0TSorK/nkk0+YNm1a21QthBCiSc0aRfPQQw+xaNEipk2bxqJFi3j44YcBmDdvHtu2bQMgJyeHrl27ctFFF3HllVdy22230a1bt7arXAghxGlpSqmWdXq3oTPpgwfpswsX0ubwIG1uvqb64ENiJqsQQoiTScALIUSIkoAXQogQFVTj4HX99LsxtdW5HZW0OTxIm8PDmbS5qXOC6iarEEKI1iNdNEIIEaIk4IUQIkRJwAshRIiSgBdCiBAlAS+EECFKAl4IIUKUBLwQQoQoCXghhAhREvBCCBGiOnzA5+fnc9VVVzFt2jSuuuoqCgoKAl1Sq1q4cCHZ2dn079+fPXv2HP96KLf7yJEjzJs3j2nTpnHJJZdw++23H9/kPZTbfeuttzJ79mzmzJnD3Llz2blzJxDabQZ4+umnT3h/h3J7s7OzmT59Ojk5OeTk5PDll18Cbdhm1cFde+216r333lNKKfXee++pa6+9NsAVta4NGzaow4cPqwsuuEDt3r37+NdDud1HjhxRa9euPf73P/7xj+pXv/qVUiq0211dXX388xUrVqg5c+YopUK7zdu3b1c33nijmjx58vH3dyi394c/x99pqzZ36IAvLy9XI0eOVD6fTymllM/nUyNHjlQVFRUBrqz1ff+NEU7tVkqpZcuWqZ/97Gdh1e5///vf6tJLLw3pNrvdbnXllVeqwsLC4+/vUG6vUo0HfFu2OahWk2wph8NBWloahmEAYBgGqampOByOk/aMDSXh1G7TNPnnP/9JdnZ2WLT717/+NV999RVKKV566aWQbvMTTzzB7NmzT9jaM5Tb+517770XpRQjR47knnvuadM2d/g+eBHaHnnkEaKjo/nJT34S6FLaxe9+9zu++OIL7r77bh599NFAl9NmNm3axLZt25g7d26gS2lXr7/+Ou+//z7vvPMOSin+93//t02fr0MHvN1up6SkBL/fD4Df76e0tBS73R7gytpWuLR74cKFHDhwgMcffxxd18Om3QBz5sxh3bp1pKenh2SbN2zYwP79+7nwwgvJzs6muLiYG2+8kcLCwpBs73e+a4fNZmPu3Ll8++23bfq+7tABn5SURFZWFkuWLAFgyZIlZGVlhcyvcqcSDu3+85//zPbt23nmmWew2WxAaLe7rq4Oh8Nx/O+fffYZnTp1Ctk233TTTaxevZrPPvuMzz77jPT0dF5++WVmzpwZku0FqK+vp6amYWNtpRRLly4lKyurTV/jDr/hR15eHvPnz6e6upr4+HgWLlxIr169Al1Wq/ntb3/L8uXLKS8vp3PnziQkJPDhhx+GdLv37t3LrFmzyMzMJDIyEoCuXbvyzDPPhGy7y8vLufXWW3E6nei6TqdOnbjvvvsYNGhQyLb5+7Kzs3n++efp169fyLb34MGD3HHHHfj9fkzTpHfv3vzmN78hNTW1zdrc4QNeCCFE4zp0F40QQohTk4AXQogQJQEvhBAhSgJeCCFClAS8EEKEKAl4IYQIURLwQggRoiTghRAiRP1/ZnV0V+9HfpsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.linspace(0, N_EPOCHS,N_EPOCHS)\n",
    "\n",
    "plt.plot(x,t_f1)\n",
    "plt.plot(x,v_f1)\n",
    "plt.title(\"F1 score\")\n",
    "plt.legend([\"Train F1\", \"Valid F1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 678.936 |  Test F1 score: 54.77%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, precision_recall_fscore_support\n",
    "model.load_state_dict(torch.load('tut5-model_conll.pt'))\n",
    "\n",
    "test_loss, test_f1, preds, labels = evaluate(model, test_iterator, TAG_PAD_IDX, full_report=False)\n",
    "print(f'Test Loss: {test_loss:.3f} |  Test F1 score: {test_f1*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict =  [item for sublist in preds for item in sublist]\n",
    "true =  [item for sublist in labels for item in sublist]\n",
    "confusion = confusion_matrix(true, predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrice de confusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>O</th>\n",
       "      <th>LOC</th>\n",
       "      <th>PER</th>\n",
       "      <th>ORG</th>\n",
       "      <th>MISC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>38435</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>19</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LOC</th>\n",
       "      <td>406</td>\n",
       "      <td>1441</td>\n",
       "      <td>1</td>\n",
       "      <td>56</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PER</th>\n",
       "      <td>2317</td>\n",
       "      <td>29</td>\n",
       "      <td>402</td>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ORG</th>\n",
       "      <td>1076</td>\n",
       "      <td>265</td>\n",
       "      <td>21</td>\n",
       "      <td>1069</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MISC</th>\n",
       "      <td>352</td>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>531</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          O   LOC  PER   ORG  MISC\n",
       "O     38435    21    3    19    42\n",
       "LOC     406  1441    1    56    20\n",
       "PER    2317    29  402    21     3\n",
       "ORG    1076   265   21  1069    63\n",
       "MISC    352    11    8    13   531"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_df =pd.DataFrame(confusion)\n",
    "\n",
    "confusion_df.columns=[i for i in TAG.vocab.itos][1:]\n",
    "s = pd.Series([i for i in TAG.vocab.itos][1:])\n",
    "confusion_df = confusion_df.set_index([s])\n",
    "\n",
    "confusion_df['LOC'] = confusion_df['B-LOC'] + confusion_df['I-LOC']\n",
    "confusion_df['PER'] = confusion_df['B-PER'] + confusion_df['I-PER']\n",
    "confusion_df['ORG'] = confusion_df['B-ORG'] + confusion_df['I-ORG']\n",
    "confusion_df['MISC'] = confusion_df['B-MISC'] + confusion_df['I-MISC']\n",
    "\n",
    "\n",
    "confusion_df = confusion_df.drop(columns=[ i for i in TAG.vocab.itos if i != 'O' and i != '<pad>'])\n",
    "\n",
    "confusion_df.loc['LOC'] = confusion_df.loc['B-LOC'] + confusion_df.loc['I-LOC']\n",
    "confusion_df.loc['PER'] = confusion_df.loc['B-PER'] + confusion_df.loc['I-PER']\n",
    "confusion_df.loc['ORG'] = confusion_df.loc['B-ORG'] + confusion_df.loc['I-ORG']\n",
    "confusion_df.loc['MISC'] = confusion_df.loc['B-MISC'] + confusion_df.loc['I-MISC']\n",
    "\n",
    "confusion_df = confusion_df.drop([i for i in TAG.vocab.itos if i != 'O' and i != '<pad>'])\n",
    "\n",
    "confusion_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Precision-Recall-F1Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1-score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>0.902527</td>\n",
       "      <td>0.997793</td>\n",
       "      <td>0.947772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LOC</th>\n",
       "      <td>0.815507</td>\n",
       "      <td>0.748960</td>\n",
       "      <td>0.780818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PER</th>\n",
       "      <td>0.924138</td>\n",
       "      <td>0.145022</td>\n",
       "      <td>0.250702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ORG</th>\n",
       "      <td>0.907470</td>\n",
       "      <td>0.428629</td>\n",
       "      <td>0.582244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MISC</th>\n",
       "      <td>0.805766</td>\n",
       "      <td>0.580328</td>\n",
       "      <td>0.674714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Precision    Recall  F1-score\n",
       "O      0.902527  0.997793  0.947772\n",
       "LOC    0.815507  0.748960  0.780818\n",
       "PER    0.924138  0.145022  0.250702\n",
       "ORG    0.907470  0.428629  0.582244\n",
       "MISC   0.805766  0.580328  0.674714"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm = confusion_df.to_numpy()\n",
    "\n",
    "TP = np.diag(cm)\n",
    "FP = np.sum(cm, axis=0) - TP\n",
    "FN = np.sum(cm, axis=1) - TP\n",
    "\n",
    "num_classes = 4\n",
    "TN = []\n",
    "for i in range(num_classes):\n",
    "    temp = np.delete(cm, i, 0)    \n",
    "    temp = np.delete(temp, i, 1) \n",
    "    TN.append(sum(sum(temp)))\n",
    "    \n",
    "precision = TP/(TP+FP)\n",
    "recall = TP/(TP+FN)\n",
    "f1 = (2*precision*recall)/(precision+recall)\n",
    "\n",
    "prf_df =pd.DataFrame()\n",
    "prf_df['Precision'] = precision\n",
    "prf_df['Recall'] = recall\n",
    "prf_df['F1-score'] = f1\n",
    "\n",
    "s = pd.Series([i for i in confusion_df.index])\n",
    "prf_df = prf_df.set_index([s])\n",
    "\n",
    "prf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tag_sentence(model, device, sentence, text_field, tag_field):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    if isinstance(sentence, str):\n",
    "        nlp = spacy.load('en')\n",
    "        tokens = [token.text for token in nlp(sentence)]\n",
    "    else:\n",
    "        tokens = [token for token in sentence]\n",
    "\n",
    "    if text_field.lower:\n",
    "        tokens = [t.lower() for t in tokens]\n",
    "        \n",
    "    max_word_len = max([len(token) for token in tokens])\n",
    "    numericalized_tokens = [text_field.vocab.stoi[t] for t in tokens]\n",
    "    unk_idx = text_field.vocab.stoi[text_field.unk_token]  \n",
    "    unks = [t for t, n in zip(tokens, numericalized_tokens) if n == unk_idx]\n",
    "    \n",
    "    token_tensor = torch.LongTensor(numericalized_tokens)    \n",
    "    token_tensor = token_tensor.unsqueeze(-1).to(device)\n",
    "    predictions, _ = model(token_tensor)\n",
    "    predicted_tags = [tag_field.vocab.itos[t] for t in predictions[0]]\n",
    "    \n",
    "    return tokens, predicted_tags, unks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Japan', 'then', 'laid', 'siege', 'to', 'the', 'Syrian', 'penalty', 'area', 'for', 'most', 'of', 'the', 'game', 'but', 'rarely', 'breached', 'the', 'Syrian', 'defence', '.']\n",
      "Pred. Tag\t\t\t\tActual Tag\t\t\t\tCorrect?\t\t\t\tToken\n",
      "\n",
      "B-LOC \t\t\t\t B-LOC \t\t\t\t ✔ \t\t\t\t\t Japan\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t then\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t laid\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t siege\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t to\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t the\n",
      "B-MISC \t\t\t\t B-MISC \t\t\t\t ✔ \t\t\t\t\t Syrian\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t penalty\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t area\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t for\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t most\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t of\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t the\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t game\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t but\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t rarely\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t breached\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t the\n",
      "B-MISC \t\t\t\t B-MISC \t\t\t\t ✔ \t\t\t\t\t Syrian\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t defence\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t .\n"
     ]
    }
   ],
   "source": [
    "example_index = 15\n",
    "\n",
    "sentence = vars(test_data.examples[example_index])['text']\n",
    "actual_tags = vars(test_data.examples[example_index])['tag']\n",
    "\n",
    "print(sentence)\n",
    "tokens, pred_tags, unks = tag_sentence(model, \n",
    "                                       device, \n",
    "                                       sentence, \n",
    "                                       TEXT, \n",
    "                                       TAG\n",
    "                                       )\n",
    "\n",
    "print(\"Pred. Tag\\t\\t\\t\\tActual Tag\\t\\t\\t\\tCorrect?\\t\\t\\t\\tToken\\n\")\n",
    "for token, pred_tag, actual_tag in zip(tokens, pred_tags, actual_tags):\n",
    "    correct = '✔' if pred_tag == actual_tag else '✘'\n",
    "    space = 5 if pred_tag == 'O'else 4\n",
    "    space1 = 5 if actual_tag == 'O'else 4   \n",
    "    print(pred_tag,\"\\t\"*space, actual_tag, \"\\t\"*space1, correct,\"\\t\"*5, token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "xQRKbOpwo96O",
    "outputId": "fcd3716d-d256-45e9-fd75-b85d2010ef42"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n",
      "Pred. Tag\tToken\n",
      "\n",
      "O \t\t The\n",
      "O \t\t will\n",
      "O \t\t deliver\n",
      "O \t\t a\n",
      "O \t\t speech\n",
      "O \t\t about\n",
      "O \t\t the\n",
      "O \t\t conflict\n",
      "O \t\t in\n",
      "B-LOC \t North\n",
      "I-LOC \t Korea\n",
      "O \t\t tomorrow\n",
      "O \t\t in\n",
      "B-LOC \t New\n",
      "I-LOC \t York\n",
      "O \t\t with\n",
      "O \t\t my\n",
      "O \t\t friend\n",
      "O \t\t Mary\n",
      "O \t\t Kate\n",
      "O \t\t .\n"
     ]
    }
   ],
   "source": [
    "sentence = 'The will deliver a speech about the conflict in North Korea tomorrow in New York with my friend Mary Kate.'\n",
    "tokens, tags, unks = tag_sentence(model, \n",
    "                                  device, \n",
    "                                  sentence, \n",
    "                                  TEXT, \n",
    "                                  TAG\n",
    "                                )\n",
    "\n",
    "print(unks)\n",
    "print(\"Pred. Tag\\tToken\\n\")\n",
    "\n",
    "\n",
    "for token, tag in zip(tokens, tags):\n",
    "    space = 2 if tag == 'O'else 1\n",
    "    print(tag, \"\\t\"*space, token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "vj6lBUSJmp2K"
   ],
   "name": "6 - Transformer Model for Indonesian NER with PyTorch on GPU",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}