training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import AdamW\n",
    "from torch.nn import BCEWithLogitsLoss, MSELoss\n",
    "from transformers import RobertaTokenizerFast, RobertaConfig, RobertaModel, RobertaForSequenceClassification, get_linear_schedule_with_warmup\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from model import Model\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at microsoft/codebert-base were not used when initializing RobertaForSequenceClassification: ['pooler.dense.bias', 'pooler.dense.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at microsoft/codebert-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RobertaForSequenceClassification(\n",
       "  (roberta): RobertaModel(\n",
       "    (embeddings): RobertaEmbeddings(\n",
       "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
       "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
       "      (token_type_embeddings): Embedding(1, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): RobertaEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x RobertaLayer(\n",
       "          (attention): RobertaAttention(\n",
       "            (self): RobertaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): RobertaSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): RobertaIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): RobertaOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (classifier): RobertaClassificationHead(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "    (out_proj): Linear(in_features=768, out_features=1, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "tokenizer = RobertaTokenizerFast.from_pretrained(\"microsoft/codebert-base\", do_lower_case=True)\n",
    "\n",
    "config = RobertaConfig.from_pretrained(\"microsoft/codebert-base\")\n",
    "config.num_labels = 1\n",
    "model = RobertaForSequenceClassification.from_pretrained(\"microsoft/codebert-base\", config=config)\n",
    "model.to(device)\n",
    "#model = Model(model)\n",
    "#model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = [\"<type>\", \"<specifier>\", \"<decl>\", \"<parameter_list>\", \"<decl_stmt>\", \"<comment>\", \"<directive>\", \"<parameter_list>\", \"<expr_stmt>\", \"<control>\", \"<argument_list>\", \"<parameter>\", \"<index>\",\n",
    "          \"<argument>\", \"<default>\", \"<call>\", \"<then>\", \"<label>\", \"<ternary>\", \"<condition>\", \"<init>\"]\n",
    "tokenizer.add_tokens(tokens)\n",
    "model.resize_token_embeddings(len(tokenizer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dataset.optimize_data as rename\n",
    "tokenizer.add_tokens(rename.keywords)\n",
    "tokenizer.add_tokens(rename.standard_lib_funcs)\n",
    "model.resize_token_embeddings(len(tokenizer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "high_freq_words = ['avctx','goto','uintt','sizeof','uint8t','avlog','codec','intt','\\\\n\\\\t','AVLOGERROR','\\\\n','pkt','dst','errp','opaque','arg1','channels','bufsize','ifdef','AVERROR','fprintf','linesize','samples','sps','d\\\\n','cpuenv','stderr','opcode','cpuR','arg2','opts','AVERRORINVALIDDATA','stride','getbits','t1','t0','avpkt','cpuT0','geterrno','memcpy','codecid','privdata','vdev','memset','pts','strcmp','ENOMEM','packet','AVCodecContext','0x','mbx','tcgtempfree','cpuir','localerr','gfree','streams','timebase','samplerate','TCGv','getbits1','idx','EINVAL','arg3','mbtype','modrm','movq','mv','amlappend','spapr','mby','tempblock','tb','regs','mm0','pixfmt','FIXME','qemu','errorsetg','bitrate','s\\\\n','scan8','DPRINTF','codectype','BlockDriverState','targetulong','irq','TCGvi','errorreport','0xff','extradata','acb','streamindex','avfreep','0xf','hci','mm1','blk','AVFrame','predictor','ramsize','cpumodel','avfree','pcstart','picttype','datasize','nbsectors','qxl','AVFormatContext','0x00','alac','CPUState','sizet','invalid','strlen','snprintf','REGa','framesize','chr','avmallocz','qiov','errno','t2','RRRS','mm2','xfer','tcgtempnew','aviorb','s\\'','nbstreams','memidx','pfl','kernelfilename','nbsamples','entries','AVNOPTSVALUE','tcgtempfreei','cpuA0','qdev','AVPacket','saddr','src2','tcgtempnewi','sectornum','dinfo','retval','RRRT','putbits','hdr','tmpz','csbase','insn','i8','gmalloc0','mm3','targetphysaddrt','efault','qemuirq','refcount','INTMAX','EIO','AVLOGDEBUG','AVRL','src1','tmp1','unlockuser','RRRR','tcggenmovii','unimplemented','Rn','AVLOGWARNING','fmt','motionval','cnt','FFMIN','mm4','xen9pdev','fdt','tcgconsti','xendev','exceptionindex','AVStream','timestamp','islit','avmalloc','MemoryRegion','putuser','cputmp2i','0x80','currentpictureptr','op1','mbheight','cputobe','dstFormat','illegalop','mbwidth']\n",
    "tokenizer.add_tokens(high_freq_words)\n",
    "model.resize_token_embeddings(len(tokenizer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_length = 512\n",
    "\n",
    "def convert_examples_to_features(js):\n",
    "    code=' '.join(js['func'].split())\n",
    "    inputs = tokenizer(\n",
    "        code, \n",
    "        max_length=max_length, \n",
    "        return_tensors=\"pt\",\n",
    "        padding=\"max_length\",\n",
    "        truncation=True\n",
    "    )\n",
    "    return inputs, js['target']\n",
    "\n",
    "def convert_examples_to_features2(js):\n",
    "    code=' '.join(js['func'].split())\n",
    "    code_tokens = tokenizer.tokenize(code, return_tensors=\"pt\", max_length=510, padding=\"max_length\")\n",
    "    token_count = len(code_tokens)\n",
    "    code_tokens = code_tokens[:510]\n",
    "    source_tokens =[tokenizer.cls_token]+code_tokens+[tokenizer.sep_token]\n",
    "    source_ids =  tokenizer.convert_tokens_to_ids(source_tokens)\n",
    "    source_ids = torch.tensor(source_ids)\n",
    "    return {\"input_ids\": source_ids, \"attention_mask\": source_ids.ne(1)}, js['target'], token_count\n",
    "\n",
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, file_path, skip):\n",
    "        self.examples = []\n",
    "        with open(file_path) as f:\n",
    "            for line in f:\n",
    "                js=json.loads(line.strip())\n",
    "                input, target, token_count = convert_examples_to_features2(js)\n",
    "                if token_count > 800 and skip:\n",
    "                    continue\n",
    "                self.examples.append([input, target])\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.examples)\n",
    "\n",
    "    def __getitem__(self, i):       \n",
    "        return self.examples[i][0], self.examples[i][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17954"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset = CustomDataset('dataset/train_renamed.jsonl', True)\n",
    "len(train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 4\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, pin_memory=True, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2732"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dataset = CustomDataset('dataset/test_renamed.jsonl', False)\n",
    "batch_size = 4\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=batch_size, pin_memory=True, shuffle=False)\n",
    "len(test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "adam_epsilon = 1e-8\n",
    "learning_rate = 2e-5\n",
    "max_grad_norm = 1.0\n",
    "weight_decay = 0.00\n",
    "no_decay = ['bias', 'LayerNorm.weight']\n",
    "epochs = 5\n",
    "max_steps = epochs * len(train_dataloader)\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': weight_decay},\n",
    "    {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "]\n",
    "optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate, eps=adam_epsilon)\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=max_steps * 0.1, num_training_steps=max_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "update_loss_steps = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0: loss 0.6835, acc 0.5620: 100%|██████████| 4489/4489 [14:48<00:00,  5.05it/s]\n",
      "100%|██████████| 683/683 [00:41<00:00, 16.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: loss 0.6570, acc 0.5864\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: loss 0.6401, acc 0.6272: 100%|██████████| 4489/4489 [14:40<00:00,  5.10it/s]\n",
      "100%|██████████| 683/683 [00:41<00:00, 16.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: loss 0.6335, acc 0.6138\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: loss 0.5760, acc 0.7049: 100%|██████████| 4489/4489 [14:33<00:00,  5.14it/s]\n",
      "100%|██████████| 683/683 [00:40<00:00, 16.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: loss 0.6515, acc 0.6464\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: loss 0.5407, acc 0.7788: 100%|██████████| 4489/4489 [14:31<00:00,  5.15it/s]\n",
      "100%|██████████| 683/683 [00:40<00:00, 16.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: loss 1.0327, acc 0.6409\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4: loss 0.5320, acc 0.8322: 100%|██████████| 4489/4489 [14:31<00:00,  5.15it/s]\n",
      "100%|██████████| 683/683 [00:40<00:00, 16.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set: loss 1.3143, acc 0.6376\n"
     ]
    }
   ],
   "source": [
    "stats = []\n",
    "for i, epoch in enumerate(range(epochs)):\n",
    "    #TRAINING\n",
    "    bar = tqdm(train_dataloader,total=len(train_dataloader))\n",
    "    bar.set_description(f\"Epoch {epoch}\")\n",
    "    model.train()\n",
    "    total_loss = 0\n",
    "    val_results = []\n",
    "    val_labels = []\n",
    "    for step, batch in enumerate(bar):\n",
    "        inputs = batch[0]\n",
    "        inputs = {key: value.squeeze(1).to(device) for key, value in inputs.items()}\n",
    "        labels = batch[1].to(device)\n",
    "        ids, mask = inputs['input_ids'], inputs['attention_mask']\n",
    "        #outputs = model(**inputs)\n",
    "        outputs = model(ids, mask)\n",
    "        #result = outputs[:,0]\n",
    "        result = outputs.logits[:,0]\n",
    "        loss = BCEWithLogitsLoss()(result, labels.float())\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        scheduler.step()\n",
    "        total_loss += loss.mean().item()\n",
    "        val_results.append(result.cpu().detach().numpy())\n",
    "        val_labels.append(labels.cpu().numpy())\n",
    "        if step % update_loss_steps == 99:\n",
    "            results=np.concatenate(val_results,0)\n",
    "            labels=np.concatenate(val_labels,0)\n",
    "            preds=results>0.5\n",
    "            eval_acc=np.mean(labels==preds)\n",
    "            loss = total_loss / step\n",
    "            stats.append({'steps': (epoch*len(train_dataloader))+step, 'loss': loss, 'acc': eval_acc})\n",
    "            bar.set_description(f\"Epoch {epoch}: loss {loss:.4f}, acc {eval_acc:.4f}\")\n",
    "    \n",
    "    #EVAL\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        bar = tqdm(test_dataloader,total=len(test_dataloader))\n",
    "        total_loss = 0\n",
    "        val_results = []\n",
    "        val_labels = []\n",
    "        for step, batch in enumerate(bar):\n",
    "            inputs = batch[0]\n",
    "            inputs = {key: value.squeeze(1).to(device) for key, value in inputs.items()}\n",
    "            labels = batch[1].to(device)\n",
    "            ids, mask = inputs['input_ids'], inputs['attention_mask']\n",
    "            #outputs = model(**inputs)\n",
    "            outputs = model(ids, mask)\n",
    "            #result = outputs[:,0]\n",
    "            result = outputs.logits[:,0]\n",
    "            loss = BCEWithLogitsLoss()(result, labels.float())\n",
    "            total_loss += loss.mean().item()\n",
    "            val_results.append(result.cpu().detach().numpy())\n",
    "            val_labels.append(labels.cpu().numpy())\n",
    "    results=np.concatenate(val_results,0)\n",
    "    labels=np.concatenate(val_labels,0)\n",
    "    preds=results>0.5\n",
    "    eval_acc=np.mean(labels==preds)\n",
    "    loss = total_loss / step\n",
    "    time.sleep(1)\n",
    "    print(f\"Test set: loss {loss:.4f}, acc {eval_acc:.4f}\")\n",
    "    model.save_pretrained('Limit800/epoch'+str(epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAFTCAYAAAAdqYl1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABYzUlEQVR4nO3dd3hUVf7H8fc3IYWEQAihhRYUkF5DUcResIGuv1XBuiro7qpr3VXXtrrF3bWvFRFRLNhWRcW+qFgQEkFKUHoJNRBqgNTz++MOMMRAApnkziSf1/PMw8wtcz8T4OY75557jjnnEBEREZFDE+V3ABEREZFIpmJKREREpApUTImIiIhUgYopERERkSpQMSUiIiJSBSqmRERERKpAxZSIlMvMjjOzHL9zSGjp71Uk9FRMiVQDM1tmZif5nUNERKqfiikRiThmVs/vDAcrEjOLSOWomBKpQWYWZ2aPmNnqwOMRM4sLrEs1s/fNbLOZ5ZnZVDOLCqz7k5mtMrNtZvazmZ24n/c/w8xmmtlWM1tpZvcErUs3M2dml5rZCjPbYGZ/Dlpf38zGm9kmM8sG+lfwWR4NHGOrmWWZ2ZCgddFmdruZLQ5kzjKzNoF13czs08BnXGdmtweWjzezvwa9xz6XowKtfX8ys9lAvpnVM7Nbg46RbWbnlMk4yszmB63va2a3mNlbZbZ7zMwe3c/nXGZmtwX232Rmz5tZfND6M81sVuDv7Vsz63mgzOW8f+egn8fPZnZe0LrxZvZ0YP02M/vSzNoFrT/KzGaY2ZbAn0cFrUsJZF0dyP1OmePeZGbrzWyNmf2mvM8uIpXknNNDDz1C/ACWASeVs/xeYBrQDGgKfAvcF1j3D+BpICbwGAIYcASwEkgLbJcOHL6f4x4H9MD7otQTWAecHbSfA54F6gO9gAKgS2D9/cBUIAVoA8wFcg7wGS8CmgD1gJuAtUB8YN0twJxAdgscqwmQBKwJbB8feD0wsM944K9lPktOmZ/prEC2+oFlvwbSAp/3fCAfaBm0bhVeUWhAB6Ad0DKwXXJgu3rAeqDfAf4u5waOmwJ8szsn0Cew70AgGrg0sH3c/jKXee/EwN/tbwI5+gAbgK5BP5NtwDFAHPAo8HVgXQqwCbg4sO+IwOsmgfUfAK8BjfH+PR0b9HMtxvu3GAOcDuwAGvv9/0YPPSL14XsAPfSojQ/2X0wtBk4Pen0qsCzw/F7gXaBDmX06BH5hnwTEHGSOR4CHA8/T8Yqp1kHrpwMXBJ4vAYYGrRvNAYqpco61CegVeP4zMLycbUYAM/ez/3gqLqYuryDDrN3HBT4G/rCf7T4ERgWenwlkV/B3eXXQ69OBxYHnTxEohoPW/xxUuBwwM14BOLXMsmeAu4N+JhOD1jUASvCKs4uB6WX2/Q64DK9gLC2vQAr8XHcC9YKWrQcGVcf/BT30qAsPXeYTqVlpwPKg18sDywD+DSwCPjGzJWZ2K4BzbhFwPXAPsN7MJppZGuUws4FmNsXMcs1sC3A1kFpms7VBz3fg/YLenW1lmWz7ZWY3By6hbTGzzUCjoGO1wSscy9rf8soKzoeZXRJ0iW0z0L0SGQBewGtZI/DnhIM4bvDfWTvgpt3HD2RoE7T+F5nLaAcMLLP/hUCL8vZ3zm0H8gLvX/bf0u5srQIZ8pxzm/Zz3I3OueKg18H/DkTkIKmYEqlZq/F+ge7WNrAM59w259xNzrnDgGHAjbv7RjnnXnHOHR3Y1wH/3M/7vwJMAto45xrhXTa0SmZbg/dLODhbuQL9o/4InIfX+pEMbAk61krg8HJ2XQkctp+3zQcSgl63KGcbF5ShHd4ly2vwLm0l412OqygDwDtATzPrjtcy9fJ+ttut7M9lddAx/uacSw56JDjnXi0vczlWAl+W2b+Bc+635R3bzBrgXd5bzS//Le3OtirwvilmllzB5xKREFAxJVJ9YswsPuhRD3gVuMPMmppZKnAX8BLs6cjcwcwMrzApAUrN7AgzO8G8juq78C7RlO7nmEl4LRK7zGwAMPIg8r4O3GZmjc2sNXDtAbZNwut3kwvUM7O7gIZB68cC95lZR/P0NLMmwPtASzO73rzO+ElmNjCwzyzg9EDH6RZ4rXEHkohXqOQCBDpRdy+T4WYz6xfI0GF3523n3C7gTbzic7pzbkUFx/q9mbU2sxTgz3h9kcAr5q4OtAiamSWadxNAUgXvt9v7QCczu9jMYgKP/mbWJWib083saDOLBe4DpjnnVgKTA/uONK8z/vlAV+B959wavEuZTwb+PmPM7JhKZhKRg6RiSqT6TMYrfHY/7gH+CmQCs/E6aP8QWAbQEfgM2I7X9+VJ59wUvI7H9+N1TF6L13n9tv0c83fAvWa2Da9Qe/0g8v4F7zLRUuATDnzp62PgI2BBYJ9d7Hs566HAsT8BtgLP4XXA3gacDJwV+CwLgeMD+0wAfsTrZ/QJewuWcjnnsoEH8X5W6/A63n8TtP4N4G94BdM2vNaolKC3eCGwT0WX+Ai8xyd4/coWE/g7c85lAqOAx/H6jC3C67NUKYGfxynABXgtTWvxWh3jyhz7brzLe/0IXJ50zm3Ea1W7CdiI11J4pnNuQ2C/i4Ei4Ce8PlHXVzaXiBwcc+5ALdAiIrWTmbXFKzRaOOe2HmC7ZcCVzrnPaipb0LHH43XCv6Omjy0ilaeWKRGpc8wbv+tGvDvl9ltIiYhUhkbkFZE6xcwS8S4LLgeG+hxHRGoBXeYTERERqQJd5hMRERGpAhVTIiIiIlWgYkpERESkClRMiYiIiFSBiikRERGRKlAxJSIiIlIFKqZEREREqkDFlFSZmS0zs5P8ziEiYmZfmNmmwMTgIjVCxZSIiNQKZpYODAEcMKwGj6vZROo4FVNSLcwszsweMbPVgccju78pmlmqmb1vZpvNLM/MpgbmSsPM/mRmq8xsm5n9bGYn+vtJRCSCXAJMA8YDl+5eaGZtzOy/ZpZrZhvN7PGgdaPMbH7gnJNtZn0Dy52ZdQjabryZ/TXw/Dgzywmcr9YCz5tZ48B5LTfQMva+mbUO2j/FzJ4PnA83mdk7geVzzeysoO1izGyDmfWprh+ShJ6KKakufwYGAb2BXsAA4I7AupuAHKAp0By4HXBmdgRwDdDfOZcEnAosq9HUIhLJLgFeDjxONbPmZhYNvI83F2M60AqYCGBmvwbuCezXEK81a2Mlj9UCSAHaAaPxfp8+H3jdFtgJPB60/QQgAegGNAMeDix/EbgoaLvTgTXOuZmVzCFhQE2TUl0uBK51zq0HMLO/AM8AdwJFQEugnXNuETA1sE0JEAd0NbNc59wyP4KLSOQxs6PxCpnXnXMbzGwxMBKvpSoNuMU5VxzY/OvAn1cC/3LOzQi8XnQQhywF7nbOFQRe7wTeCsrzN2BK4HlL4DSgiXNuU2CTLwN/vgTcaWYNnXNbgYvxCi+JIGqZkuqShvdNcLflgWUA/8Y7aX1iZkvM7FaAQGF1Pd43xfVmNtHM0hARqdilwCfOuQ2B168ElrUBlgcVUsHaAIsP8Xi5zrldu1+YWYKZPWNmy81sK/AVkBxoGWsD5AUVUns451YD3wDnmlkyXtH18iFmEp+omJLqshrvW+JubQPLcM5tc87d5Jw7DK9Z/cbdfaOcc68453Z/w3TAP2s2tohEGjOrD5wHHGtmawP9mG7A62KwDmi7n07iK4HD9/O2O/Auy+3Wosx6V+b1TcARwEDnXEPgmN3xAsdJCRRL5XkB71Lfr4HvnHOr9rOdhCkVUxIqMWYWv/sBvArcYWZNzSwVuAuvORszO9PMOpiZAVuAEqDUzI4wsxMCHdV34TWbl/rzcUQkgpyNdx7pitdPszfQBa8LwdnAGuB+M0sMnKMGB/YbC9xsZv3M08HMdn8JnAWMNLNoMxsKHFtBhiS8c9ZmM0sB7t69wjm3BvgQeDLQUT3GzI4J2vcdoC/wB7w+VBJhVExJqEzGO5HsfsQDmcBsYA7wA/DXwLYdgc+A7cB3wJPOuSl4/aXuBzYAa/E6ad5Wcx9BRCLUpcDzzrkVzrm1ux94HcBHAGcBHYAVeDe/nA/gnHsD+BveJcFteEVNSuA9/xDYbzNeH9B3KsjwCFAf7/w1DfiozPqL8fqL/gSsx+vSQCDH7v5W7YH/Vv5jS7gw58q2VIqIiEhNMrO7gE7OuYsq3FjCju7mExER8VHgsuAVeK1XEoF0mU9ERMQnZjYKr4P6h865r/zOI4dGl/lEREREqkAtUyIiIiJVoGJKREREpAp864Cemprq0tPT/Tq8iPggKytrg3Ouqd85QkHnMJG65UDnL9+KqfT0dDIzM/06vIj4wMyWV7xVZNA5TKRuOdD5S5f5RERERKpAxZSIiIhIFaiYEhEREamCsBoBvaioiJycHHbt2uV3lGoXHx9P69atiYmJ8TuKiIRIXTmH6fwlsq+wKqZycnJISkoiPT0dM/M7TrVxzrFx40ZycnJo376933FEJETqwjlM5y+RXwqry3y7du2iSZMmtfYktJuZ0aRJk1r/7VWkrqkL5zCdv0R+KayKKaBWn4SC1ZXPKVLX1IX/23XhM4ocjLArpvy0efNmnnzyyYPe7/TTT2fz5s2hDyQichB0DhPxh4qpIPs7ERUXFx9wv8mTJ5OcnFxNqUREKkfnMBF/qJgKcuutt7J48WJ69+5N//79GTJkCMOGDaNr164AnH322fTr149u3boxZsyYPfulp6ezYcMGli1bRpcuXRg1ahTdunXjlFNOYefOnX59HJHqtXUNzBgLpaV+J5EAncNEKsc5x/PfLGVTfmHo3tCPR79+/VxZ2dnZv1hWk5YuXeq6devmnHNuypQpLiEhwS1ZsmTP+o0bNzrnnNuxY4fr1q2b27Bhg3POuXbt2rnc3Fy3dOlSFx0d7WbOnOmcc+7Xv/61mzBhwn6P5/fnFTlkpaXOvTLCufuaOZe3rNK7AZnOp3NOqB91/Rzm92cVOVQ7C4vdH179wbX70/vuqS8WVXq/A52/KhwawczGAWcC651z3ctZPxy4DygFioHrnXNfV7XI+8t788hevbWqb7OPrmkNufusbpXefsCAAfvc+vvYY4/x9ttvA7By5UoWLlxIkyZN9tmnffv29O7dG4B+/fqxbNmyKucWCTvZ78DPH8DJ90Ljdn6nCUs6h4mEn43bC7hqQhaZyzdx8ymduOqYw0LyvpUZZ2o88Djw4n7Wfw5Mcs45M+sJvA50Dkk6nyUmJu55/sUXX/DZZ5/x3XffkZCQwHHHHVfurcFxcXF7nkdHR6uJXGqfHXkw+RZo2RsG/d7vNHIAOoeJ7LVw3TYuf2EG67cW8MTIvpzRs2XI3rvCYso595WZpR9g/fagl4mAC0Gug/r2FSpJSUls27at3HVbtmyhcePGJCQk8NNPPzFt2rQaTicSJj6+HXZugovfhuiwGvc3rOgcJhI+pi7M5Xcv/UBcTDSvXXUkvdskh/T9Q3ImNLNzgH8AzYAzQvGefmjSpAmDBw+me/fu1K9fn+bNm+9ZN3ToUJ5++mm6dOnCEUccwaBBg3xMKuKThZ/Bj6/CMbdAix5+p5EydA4T+aWXpi3n7knz6NisAc9d1p9WyfVDfgzz+lRVsJHXMvV+eX2mymx3DHCXc+6k/awfDYwGaNu2bb/ly5fvs37+/Pl06dKlcslrgbr2eSXCFWyDJ4+EmPpw9ddQL67ifcowsyznXEY1pKtxGRkZLjMzc59lden/dF36rBKZSkodf/tgPuO+WcoJnZvx2Ig+NIg79DakA52/Qjo0gnPuK+AwM0vdz/oxzrkM51xG06ZNQ3loEalun98HW3Jg2OOHVEiJiNSU7QXFjH4xk3HfLOU3g9N59pKMKhVSFanyO5tZB2BxoAN6XyAO2FjlZCISPlZMg+ljYOBV0Hag32lERPZr1eadXDF+BgvXb+e+s7tz8aDqv+O4MkMjvAocB6SaWQ5wNxAD4Jx7GjgXuMTMioCdwPmuMtcORSQyFO2CSddCozZwwp1+pxER2a9ZKzdz5QuZFBSV8Pxl/TmmU81cBavM3XwjKlj/T+CfIUskIuHlq3/DhgVw0X8hroHfaUREyjV5zhpueG0WzRrG8eqogXRsnlRjx9Z9zSKyf2tmwzePQK+R0OFEv9OIiPyCc44nv1jMvz/+mX7tGjPm4n40aVCz/TpVTIlI+UqKYdI1UL8xnPo3v9OIiPxCQXEJt/93Lm/9kMPw3mn889yexMdE13gOTXQcZH8zrlfGI488wo4dO0KcSMRH3z0Oa36E0x+AhBS/00gl6BwmdUlefiEXj53OWz/kcMNJnXjk/N6+FFKgYmofOhGJBGxYBF/8AzqfCV2H+51GKknnMKkrFq3fzjlPfsOsnM08NqIPfzipI2bmWx5d5gty6623snjxYnr37s3JJ59Ms2bNeP311ykoKOCcc87hL3/5C/n5+Zx33nnk5ORQUlLCnXfeybp161i9ejXHH388qampTJkyxe+PInLoSkvhvesgOg7OeBB8PEHJwdE5TOqCbxZt4LcvZRFbL4qJowfRt21jvyOpmAp2//33M3fuXGbNmsUnn3zCm2++yfTp03HOMWzYML766ityc3NJS0vjgw8+ALz5rho1asRDDz3ElClTSE0td7xSkciR9Tws/8YbnDOphd9p5CDoHCa13avTV3DnO3M5rGkiz13anzYpCX5HAsK5mPrwVlg7J7Tv2aIHnHZ/pTb95JNP+OSTT+jTpw8A27dvZ+HChQwZMoSbbrqJP/3pT5x55pkMGTIktBlF/LRlFXx6N7Q/Fvpc5HeayKZzmEjIlJQ67v9wPs9OXcqxnZry+Mg+JMXH+B1rj/AtpnzmnOO2227jqquu+sW6H374gcmTJ3PHHXdw4oknctddd/mQUCTEnIP3bwBXAmc9qst7EU7nMKkt8guK+cPEWXw2fx2XHtmOO8/sSr3o8OryHb7FVCW/fYVSUlIS27ZtA+DUU0/lzjvv5MILL6RBgwasWrWKmJgYiouLSUlJ4aKLLiI5OZmxY8fus6+ayCVizX0LFn4Mp/4DUtr7nSby6RwmUmVrtuzkivGZ/LR2K38Z1o1Lj0r3O1K5wreY8kGTJk0YPHgw3bt357TTTmPkyJEceeSRADRo0ICXXnqJRYsWccsttxAVFUVMTAxPPfUUAKNHj2bo0KGkpaWp86ZEnvwN8OEfoVWGN/+eRCSdw6Q2mZOzhStemMGOwhKeu6w/xx/RzO9I+2V+TaOXkZHhMjMz91k2f/58unTp4kseP9S1zyth7K0rYd47cPVUaFZ9/ybNLMs5l1FtB6hBdf0cVpc+q9S8j+au4frXZtEkMY5xl/XniBY1NzXM/hzo/KWWKZG67uePYM4bcNxt1VpIiYhUxDnH018u4Z8f/UTvNsk8e0kGTZNqdmqYQ6FiSqQu27XV63TerCscfaPfaUSkDissLuWOd+bwemYOZ/ZsyQO/7uXbiOYHS8WUSF322d2wfS2c/xLUi/U7jYjUUZt3FHL1S1lMW5LHdSd25PoTOxIVFTl3FIddMeWc83VI+JriV181kT2WfQ2Z4+DIa6B1P7/T1Bp14Rym85eE0pLc7VzxQiarNu3kkfN7c3afVn5HOmhhNVBDfHw8GzdurPX/UZ1zbNy4kfj4eL+jSF1VtBMmXQuN0+H42/1OU2vUhXOYzl8SSt8t3sg5T37Llp1FvDJqYEQWUhBmLVOtW7cmJyeH3Nxcv6NUu/j4eFq3bu13DKmrvrgf8pbAJZMgNtHvNCFhZkOBR4FoYKxz7v4y69sCLwDJgW1udc5NDqy7DbgCKAGuc859fCgZ6so5TOcvCYXXZ6zk9rfnkJ6ayLhL+9O2SXhMDXMowqqYiomJoX17DRYoUq1Wz4Rv/wN9L4HDjvU7TUiYWTTwBHAykAPMMLNJzrnsoM3uAF53zj1lZl2ByUB64PkFQDcgDfjMzDo550oONofOYSIVKy11/PPjn3jmyyUM6ZjK4yP70qh++EwNcyjC6jKfiFSzkiJ491pIbAon3+d3mlAaACxyzi1xzhUCE4HhZbZxQMPA80bA6sDz4cBE51yBc24psCjwfiISYjsKi/nty1k88+USLhzYlnGX9Y/4QgrCrGVKRKrZN4/CujlwwStQP9nvNKHUClgZ9DoHGFhmm3uAT8zsWiAROClo32ll9o3MjhsiYWzd1l1c8cIMsldv5a4zu/Kbwem15mYNtUyJ1BW5C+DLf0K3c6DzGX6n8cMIYLxzrjVwOjDBzA7qHGhmo80s08wya3u/KJFQmrtqC8Mf/4alufmMvTSDy49uX2sKKVAxJVI3lJbCpGu8zuan/cvvNNVhFdAm6HXrwLJgVwCvAzjnvgPigdRK7ktgvzHOuQznXEbTpk1DFF2kdvs0ex2/fvo7ogze/O1RnNC5ud+RQk7FlEhdMGMsrPweTv0HNAjfyUKrYAbQ0czam1ksXofySWW2WQGcCGBmXfCKqdzAdheYWZyZtQc6AtNrLLlILeWc49mvljB6Qiadmjfgnd8PpkvLhhXvGIEqLKbMbJyZrTezuftZf6GZzTazOWb2rZn1Cn1METlkm1fAZ/fA4SdCrwv8TlMtnHPFwDXAx8B8vLv25pnZvWY2LLDZTcAoM/sReBW4zHnm4bVYZQMfAb8/lDv5RGSvopJSbn97Dn+bPJ/Turdg4ugjadaw9o5NVpkO6OOBx4EX97N+KXCsc26TmZ0GjOGXHT9FxA/OwXvXe8/PegRqUR+FsgJjRk0us+yuoOfZwOD97Ps34G/VGlCkjtiyo4jfvpzFt4s3cs3xHbjx5E4RNTXMoaiwmHLOfWVm6QdY/23Qy2l4/Q1EJBz8OBEWfw6n/RuS2/qdRkRquWUb8rn8hRmszNvBg7/uxbn96kZJEOqhEa4APtzfSjMbDYwGaNtWJ3aRarV9PXx0K7QZBP2v9DuNiNRy3y/ZyFUvZWHAy1cOYkD7FL8j1ZiQdUA3s+Pxiqk/7W8b3QkjUoM+/CMU7YBh/4Eo3WsiItXnrawcLnrue1ISY3n7d4PrVCEFIWqZMrOewFjgNOfcxlC8p4hUwfz3Yd7bcMKd0LST32lEpJYqLXU8+OnPPDFlMUcd3oSnLuxHo4TIH9H8YFW5mApMHvpf4GLn3IKqRxKRKtm5GT64CZr3gMF/8DuNiNRSOwtLuOmNWUyes5YRA9pw7/DuxETXzVbwCospM3sVOA5INbMc4G4gBsA59zRwF9AEeDIwmmmxcy6jugKLSAU+vRPyc2HkaxBd974hikj1W791F6NezGT2qi3ccUYXrqhlI5ofrMrczTeigvVXAurdKhIOlnwBP7wIg6+HtN4+hxGR2ih79VaufGEGm3cWMebiDE7uWvtGND9YmuhYpLYozIf3/gAph8Nxt/qdRkRqoc/nr+O6V2eSFB/D61cdSfdWjfyOFBZUTInUFlP+DpuWwWWTIaa+32lEpBZxzjHum2X87YNsuqU1YuylGTSvxSOaHywVUyK1QU4mTHsSMi6H9HIH+RYROSRFJaXcM2keL3+/gqHdWvDQ+b1IiFX5EEw/DZFIV1wI714DSS3hpL/4nUZEapEtO4u45pUfmLpwA1cfezh/PPWIWj81zKFQMSUS6b5+CHLnw8jXIb52zsguIjVvxcYdXP7CDJZtyOdf/9eT8zLa+B0pbKmYEolk6+fDVw9Aj19Dp1P9TiMitUTmsjxGT8iipNQx4YqBHHl4E78jhTUVUyKRqrTEu7wX3xCG3u93GhGpJd6fvZobX/uRVo3r89ylGRzWtIHfkcKeiimRSPX9M7AqE859DhJT/U4jIrXAt4s3cP3EWfRpm8yzl2SQnBDrd6SIoGJKJBLlLYX/3QedhkL3c/1OIyK1wOLc7Vw9IYv2qYmMvbQ/jeprBoXKqpuT6IhEMue8wTktGs54COrwFA4iEhp5+YVcPn4GMdFRjLtMhdTBUsuUSKSZ+RIs/RLOfBgatfI7jYhEuILiEq6akMmaLbuYOHoQbVIS/I4UcdQyJRJJtq6Bj/8M7Y6Gvpf5nUZEIpxzjj+9OZsZyzbx4K970bdtY78jRSQVUyKRwjmYfDOUFMCwxyBK/31FpGoe+3wR78xazc2ndOKsXml+x4lYOhuLRIrsd+Gn9+G426DJ4X6nEZEI9+6sVTz82QLO7dua3x/fwe84EU3FlEgk2JHntUq17AVHXuN3GhGJcJnL8rjljdkMbJ/CP37VA9ONLFWiDugikeDjP8POTXDx2xCt/7YicuiWb8xn9IQsWjWuzzMX9yO2ntpVqko/QZFwt+gz+PEVGHw9tOjhdxoRiWBbdhTxm/EzKHWOcZf116CcIaJiSiScFWyH926A1E5wzC1+pxGRCFZYXMrVL2WxMm8Hz1zUj/apiX5HqjV0vUAknP3vPtiyEi7/CGLi/U4jIhHKOccd78zhuyUbeei8Xgw8TBMXh5JapkTC1Yrvvfn3BoyGtoP8TiMiEezpL5fwemYO153QgV/1be13nFpHxZRIOCraBZOugUat4cS7/E4jIhFs8pw1/POjnzirVxo3nNzJ7zi1ki7ziYSjqQ/AhgVw0VsQ18DvNCISoWat3MwNr82ib9tk/v1/PTUEQjWpsGXKzMaZ2Xozm7uf9Z3N7DszKzCzm0MfUaSOWTsHvn4Yeo2EDif5nUZEIlTOph1c+UImzRrG8ewlGcTHRPsdqdaqzGW+8cDQA6zPA64DHghFIJE6raQY3r0G6jeGU//mdxoRiVDbdhVxxfhMCopLeP6y/jRpEOd3pFqtwmLKOfcVXsG0v/XrnXMzgKJQBhOpk6Y9AWtmwen/hoQUv9OISAQqLinlmldmsjh3O09f1I8OzZL8jlTrqc+USLjYuBim/B06nwldz/Y7jYhEIOcc97w3jy8X5HL/r3owuEOq35HqhBq9m8/MRptZppll5ubm1uShRcJbaSlMug6i4+D0B0CdREXkEDz/zTJemraCq449jAsGtPU7Tp1Ro8WUc26Mcy7DOZfRtGnTmjy0SHj7YTws/xpO/Ss0bOl3GhGJQJ9lr+O+D7IZ2q0Ffzq1s99x6hSNMyXity2r4NO7of0x0Odiv9OISASau2oL102cSY9WjXj4/N5ERal1uyZV2GfKzF4FjgNSzSwHuBuIAXDOPW1mLYBMoCFQambXA12dc1urK7RIreEcfHAjlBTBWY/p8p6IHLS1W3Zx5QuZJNePYewlGdSP1RAINa3CYso5N6KC9WsBjU0vcijmvgULPoJT/w4p7f1OE9HMbCjwKBANjHXO3V9m/cPA8YGXCUAz51xyYF0JMCewboVzbliNhBapovyCYq54YQbbdhXx5m+PollDzeHpB93NJ+KX/I3w4R+hVT8YeLXfaSKamUUDTwAnAznADDOb5JzL3r2Nc+6GoO2vBfoEvcVO51zvGoorEhIlpY4/TJzF/DVbee7S/nRp2dDvSHWW+kyJ+OWjW2HXVhj2OESpWb6KBgCLnHNLnHOFwERg+AG2HwG8WiPJRKrJ3yfP57P567hnWDeO79zM7zh1moopET8s+BjmvA7H3AzNu/qdpjZoBawMep0TWPYLZtYOaA/8L2hxfGDYlmlmdna1pRQJkQnTlvPc10u57Kh0Ljky3e84dZ4u84nUtF1b4f0boGkXOPpGv9PURRcAbzrnSoKWtXPOrTKzw4D/mdkc59zisjua2WhgNEDbthrDR/zx5YJc7pk0jxM6N+POM/VlLByoZUqkpn12D2xdDcMfh3qxfqepLVYBbYJetw4sK88FlLnE55xbFfhzCfAF+/anCt5OY+WJr35eu43fv/wDnZon8diIPkRrCISwoGJKpCYt+wYyn4NBv4PWGX6nqU1mAB3NrL2ZxeIVTJPKbmRmnYHGwHdByxqbWVzgeSowGMguu6+I39Zv28Xl42eQEBvNuMsyaBCni0vhQn8TIjWlaCdMuhaS28EJf/Y7Ta3inCs2s2uAj/GGRhjnnJtnZvcCmc653YXVBcBE55wL2r0L8IyZleJ9wbw/+C5AkXCwq6iEUS9mkZdfyOtXHUnLRvX9jiRBVEyJ1JQv/wl5i+GSdyE20e80tY5zbjIwucyyu8q8vqec/b4FelRrOJEqKC113Pj6LGbnbObpi/rRo3UjvyNJGbrMJ1ITVs+Cbx7zpos57Di/04hIBHngk5+ZPGctt5/WhVO7tfA7jpRDxZRIdSspgknXQGJTOOWvfqcRkQjyeuZKnvxiMSMHtuXKIZolIVzpMp9Idfv2MVg7B85/Geon+51GRCLEt4s3cPt/5zCkYyp/GdYN09ydYUstUyLVKXcBfPFP6Ho2dDnT7zQiEiEW527n6glZtE9N5IkL+xITrV/X4Ux/OyLVpbTUu3svpj6c/m+/04hIhMjLL+Ty8TOIrRfFuMv60zA+xu9IUgFd5hOpLpnPwcppcPbT0EDzZolIxQqKSxj9YiZrt+zi1dGDaJOS4HckqQQVUyLVYfMKb6Tzw0+AXhf4nUZEIoBzjj+9OZvM5Zt4fGQf+rZt7HckqSRd5hMJNee8ufecgzMfAXUaFZFKePTzhbwzazW3nHoEZ/ZM8zuOHAQVUyKhtuQLWPQZnHAHNG7ndxoRiQDvzFzFI58t5Ny+rfndcYf7HUcOkoopkVCb+iAktYT+V/idREQiQOayPP745mwGtk/hH7/qoSEQIpCKKZFQWvE9LJsKR10L9eL8TiMiYW75xnxGT8iiVeP6PHNxP2Lr6ddyJNLfmkgoTX0Q6qdAv8v8TiIiYW7LjiJ+M34Gpc4x7rL+JCfE+h1JDpGKKZFQWTMbFn4Mg36niYxF5IAKi0u5+qUsVubtYMzFGbRP1TkjkmloBJFQ+fohiGsIA0b5nUREwphzjjvemcN3Szby8Pm9GNA+xe9IUkVqmRIJhQ0LYd470P9Kzb8nIgf09JdLeD0zh+tO7Mg5fVr7HUdCoMJiyszGmdl6M5u7n/VmZo+Z2SIzm21mfUMfUyTMff0I1Iv3LvGJiOzH5Dlr+OdHPzGsVxo3nNTR7zgSIpVpmRoPDD3A+tOAjoHHaOCpqscSiSCbV8DsidDvUmjQ1O80IhKmZq3czA2vzaJfu8b86/96agiEWqTCYso59xWQd4BNhgMvOs80INnMWoYqoEjY++YxwLzhEEREypGzaQdXvpBJs4ZxjLm4H/Ex0X5HkhAKRZ+pVsDKoNc5gWUitd+2dfDDi978e43U90FEfmnrriKuGJ9JQXEJz1/WnyYNNAZdbVOjHdDNbLSZZZpZZm5ubk0eWqR6THsCSovg6Bv8TiIiYai4pJRrXpnJ4tztPH1RPzo0S/I7klSDUBRTq4A2Qa9bB5b9gnNujHMuwzmX0bSp+pZIhNu5CWY8B93OgSaaS0tE9uWc45735vHVglz+enZ3BndI9TuSVJNQFFOTgEsCd/UNArY459aE4H1Fwtv3Y6BwOxx9o99JRCQMjftmGS9NW8FVxx7GBQPa+h1HqlGFg3aa2avAcUCqmeUAdwMxAM65p4HJwOnAImAH8JvqCisSNgq2w/dPQafToEV3v9OISJj5LHsdf/0gm6HdWvCnUzv7HUeqWYXFlHNuRAXrHfD7kCUSiQRZz3uX+Ybc5HcSEQkzc1dt4bqJM+nRqhEPn9+bqCgNgVDbaQR0kYNVtAu+/Q+0Pwba9Pc7jYiEkbVbdnHFCzNIrh/D2EsyqB+rIRDqAhVTIgdr1suwfR0MudnvJCISRvILirnihRnkF5Qw7jf9adYw3u9IUkNUTIkcjJIi+OYRaJXhtUyJiAAlpY4/TJzF/DVb+c/IPnRu0dDvSFKDVEyJHIy5b3nTxxxzM2gqCBEJ+Pvk+Xw2fx33DOvG8Uc08zuO1DAVUyKVVVoKUx+CZt2g46l+pxGRMDFh2nKe+3opvxmcziVHpvsdR3ygYkqksn56Hzb8DENuhCj91xER+HJBLvdMmseJnZtxxxld/Y4jPtFvBJHKcA6mPgAph3kjnotInffz2m38/uUfOKJ5Eo+N6EO0hkCos1RMiVTG4s9hzY/eHHxRutVZpK5bv20Xl4+fQUJsNM9dlkFiXIXDNkotpr99kcr46kFo2Ap6XuB3EhHx2c7CEka9mEVefiGvX3UkLRvV9zuS+EwtUyIVWf4trPgWjroO6sX6nUZEfFRa6rjpjVnMztnMoxf0pkfrRn5HkjCgYkqkIlMfhIRU6HuJ30lExGcPfPIzk+es5c+nd+GUbi38jiNhQsWUyIGsngmLPoMjfwexCX6nEREfvZWVw5NfLGbkwLZccXR7v+NIGFExJXIgUx+CuEbQ/0q/k0gFzGyomf1sZovM7NZy1j9sZrMCjwVmtjlo3aVmtjDwuLRGg0tEmLliE7e9PYejDm/CX4Z1wzRorwRRB3SR/cn9Gea/5412Hq9+EeHMzKKBJ4CTgRxghplNcs5l797GOXdD0PbXAn0Cz1OAu4EMwAFZgX031eBHkDC2busurpqQRfOGcTwxsi8x0WqHkH3pX4TI/nz9MMTUh4G/9TuJVGwAsMg5t8Q5VwhMBIYfYPsRwKuB56cCnzrn8gIF1KfA0GpNKxGjoLiEq1/KYntBMc9ekkHjRN2EIr+kYkqkPJuWwezXod9vILGJ32mkYq2AlUGvcwLLfsHM2gHtgf8d7L5StzjnuOPtucxcsZkHf91LkxfLfqmYEinPN496g3MedY3fSST0LgDedM6VHOyOZjbazDLNLDM3N7caokk4eeHbZbyRlcN1J3bktB4t/Y4jYUzFlEhZ29bCzJeg90homOZ3GqmcVUCboNetA8vKcwF7L/Ed1L7OuTHOuQznXEbTpk2rEFfC3beLNnDfB/M5uWtzrj+xo99xJMypmBIp69v/QGkxDP6D30mk8mYAHc2svZnF4hVMk8puZGadgcbAd0GLPwZOMbPGZtYYOCWwTOqolXk7+P0rP3BYaiIPndeLKM25JxXQ3XwiwXbkQebz0P3/vEmNJSI454rN7Bq8IigaGOecm2dm9wKZzrndhdUFwETnnAvaN8/M7sMryADudc7l1WR+CR/5BcWMejGTklLHs5dkkBQf43ckiQAqpkSCff80FOV7ExpLRHHOTQYml1l2V5nX9+xn33HAuGoLJxHBOcctb/7IgnXbGP+bAaSnJvodSSKELvOJ7FawzSumOp8Jzbv6nUZEatgTUxYxec5abjutC8d0Up84qbxKFVOVGFm4nZl9bmazzewLM2sd+qgi1WzGc7BrCwy50e8kIlLDPs1exwOfLOCcPq24coimipGDU2ExFTSy8GlAV2CEmZX92v4A8KJzridwL/CPUAcVqVZFO+G7J+Cw46FVP7/TiEgNWrR+Gze8NouerRvxj1/10FQxctAq0zJVmZGFu7J3ALwp5awXCW8zX4L89d7UMSJSZ2zZUcSoF7OIj4nmmYv7ER8T7XckiUCVKaYqMzrwj8CvAs/PAZLMTMNGS2QoKfIG6WwzENoN9juNiNSQklLHtRNnkrNpB09f1JeWjer7HUkiVKg6oN8MHGtmM4Fj8Qa8+8Xowho9WMLS7Ndhy0oYcjOoeV+kzvjXRz/x1YJc7h3enYz0FL/jSASrTDFV4ejAzrnVzrlfOef6AH8OLNtc9o00erCEndIS+PohaNEDOp7sdxoRqSHvzlrFM18t4eJB7RgxoK3fcSTCVaaYqnBkYTNLNbPd73UbGq9FIsX8SbBxEQy5Sa1SInXEnJwt/PHN2Qxon8JdZ2kYFKm6Cosp51wxsHtk4fnA67tHFjazYYHNjgN+NrMFQHPgb9WUVyR0nIOvHoQmHaHLsIq3F5GIl7utgNETMkltEMdTF/YlJlrDLUrVVWoE9IpGFnbOvQm8GdpoItVs4Sewbg4MfxKidAePSG1XWFzK717OYtOOQt68+iiaNIjzO5LUEppORuom5+CrB6BRG+h5nt9pRKQG3PPePGYs28R/RvShe6tGfseRWkTtm1I3LfsacqbD4D9AtCYyFantXpq2nFe+X8Fvjzucs3ql+R1HahkVU1I3TX0QEptBn4v8TiIi1ez7JRu5Z9I8jj+iKTefcoTfcaQWUjEldc+qLFgyBY78PcRokD6R2mzV5p387uUfaNskgUdH9CE6SnftSuipmJK6Z+pDEJ8M/a/wO4mIVKOdhSWMfjGTwuJSnr0kg4bxuqQv1UPFlNQt67Lhp/dh4NUQl+R3GhGpJs45/vjWbLLXbOXREb05vGkDvyNJLaZiSuqWrx+CmEQYeJXfSUSkGj3z1RLe+3E1t5x6BCd0bu53HKnlVExJ3ZG3BOa+Bf0vhwTNwyVSW035aT3//OgnzuzZkt8ee7jfcaQOUDEldcfXj0BUDBx5jd9JRKSaLMndznUTZ9KlRUP+9X89MU0TJTVAxZTUDVtXw6xXvKEQklr4nUZEqsHWXUWMejGTmOgoxlzSj4RYjUstNUPFlNQN3/4HXCkMvs7vJCJSDUpKHddPnMXyjTt48sK+tG6c4HckqUNUTEntl78BssZ708Y0Tvc7jYhUg4c+/Zn//bSeu8/qyqDDmvgdR+oYFVNS+017Cop2wtE3+J1ERKrB+7NX88SUxYwY0IaLBrXzO47UQSqmpHbbtQWmPwtdzoKmmkZCpLaZt3oLt7wxm4x2jfnLsO7qcC6+UDEltduMsVCwBYbc5HcSEQmxjdsLGP1iFo3qx/DkRX2JradfaeIP3eogtVfhDvjuSehwEqT19juNiIRQUUkpv3/lBzZsL+CNq4+kWVK835GkDlMxJbXXDy/Cjg0w5Ga/k4hIiP31/WymLcnj4fN70bN1st9xpI5Tm6jUTsWF8O1j0PYoaHek32lEJIQmTl/BC98tZ9SQ9pzTp7XfcURUTEktNXsibF0Fx6ivlEhtkrU8jzvfncuQjqn8aWhnv+OIACqmpDYqKYavH4aWveHwE/1OIyIhsmbLTq6a8AOtkuvz+Ii+1IvWrzAJD/qXKLVP9jvepMZDbgLdJi1SK+wqKuGqCVnsLCxmzCUZNEqI8TuSyB7qgC61S2kpTH0IUo+Azmf6nUZEQsA5x+3/ncPsnC2MubgfnZon+R1JZB9qmZLaZcFHsH4eDLkRovTPW6Q2eO7rpfx35ipuPLkTp3TTROUSfir128bMhprZz2a2yMxuLWd9WzObYmYzzWy2mZ0e+qgiFXAOpj4AyW2h+//5nUZEQmDqwlz+Pnk+Q7u14JrjO/gdR6RcFRZTZhYNPAGcBnQFRphZ1zKb3QG87pzrA1wAPBnqoCIVWvolrMqCwddDtK5gi0S65RvzueaVmXRslsSD5/UiKkp9ICU8VaZlagCwyDm3xDlXCEwEhpfZxgENA88bAatDF1GkkqY+CA1aQO8L/U4iIlW0vaCYUS9mYgbPXpJBYpy+IEn4qkwx1QpYGfQ6J7As2D3ARWaWA0wGrg1JOpHKWjkDln4FR10DMZpWQiSSlZY6bnxtFotz83liZF/aNknwO5LIAYWqh+4IYLxzrjVwOjDBzH7x3mY22swyzSwzNzc3RIcWwWuVqt8Y+v3G7yTik4r6dga2Oc/Mss1snpm9ErS8xMxmBR6Tai61lOfRzxfySfY6/nx6FwZ3SPU7jkiFKtNuugpoE/S6dWBZsCuAoQDOue/MLB5IBdYHb+ScGwOMAcjIyHCHmFlkX2vnwoIP4fg/Q1wDv9OID4L6dp6M13o+w8wmOeeyg7bpCNwGDHbObTKzZkFvsdM517smM0v5Ppq7lkc/X8j/9WvNbwan+x1HpFIq0zI1A+hoZu3NLBavg3nZb24rgBMBzKwLEA+o6UlqxtQHITYJBozyO4n4pzJ9O0cBTzjnNgE459YjYeWntVu58fVZ9GqTzF/P7o5p0F2JEBUWU865YuAa4GNgPt5de/PM7F4zGxbY7CZglJn9CLwKXOacU8uTVL8Ni2De29D/Cu8yn9RVlenb2QnoZGbfmNk0MxsatC4+0AVhmpmdXc1ZpRybdxQy+sUsGsTVY8zF/YiPifY7kkilVer2COfcZLyO5cHL7gp6ng0MDm00kUr45mGoFwdH/t7vJBL+6gEdgePwuit8ZWY9nHObgXbOuVVmdhjwPzOb45xbXPYNzGw0MBqgbdu2NRa8tisuKeWaV2aydssuJl41iOYNdROJRBYNES2Ra/NK+HEi9L0EGjSreHupzSrTtzMHmOScK3LOLQUW4BVXOOdWBf5cAnwB9CnvIM65Mc65DOdcRtOmTUP7Ceqwf3z4E18v2sBfz+lO37ZqYZbIo4E7JHJ9+x/vz6Ou8zeHhIM9fTvxiqgLgJFltnkH787j580sFe+y3xIzawzscM4VBJYPBv5VY8nruDezcnju66VcdlQ652W0qXgHP21cDD99APENIbEpJDaDxFTvuW5+qdNUTElk2p4LP7wAPS+A5DA/AUu1c84Vm9nuvp3RwLjdfTuBTOfcpMC6U8wsGygBbnHObTSzo4BnzKwUr7X+/uC7AKX6zFq5mdvfnsNRhzfhz2d08TvO/m1eAV/+C2a9Aq6k/G1iEvYWVsFFVoNmgWWpgeVNISEFotQnrDZRMSWRadoTUFwAR9/gdxIJE5Xo2+mAGwOP4G2+BXrUREbZa/3WXVw1IZNmSXE8PrIvMdFh2Otk21r46gHIGg9m3h3DR17jPd++HvI3QH4u5Ac9374etuTA6pne6/KKL4uChCaBIivo0SD4dVBBFqtBS8OdiimJPDs3wfSx0O1sSNXEpyKRpqC4hKteymLrzmL++7ujSEmM9TvSvvI3eje3TH8WSouhz0VwzC3QqPXebYKf709pKezavLfIys8NFF1Bz7ev9+YUzd8AhdvKf5/YBgdo9Qpe3tS7qzkqDAvTWk7FlESe6WO9k86Qm/xOIiIHyTnHne/MZeaKzTx1YV+6tGxY8U41Zedm+O4JmPYkFOZDz/PhuD9BymGH9n5RUd4lvYQUaHpExdsX7oAdu1u4cgMFV5nHpmWQM8PbzpX+8j0sOqjA2l+rV9BD02+FhIopiSwF270TXcdToYWuzIhEmhe/W87rmTlce0IHTuvR0u84noLtMP0Z+OYxryWp63A47nZo1rlmc8QmQGxbSK7EsBulpbAzb2+Rtb/LjnlLvOdF+eW/T1zD8ouvhBSvRSwuyetcH5u093lckvc6WiXEbvpJSGT54QXvBHLMzX4nEZGD9O3iDdz7fjYndWnGDSd18jsOFO2CzHHw9UNe4dHxVDjhz9Cyl9/JKhYVFSiCUoFKdN4vzA+6zLify44bF8OKabBjI1CJcbfrxQcKq91FV9K+BdjuomvP8wZe8RbX4Jf71AuzS70HScWURI7iAm84hPQh0GaA32lE5CCszNvB71/+gfapiTx8fm+ionycKqa4EGZO8DqXb1sN7Y+FE+6o3eeV2ETv0Ti94m1LiqFgKxRuh4JtXstd4ba9zwu2Ba3btu9229fCxoV7tyveWbl80XFlCrCk8ouuyhRq9eKq9KM6FCqmJHLMegW2rYGzn/I7iYgchB2FxYx6MZOSUsezl2SQFB/jT5DSEpj9GnxxP2xeDq0HwK+egfbH+JMnXEXX29vXq6pKivcWW4XbA0VWJQu17euhcMne5fu7VFlWVExQ0dUwUGjtp3XssOOgZc8qf0wVUxIZSorhm0egVT/vH7+IRATnHDe/8SML1m3j+d8MoH1qYs2HKC2F7Hfgi3/AhgXQoieMfAM6nuwNcyDVJ7oe1E/2HlVVWrK3IAtuGTtgoRZYtiMPNi0P2j9w5+QZD6qYkjpk7lveXSyn/l0nP5EI8sSURUyes5bbT+/MsZ1qeAoe52DBR/C/v8G6OdC0M5w3AbqcpfNIJIqKhvhG3qOqSku9lq6o0JRBKqYk/JWWeh1Em3WFTqf5nUZEKumz7HU8+OkCzu6dxqghhzi8wKFwDpZ8Af/7K6zKhMbt4VfPQvdzNfK4eKKivMt+IaJiSsLfzx9A7k/wq7EajE4kQixav43rX5tFt7SG3H9uT6ymWoKWf+cVUcu/hoat4azHoPdIiPapn5bUCSqmJLw5B1Mf9L5ZdjvH7zQiUglbdhYx6sUs4mOiGHNxBvExNdAatOoHmPI3WPQZNGgOp/0b+l3qy51dUveomJLwtvh/3hxXZz2qAeJEIkBJqeO6V2eSs2kHr4waRFpy/eo94Lpsr4j66X1vKpWT74X+ozSfndQo/XaS8Db1IUhKg14j/E4iIpXwr49/4ssFufz9nB70Tw/BrfX7s3ExTPm7d3NKXJI3Yvmg30J8GE1PI3WGiikJXyumef0eTv2HmupFIsC7s1bxzJdLuGhQW0YOrMSUKIdi8wr48p8w61XvvHD09XDUdaEZE0nkEKmYkvA19UFIaOL1exCRsDYnZwt/fHM2A9JTuOvMbqE/wNY13jkhazxYFAy8Co6+ARo0C/2xRA6SiikJT2t+hIWfwAl3elMgiEjYytm0g9ETMmmSGMuTF/Ultl4I77rN3wjfPAzTn4XSYuhzsTc3Z6PWoTuGSBWpmJLwNPVBb8j//lf6nUREDmDtll2MfPZ78guKeXX0IFIbhOiS/M7N8N3jMO0pKNoBPc+HY/8IKTU4XpVIJamYkvCTuwCyJ8GQG0MzBYGIVIvcbQWMHDuNvPxCJlwxgG5pIRiZumA7fP80fPsY7NoCXc+G42+HpkdU/b1FqomKKQk/Xz8M9eJh0O/8TiIi+5GXX8hFY79nzeZdvHD5APq0bVy1NyzaCZnjvDt4d2yATkPh+D+HZN40kepWqWLKzIYCjwLRwFjn3P1l1j8MHB94mQA0c84lhzCn1BWblnuzug8YDYmpfqcRkXJs2VnExc99z9KN+Tx/WX8GtK/CnXTFhTBzAnz1AGxb7U1kfvwd0KZ/yPKKVLcKiykziwaeAE4GcoAZZjbJOZe9exvn3A1B218L9KmGrFIXfPuYd6fOUdf6nUREyrG9oJhLx01nwbptjLkkg8EdDvFLT0kxzHkdvrgfNi+HNgPhV2Og/ZDQBhapAZVpmRoALHLOLQEws4nAcCB7P9uPAO4OTTypU7atgx8mQO8R0KiV32lEpIwdhcVc/vwM5qzawpMX9uX4Iw5hWILSUsh+G6b8AzYuhJa94IwHocNJUFPz94mEWGWKqVbAyqDXOcDA8jY0s3ZAe+B/+1k/GhgN0LZtNQ3oJpHru8ehtAgGX+93EhEpY1dRCaNezCRzeR6PXtCHU7u1OLg3cA5+/tCb+mXdXGjaBc5/CTqfqSJKIl6oO6BfALzpnCspb6VzbgwwBiAjI8OF+NgSyXbkeZ1Pu/0KmhzudxoRCVJQXMJvX8ri28UbeeD/enFWr7TK7+wcLJkC//srrMryhjb41Vjo/iuIqoEJkEVqQGWKqVVAm6DXrQPLynMB8PuqhpI6aPoYKNzuDYcgImGjqKSUa1+ZyZSfvfn2zu13EINlLv8O/ncfLP8GGrWBYf/x5tmMjqm+wCI+qEwxNQPoaGbt8YqoC4CRZTcys85AY+C7kCaU2q9gmzcw3xGnQ/NqmIZCRA5JSanjhtdm8Un2Ou45q2vl59tb9YPXErX4c2jQHE77tzctlObYlFqqwmLKOVdsZtcAH+MNjTDOOTfPzO4FMp1zkwKbXgBMdM6F/PLdvNVb6NCsAXH11CRcK2U+D7s2w5Cb/E4iIgGlpY5b3vyR92ev4bbTOnPZ4PYV77RuHkz5O/z0PtRPgZPvhf6jIDah+gOL+KhSfaacc5OByWWW3VXm9T2hi7XXjsJihj/+DVFRRu/WyfRv35j+6Sn0bdeYhvFqKo54Rbu8juftj4XWGX6nERHAOcef35nLf39YxQ0ndeKqYyvox5iT5f0/nvc2xCV5g20OvBriG9ZMYBGfhf0I6NFRxpMX9mXGsjymL9vEM18u4Ykpi4ky6NyiIQPap9A/PYX+7RvTLCne77hysGa9BNvXwblj/U4iIniF1F/ey+bV6Sv43XGHc92JHcrfsKQI5k+CaU9DznSITYKjr4ejroOEKgziKRKBwr6YiqsXzSndWnBK4DbcHYXFzFyxmelL85ixLI/XZqxk/LfLAEhvkhAorLwCK71JAqZbbsNXSRF8/Si0HgDpGqhPxG/OOe7/6CfGf7uMywe355ZTj/jlOXRHHmSNhxljYesqaNwehv4Teo9US5TUWWFfTJWVEFuPwR1S94y6W1RSyrzVW5mxNI/py/L4bP463sjKAaBpUhz9073Lgv3TU+jSsiHRUSquwsacN2DLCjj93xpnRiQMPPLZQp75cgkXDWrLnWd22beQWj/fu1Fk9utQvNO7NH/Gg9DxFA1xIHVexBVTZcVER9G7TTK92yQz6pjDKC11LM7dzvRlecxYmseMZZuYPGctAElx9ejbrvGeAqtXm2TiY3QS8EVpiTehafPu0OlUv9OI1HlPfrGIRz9fyK/7tebeYd29Qqq0FBZ+At8/BUu+8CYg73me1x9Kd96K7BHxxVRZUVFGx+ZJdGyexIUD2wGwavNOMpfl7bk0+MAnuQDERkfRs3Uj+rdPYUCgU3uj+urUXiPmv+dNJfF/49QqJeKz575eyr8++plhvdK4/9yeRBVth5kvw/RnIG8JJKXBiXdB38sgsYnfcUXCTq0rpsrTKrk+rXq3Ynhvb763TfmFZC7f5HVqX5rHs18t4akvFmOBTu27W64GtE+heUN1ag8552Dqg5ByOHQ92+80InXaS9OWc9/72Qzt1oKHTm5E9Ce3w8yXoGArtO7v3ZnXdbgG2hQ5gDpRTJXVODGWk7s25+SuzQHYWVjCzJWbmLHUK7DezMrhxe+WA9A2JSFQWHkFVvvURHVqr6pFn8Ha2TDscfW1EPHRG5krueOdOfyu3Rpujn6JqMc/9P5PdjsHBv4WWvfzO6JIRAj/Yso5GHMcJLWExu0gud2+f8YlVfkQ9WOjOerwVI46fG+n9uzVW/e0XE35eT1v/eB1ak9tEEtGu5Q9lwa7tEyiXnRUlTPUKVMfhIatoef5ficRqbPey1pM1ttP8VXS57RdtwS2NfEGzu1/BTQ8iLn3RCQCiqnCfO8/9qblsGyqN39bsPopZYqstpCc7j1v1AZiDv4yXUx0FL3aJNOrTTJXDjkM57xO7TOWbdpz1+BH87xO7Ymx0fRt15gBgSEZeqtT+4Et+wZWfAen/QvqxfqdRqTu2bqGxZMf4aj5L3NWzDZKk7vCkf+BHr+GmPp+pxOJSOFfTMU1gBGves+d88Y42bzMK642L9/759o58PNkKCncd/+klr9szdr9Z8NWlbrMZGZ0aJZEh2ZJjBjgzU21ZsvOPR3aZyzdxIOfLgAgJtro0Wpvp/aMdik0SqgDfQ2KC73BN7evg21rYNvawPO1gedrYds6yM+FxKbQ9xK/E0stY2ZDgUfxpr0a65y7v5xtzgPuARzwo3NuZGD5pcAdgc3+6px7oUZC16ScLJj2JKXz3qF9aQkz4gaS8H+3Ur/jcboJRKSKrBqm0quUjIwMl5mZGdo3LS31fpHvKbJW7FtwbV0FrnTv9lH1oFHrMkVW+t7XiU0rfZLZvKOQzGWbmLHcG5JhzqotFJU4zOCI5kn0T08hI70xA9qn0LJRBH37K9q5b2G0p1hat7dA2rYGdub9cl+L8n6GSS2gQQtIau792eUsaNmz5j+L+M7MspxzIZ83yMyigQXAyUAO3gTtI5xz2UHbdAReB05wzm0ys2bOufVmlgJkAhl4RVYW0M85t+lAx6yWc1iolRRB9rvw/dOQM4Pieg2YUDiEb1PO5YGrztbdyyIH4UDnr/BvmToYUVHQqJX3aHfUL9cXF8LWnF+2am1aDj9/6LWaBItJCFw23E/LVnyjPZsmJ8RyUtfmnBTUqX3Wys1ey9WyPP77Qw4Tpnmd2ls3rr/nsmD/9BQOb+pDp/aC7UEF0tqgFqQyrUm7tvxy36h63kzwDZp7P4e2A/ctlpICj8Sm6mAuNWUAsMg5twTAzCYCw4HsoG1GAU/sLpKcc+sDy08FPnXO5QX2/RQYCrxaQ9lDb0ceZD0P08fCttWQchjL+t/NudPSadoklVdHDVIhJRJCtauYqki9WEg5zHuUpzDfa80qr9ha8Z13q3Cw+OSgflr7tmrVT27LkYc34cjDvTFZiktKyV6zdU+/qy8X5PLfmasASEmMJaNdYwYd1oTTerQ49JYr57yMu1uLDtSaVLjtl/tHx+4thpp2gvbHeAVSUst9i6WEJl7hKhI+WgErg17nAAPLbNMJwMy+wbsUeI9z7qP97Nuq+qJWo3XZ3gCbs1+H4l1w2HFw5sNkxfXnknEzaJEcz0tXDqRxovorioRS3SqmKhKbCM26eI+ynINdm8svtHJ/hoWfeievYA2a7ymu6iW3o2fjdvRMa8cV3drhGvZkSV7Bng7tM5bl8Un2Ou77IJsB6SkM792K03u0IDkh1jv2zk379j3aX7FUvPOX2WMSvCxJLaBFD+hw8i9bkRo0h/qN1XdCarN6QEfgOKA18JWZ9TiYNzCz0cBogLZt24Y636EpLYWFH3tTvSz9MjBK+fmBUcq7MidnC5eNnUZqUhyvjBpEaoM4vxOL1DoqpirLzCs26jeGtN6/XF9aCvnryxRby7w/V34Pc/8LriTo7aI5vFErDk9uxwWN28HAdDa4hixYupT1q5cT/34uSz/YTJvYrTQu3UR0aeEvjxmbtLcYapURVBiVueQWl6QiSWq7VUCboNetA8uC5QDfO+eKgKVmtgCvuFqFV2AF7/tFeQdxzo0BxoDXZyoUwQ/Zrq0w6xWvP9SmpYFRyu+GfpdBQgoA89ds5eJx39MwPoZXRg3SIMQi1UTFVKhERe0tZtqWvboAlBTvv7/Wws9g+1pSgVTAxSdT0KQpq0uTmb49jRWFDdkU1Zi01un06NyZHl2OIKZRS68lTUTA63De0cza4xVHFwAjy2zzDjACeN7MUvEu+y0BFgN/N7PGge1OAW6ridCHJG8JfD/GG6W8cBu0HgAn3gldhu0zSvmi9du4aOz3xNeL5tVRg2iVHEE3vohEGBVTNSW6ntenqnF6+euLdsKOjZCQisXEEw8cBrQrdUxfmsekH1cxcc5atiwpovEXSzij506G925Fv7aNiYpSq5PUbc65YjO7BvgYrz/UOOfcPDO7F8h0zk0KrDvFzLKBEuAW59xGADO7D68gA7h3d2f0sOEcLP3Ka4X6ueJRypdtyGfks99jZrw8aiBtmyT4EFqk7qhdQyPUcgXFJXy1YAOTflzNp9lr2VVUSqvk+pzVK41hvdLo0jJJU91IWKuuoRH8UCPnsKKdMOcNmPY0rJ/n3fyRcTlkXAENW5a7S86mHZz/zDR2FpUwcfQgOjWv+iwRIlKXhkao5eLqRe+ZUzC/oJhPs9fx7qxVPDt1CU9/uZiOzRowvHcaw3q10jdRkUi2dTXMGAuZz3tjuDXv7s1l2ePXB5zVYe2WXYx89nu27SriVRVSIjVGxVSESoyrx9l9WnF2n1Zs3F7A5LlrmTRrFQ98soAHPllAn7bJDO+Vxhk902iapLt3RCJCTqZ3V172O1BaAp3P8O7KSz+6wptI1m/bxchnp5GXX8hLVw6kW1qjA24vIqGjYqoWaNIgjosHtePiQe3I2bSD935cw6QfV3PPe9nc+342gzukMrx3K07t1pykeA3UJxJWyoxSTlxDGHAVDBgFKe0r9RZ5+YVcNPZ71mzZxYtXDKB3m+TqzSwi+1CfqVpswbptTJq1mnd/XMXKvJ3E1ovixM7NGN47jeOOaKYJmaXGqc9UkPyN3ijlM57bM0o5A6+G3iO94UwqacuOIkaOncai9dt5/rL+HNUh9dAzich+VbnPVFUmEBX/dGqexM2nHsFNp3Ri5srNTJq1mvdnr+bDuWtJiqvH0O4tGN67FUce3oRo3REoUjN+MUr58XDWI95gugc5s8C2XUVc8vx0Fq7bzphL+qmQEvFJhcVUYALRJwiaQNTMJpUzgehtwODdE4hWV2A5eGZG37aN6du2MXec0YVvF2/k3VleUfVGVg5Nk+I4s2dLhvduRa/WjXRHoEio7Rml/ElviIN69aHXBV5LVHkzLlTCjsJiLh8/g3mrtvDkhX057giddkX8UpmWqapMICphpl50FMd0asoxnZryt6LuTPlpPe/OWs3L01bw/DfLaNckgeG90hjWO40OzXQnkEiVZY2Hrx/xRilv2ApOugf6XrpnlPJDsauohCtfyCRr+Sb+M6Ivp3RrEaq0InIIKlNMVWUCUQlj8THRnNajJaf1aMmWnUV8PG8tk2at5vEpi3jsf4vo2rIhw3uncVavNNI0erLIoVmXDQ2awYl3QZez9hml/FAUFJdw9UtZfLdkIw+d14szepY/3pSI1JxQ3c1X7gSizrnNwRuF5SShAkCj+jGcl9GG8zLasH7rLt6fvYZ3f1zNPz78iX98+BMD2qcwvHcap3dvqRnnRQ7GqX+rcgG1W1FJKde8MpMvfs7l/l/14Jw+rUPyviJSNZUppqoygeiM4I3CapJQ2a9mDeO5/Oj2XH50e5ZtyGfSj6t5d9Yq/vz2XO5+dx7HdmrKsN5pnNy1OQmxGl1D5IBCVEgVl5Ry/Wuz+DR7HX8Z1o0LBugLqUi4qMxvwqpMICoRLj01ketO7Mi1J3Qge81WJs1azaQfV/P5T+upH+ONyD68dxpDOjYltt7B3YkkIpVTWur445uz+WD2Gm4/vTOXHpXudyQRCVJhMVXVCUSldjAzuqU1oltaI/40tDMzluXx7o+rmTzHGyA0OSGG03u0ZHivNPqnp2jyZZEQcc7x53fm8N+Zq7jp5E6MPuZwvyOJSBkatFOqpLC4lKkLc3l31mo+zV7HzqISWjaKZ1jgjsCuLRtqqAXZQ4N2HhznHH95L5vx3y7j98cfzi2ndq7W44nI/mmiY6k2sfWiOLFLc07s0pwdhd7ky5Nmrea5r5fyzFdL6NCsgVdY9UojPTXR77giEcM5x/0f/sT4b5dx5dHtufmUI/yOJCL7oWJKQiYhth7De7dieO9WbMovZPLcNbw7azUPfbqAhz5dQK823uTLJ3dtTuvG9dViJXIAD3+2kGe+WsLFg9rx5zO66P+LSBhTMSXVonFiLBcObMeFA9uxevNO3vtxNe/OWs2973uTLzeqH0O3tIZ0S2tI91aN6JbWkPapDTStjQjwxJRFPPb5Qs7LaM1fhnVTISUS5lRMSbVLS67PVccezlXHHs6i9duYtiSPeau3Mm/1Fl74bjmFxaUA1I+JpkvLpD3FVbe0RnRqnqS7BKVOee7rpfz7458Z3juNf/yqp27mEIkAKqakRnVolrTPNDVFJaUsWr+deau3MnfVFrJXb+WtrBxe/K4EgJhoo1PzpH1asLq0bKjxraRWmjBtOfe9n81p3Vvw4K97qaVWJELoN5L4KiY6ii4tvQLp//p5ozmXljqW5+1g7qote1qwPs1ex+uZOQCYwWGpiXuKq+6BIRsaJYRmcEQRP7yeuZI735nLSV2a8egFfagXrRZZkUihYkrCTlSU0T41kfapiZzVKw3w7mxas2XXnhaseau3Mn1pHu/OWr1nv9aN6+8trlp5fzZrGO/XxxCptHdnreJPb81mSMdUHh/ZV5e2RSKMiimJCGZGWnJ90pLrc3LX5nuWb9xe4BVYqwOtWKu28PG8dXvWpzaIo3urhvu0YLVJ0Z2EEj4+nLOGG1//kYHtUxhzcQbxMdF+RxKRg6RiSiJakwZxHNOpKcd0arpn2bZdRWSv3rqnyMpevZWpCzdQUuoNUNswvh5dA8XV7kuFhzXVnYRS8z6fv47rJs6kd5tknru0P/VjVUiJRCIVU1LrJMXHMPCwJgw8rMmeZbuKSvh57bZ9WrBenLbvnYSdWyYFWq+8zu4dmzcgrp5+uUn1mLowl9++9ANdWjbk+d/0JzFOp2ORSKX/vVInxMdE06tNMr3aJO9ZVlRSyuLc7cxbtfcy4dszVzFh2nIA6kWVfyehfulJVU1bspFRL2ZyWNNEXrx8AA3jdfOESCTTbwWps2Kio+jcoiGdWzTk3KA7CVfk7dhTXM1dtYXPf1rPG1l77yRsn5q4TwtWt7SGJCfE+vlRJIJkLd/E5eNn0LpxAi9fOVD/dkRqARVTIkGiooz01ETSUxM5s+feOwnXbt21TwtW5rI8Jv24907CVsn19ymuuqU1okmDWGJ0e7sEmZ2zmcvGTadZUhyvXDmQJg3i/I4kIiGgYkqkAmZGy0b1admoPicF3UmYl1/IvKAWrOzVW/kke90++8bWiyIxNpqE2HokxkWTGFePxNh6JMR6zxNio2kQV2/P+j3bxdYjIfDn3uX1SIyN1vhDEWr+mq1c/Nx0GiXE8MqoQRq2Q6QWUTElcohSEmMZ0rEpQzruvZNwe0Ex89dsZf6arWzZUUR+YQn5BcXkFxazo6CE/MJi8guK2bC9YJ9lu4pKK33c4AKtQdzeomt3gbanGKugQEvcXcSpQKt2i9Zv46Kx35MQG82rowaRllzf70giEkIqpkRCqEFcPfqnp9A/PeWg9isuKWVHUcme4mpHQQnbC4rZUVhMfmEJOwqK91uY7Qgs312g5Rd4rwuKD65AaxBoKftF0RUbXLAFF2LesiGdUnXX4wEs25DPyGe/JyrKePnKgbRJSfA7koiEmIopkTBQLzqKhtFRIb2rK7hA21OYFZTsKdDyC4KKsd2F2e5iLrA+d1vBPq/LK9Dm3HOKiqkDuGvSPIpLHRNHD+Kwpg38jiMi1UDFlEgtVV0FWn5hyT6FWaImnT6gh87rxYbtBXRqnlTxxiISkXQWFJFKqxcdRaP6UTSqr3GRKiu1QRypumtPpFZTr1MRERGRKlAxJSIiIlIFKqZEREREqkDFlIiIiEgVVKqYMrOhZvazmS0ys1vLWX+ZmeWa2azA48rQRxUREREJPxUWU2YWDTwBnAZ0BUaYWddyNn3NOdc78Bgb4pwiIgdUlS99ZlYStHxSzSYXkUhXmaERBgCLnHNLAMxsIjAcyK7OYCIilRX0pe9kIAeYYWaTnHNlz1OvOeeuKectdjrneldzTBGppSpzma8VsDLodU5gWVnnmtlsM3vTzNqEJJ2ISOXs+dLnnCsEdn/pExGpdqHqgP4ekO6c6wl8CrxQ3kZmNtrMMs0sMzc3N0SHFhGp8pe++MC5aZqZnb2/g+gcJiLlqUwxtQoIPum0Dizbwzm30TlXEHg5FuhX3hs558Y45zKccxlNmzY9lLwiIofqQF/62jnnMoCRwCNmdnh5b6BzmIiUpzJ9pmYAHc2sPV4RdQHeCWcPM2vpnFsTeDkMmF/Rm2ZlZW0ws+UHkTUV2HAQ20eiuvAZQZ+zNjnYz9iumnJU6ktf0MuxwL+C1q0K/LnEzL4A+gCLD3TAgzyH1YV/C1A3Pmdd+Iygz1me/Z6/KiymnHPFZnYN8DEQDYxzzs0zs3uBTOfcJOA6MxsGFAN5wGWVeN+D+lpnZpmBb461Vl34jKDPWZuE0Wc85C99ZtYY2OGcKzCzVGAwQYXW/hzMOSyMfk7Vqi58zrrwGUGf82BVaqJj59xkYHKZZXcFPb8NuK2qYUREDkUVv/R1AZ4xs1K8rg/3l3MXoIjIflWqmBIRCXeH+qXPOfct0KPaA4pIrRVJ08mM8TtADagLnxH0OWuTuvAZQ6Gu/JzqwuesC58R9DkPijnnQvE+IiIiInVSJLVMiYiIiISdsC+mzGycma03s7l+Z6kuZtbGzKaYWbaZzTOzP/idqTqYWbyZTTezHwOf8y9+Z6ouZhZtZjPN7H2/s1QXM1tmZnMC89ll+p0nHOn8VXvo/FW7hPr8FfaX+czsGGA78KJzrrvfeaqDmbUEWjrnfjCzJCALOLu23VFkZgYkOue2m1kM8DXwB+fcNJ+jhZyZ3QhkAA2dc2f6nac6mNkyIMM5VxfGojkkOn/VHjp/1S6hPn+FfcuUc+4rvNuYay3n3Brn3A+B59vwxr8pbyqMiOY82wMvYwKP8K7mD4GZtQbOwBsYUuownb9qD52/5EDCvpiqa8wsHW/05e99jlItAs3Hs4D1wKfOudr4OR8B/giU+pyjujngEzPLMrPRfocR/+n8VSs8gs5fB03FVBgxswbAW8D1zrmtfuepDs65Eudcb7zpPgaYWa269GFmZwLrnXNZfmepAUc75/oCpwG/D1zSkjpK56/Ip/PXoVMxFSYC1+DfAl52zv3X7zzVzTm3GZgCDPU5SqgNBoYFrsdPBE4ws5f8jVQ9guazWw+8DQzwN5H4ReevWkPnr0OkYioMBDo2PgfMd8495Hee6mJmTc0sOfC8PnAy8JOvoULMOXebc661cy4db364/znnLvI5VsiZWWKgszFmlgicAtTaO9Zk/3T+qj10/jp0YV9MmdmrwHfAEWaWY2ZX+J2pGgwGLsb7FjAr8Djd71DVoCUwxcxm401M+6lzrtbeelvLNQe+NrMfgenAB865j3zOFHZ0/qpVdP6qPUJ+/gr7oRFEREREwlnYt0yJiIiIhDMVUyIiIiJVoGJKREREpApUTImIiIhUgYopERERkSpQMSUiIiJSBSqmRERERKpAxZSIiIhIFfw/mNh+bktQ7jQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 720x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_losses = [0.6835, 0.6401, 0.5760, 0.5407, 0.5320]\n",
    "test_losses = [0.6570, 0.6335, 0.6515, 1.0327, 1.3143]\n",
    "train_accs = [0.5620, 0.6272, 0.7049, 0.7788, 0.8322]\n",
    "test_accs = [0.5864, 0.6138, 0.6464, 0.6409, 0.6376]\n",
    "\n",
    "epochs = list(range(1, len(train_losses) + 1))\n",
    "\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10,5))\n",
    "fig.suptitle('Loss and accuracy per epoch')\n",
    "ax1.plot(epochs, train_losses, label='train')\n",
    "ax1.plot(epochs, test_losses, label='test')\n",
    "ax1.legend()\n",
    "ax1.set_title('Loss')\n",
    "ax2.plot(epochs, train_accs, label='train')\n",
    "ax2.plot(epochs, test_accs, label='test')\n",
    "ax2.legend()\n",
    "ax2.set_title('Accuracy')\n",
    "#plt.show()\n",
    "plt.savefig('lossacc.png', dpi=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot([s['steps'] for s in stats], [s['loss'] for s in stats])\n",
    "plt.plot([s['steps'] for s in stats], [s['acc'] for s in stats])\n",
    "plt.legend(['loss', 'acc'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RobertaForSequenceClassification.from_pretrained(\"Limit800/epoch4\", config=config)\n",
    "model.to(device)\n",
    "pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = RobertaConfig.from_pretrained(\"microsoft/codebert-base\")\n",
    "config.num_labels = 1\n",
    "model = RobertaForSequenceClassification.from_pretrained(\"Limit800/epoch2\", config=config)\n",
    "model.to(device)\n",
    "#model = Model(model)\n",
    "#model.to(device)\n",
    "\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    bar = tqdm(test_dataloader,total=len(test_dataloader))\n",
    "    total_loss = 0\n",
    "    val_results = []\n",
    "    val_labels = []\n",
    "    for step, batch in enumerate(bar):\n",
    "        inputs = batch[0]\n",
    "        inputs = {key: value.squeeze(1).to(device) for key, value in inputs.items()}\n",
    "        labels = batch[1].to(device)\n",
    "        ids, mask = inputs['input_ids'], inputs['attention_mask']\n",
    "        #outputs = model(**inputs)\n",
    "        outputs = model(ids, mask)\n",
    "        #result = outputs[:,0]\n",
    "        result = outputs.logits[:,0]\n",
    "        loss = BCEWithLogitsLoss()(result, labels.float())\n",
    "        total_loss += loss.mean().item()\n",
    "        val_results.append(result.cpu().detach().numpy())\n",
    "        val_labels.append(labels.cpu().numpy())\n",
    "results=np.concatenate(val_results,0)\n",
    "labels=np.concatenate(val_labels,0)\n",
    "preds=results>0.5\n",
    "eval_acc=np.mean(labels==preds)\n",
    "loss = total_loss / step\n",
    "time.sleep(1)\n",
    "print(f\"loss {loss:.4f}, acc {eval_acc:.4f}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "asdl",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}