transfer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "transfer.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "791d852cbc9942d985d0e2d9d1226c2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6e73786b549e4e3d806f4b53e32b7970",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6fd266ef8db846abb81876f3651cbbe4",
              "IPY_MODEL_d5f65c01a27d4b0c93dc9b0031dbe8cf"
            ]
          }
        },
        "6e73786b549e4e3d806f4b53e32b7970": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6fd266ef8db846abb81876f3651cbbe4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9f1cca834d7f4f8384fffe3941656e83",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 760289,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 760289,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_56615bf1e55f4abc83a189ff3811e968"
          }
        },
        "d5f65c01a27d4b0c93dc9b0031dbe8cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5a552c41025449abbc3c0101eb561b7b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 760k/760k [00:00&lt;00:00, 837kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_adcd3c554bc6412cbf7f48ac4f6ad13d"
          }
        },
        "9f1cca834d7f4f8384fffe3941656e83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "56615bf1e55f4abc83a189ff3811e968": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5a552c41025449abbc3c0101eb561b7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "adcd3c554bc6412cbf7f48ac4f6ad13d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cfde8f558a6a49df8990724a4680ae73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_44cfa8a3e5174c3c94805078bf96b600",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_64ecc98a854f485289eafba567293a3a",
              "IPY_MODEL_67088a95587a4850a04d05c5415683bf"
            ]
          }
        },
        "44cfa8a3e5174c3c94805078bf96b600": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "64ecc98a854f485289eafba567293a3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6dc95ac4fcae4dd9aea1508ae933d8c2",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 534,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 534,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_69cba05a7fca499b9fdddf428a71097c"
          }
        },
        "67088a95587a4850a04d05c5415683bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_356cc86fe74b463a820946de3d47b27a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 534/534 [00:00&lt;00:00, 13.6kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8151f409ac0c48c8afbc1b3f319c7ee4"
          }
        },
        "6dc95ac4fcae4dd9aea1508ae933d8c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "69cba05a7fca499b9fdddf428a71097c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "356cc86fe74b463a820946de3d47b27a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8151f409ac0c48c8afbc1b3f319c7ee4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ec5b90deac24461e898d2a7a27c6d0cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ea2a828efa904605ad6f51ce0fe5f701",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7628b947679b4d64873e6864eebbafa6",
              "IPY_MODEL_5858c342636842009aa354cd76d5a22e"
            ]
          }
        },
        "ea2a828efa904605ad6f51ce0fe5f701": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7628b947679b4d64873e6864eebbafa6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a889ed80ec3d4c24b6f3c0aa41696bef",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 63048440,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 63048440,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b4b5f83d07f94f3b87f254f784be15bf"
          }
        },
        "5858c342636842009aa354cd76d5a22e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7690fbfda2024d3da2acde467b297d26",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 63.0M/63.0M [00:06&lt;00:00, 9.79MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8f824adb0e844ad3838b09c89cab8f3d"
          }
        },
        "a889ed80ec3d4c24b6f3c0aa41696bef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b4b5f83d07f94f3b87f254f784be15bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7690fbfda2024d3da2acde467b297d26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8f824adb0e844ad3838b09c89cab8f3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bd1e739a726b47498e30a75e0defb267": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5ceba88d246a43778da8db56f4efcc45",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_760acbcd5114495ba7aac139f6e8307c",
              "IPY_MODEL_b6e3951bf566435380d4634edd50bb3e"
            ]
          }
        },
        "5ceba88d246a43778da8db56f4efcc45": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "760acbcd5114495ba7aac139f6e8307c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d75dade5d6e94d7c90bf30083d79c3eb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 760289,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 760289,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_52c678735565411885e4bd266e92d52a"
          }
        },
        "b6e3951bf566435380d4634edd50bb3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2267b59989ad42bc8639e340f99fdfa1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 760k/760k [00:06&lt;00:00, 116kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_28e8b53b76674dcda96b7f619e632d15"
          }
        },
        "d75dade5d6e94d7c90bf30083d79c3eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "52c678735565411885e4bd266e92d52a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2267b59989ad42bc8639e340f99fdfa1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "28e8b53b76674dcda96b7f619e632d15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3256b00ff4074fdc9c0b0cc702f587a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_417d263cef464258a46427742110098b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_eb770f9a41574b49a99467b29bf2fa01",
              "IPY_MODEL_f1d2b63b0a23487abb745a8b57d07369"
            ]
          }
        },
        "417d263cef464258a46427742110098b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eb770f9a41574b49a99467b29bf2fa01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_110f8f0f6f8f4a82bef03d69f9c5e264",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 535,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 535,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5951f2c1ad7140f8a184a309489060f0"
          }
        },
        "f1d2b63b0a23487abb745a8b57d07369": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5712645714654e6a953f61bd58184523",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 535/535 [00:01&lt;00:00, 269B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e1486f399b174ede8f5e4072a368ac3d"
          }
        },
        "110f8f0f6f8f4a82bef03d69f9c5e264": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5951f2c1ad7140f8a184a309489060f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5712645714654e6a953f61bd58184523": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e1486f399b174ede8f5e4072a368ac3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "319812155fac4fa7ae7a67de2e014141": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2fd4ecec7f0946f885aaea89b5bf06c3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_340d139d2b044c7e813ed63a923a25ba",
              "IPY_MODEL_aa7869ccc4e447c79a428ac30acda73e"
            ]
          }
        },
        "2fd4ecec7f0946f885aaea89b5bf06c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "340d139d2b044c7e813ed63a923a25ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6b7c9a68853d4ee198f5d021af96e8e0",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 87181048,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 87181048,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7424791a8e614eb7921b0af2e2ce2f07"
          }
        },
        "aa7869ccc4e447c79a428ac30acda73e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d11a8c7c70204f7a97b258abd6ca5c0f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 87.2M/87.2M [00:07&lt;00:00, 11.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b5cf001b80b047fca258772018934d36"
          }
        },
        "6b7c9a68853d4ee198f5d021af96e8e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7424791a8e614eb7921b0af2e2ce2f07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d11a8c7c70204f7a97b258abd6ca5c0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b5cf001b80b047fca258772018934d36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c4d48e7604b3491d86b4eae738de53db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fc015023ebcd491c9296e18e0a464389",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ba5f50bec2404f1d8c1553787c1233cd",
              "IPY_MODEL_345918c2c56a472ea65b754551de8f2e"
            ]
          }
        },
        "fc015023ebcd491c9296e18e0a464389": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ba5f50bec2404f1d8c1553787c1233cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5db08d4f2cac4f9aaec8f50c397bc1bb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 898823,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 898823,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a844628eb4b642718b9e72b3a8ba59ff"
          }
        },
        "345918c2c56a472ea65b754551de8f2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5f1820c392e545609ac013b90aeeac96",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 899k/899k [00:01&lt;00:00, 607kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_67e425b2fda44e51848798e3046fafe8"
          }
        },
        "5db08d4f2cac4f9aaec8f50c397bc1bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a844628eb4b642718b9e72b3a8ba59ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5f1820c392e545609ac013b90aeeac96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "67e425b2fda44e51848798e3046fafe8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e101d5909ef24d6685b50dd9075cd3a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_861c33c149a649ada58255e6c55c3d13",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fcfd6939a1d140d1819312b22c12246b",
              "IPY_MODEL_0c0f0dcab70943e4acd48db536dd0931"
            ]
          }
        },
        "861c33c149a649ada58255e6c55c3d13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fcfd6939a1d140d1819312b22c12246b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e5adc77bd5dd439d979516b7d9a1578d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_08c94cb1d9464ed1ae4c7a9a1cdbb5ca"
          }
        },
        "0c0f0dcab70943e4acd48db536dd0931": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d80d9edeb14040e294a624ac532ba2c9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:00&lt;00:00, 1.42MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_64a86efec80d42439c3e1d0f8fc006f7"
          }
        },
        "e5adc77bd5dd439d979516b7d9a1578d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "08c94cb1d9464ed1ae4c7a9a1cdbb5ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d80d9edeb14040e294a624ac532ba2c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "64a86efec80d42439c3e1d0f8fc006f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b55753c0bee14c1592445e73e1586214": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2d41c2f13d08471b89e79adf2b950cfd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_55afc3097bee496f8bfe593e2b9abc2a",
              "IPY_MODEL_f4daa31329864f48b4336d70cf76330d"
            ]
          }
        },
        "2d41c2f13d08471b89e79adf2b950cfd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "55afc3097bee496f8bfe593e2b9abc2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_23fe778feb6b40a1831bea82542eec10",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 525,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 525,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_206454c229c7408d8c506000bf21ba84"
          }
        },
        "f4daa31329864f48b4336d70cf76330d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7ea630e29a53490cb21a78365c571b34",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 525/525 [00:49&lt;00:00, 10.6B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e8e44f294ed64b7a86d30033eb215598"
          }
        },
        "23fe778feb6b40a1831bea82542eec10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "206454c229c7408d8c506000bf21ba84": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7ea630e29a53490cb21a78365c571b34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e8e44f294ed64b7a86d30033eb215598": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "349a991fce5042dd9bd5ea81a0d78906": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c589a0243a2549e58615f0e0118bf67a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fed87a132cff405d8f0f95abccc878e1",
              "IPY_MODEL_a02b269ffc9b4c9eb212bd08f8cc7053"
            ]
          }
        },
        "c589a0243a2549e58615f0e0118bf67a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fed87a132cff405d8f0f95abccc878e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a6c68da98ac549648e6d99f66585ee5a",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1634375628,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1634375628,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_48bbe146d1cf4b1eb8fd1c1118bd3494"
          }
        },
        "a02b269ffc9b4c9eb212bd08f8cc7053": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8c23f3016a604464b8418fb99f080c25",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.63G/1.63G [00:48&lt;00:00, 33.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bcaefdf33b33444198ac86a193ed23f9"
          }
        },
        "a6c68da98ac549648e6d99f66585ee5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "48bbe146d1cf4b1eb8fd1c1118bd3494": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8c23f3016a604464b8418fb99f080c25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bcaefdf33b33444198ac86a193ed23f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "cjAfQzdlNaQJ",
        "colab_type": "code",
        "outputId": "f32cc758-a37b-4972-d446-ae328bde02cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g_2WFckONpCw",
        "colab_type": "code",
        "outputId": "378facbf-42be-475b-b305-0a0fbc866582",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "try:\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass\n",
        "\n",
        "import tensorflow as tf"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 2.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S8vfsJz8IOJ4",
        "colab_type": "code",
        "outputId": "e8f6013d-a3bc-46ca-914f-7c16202efb48",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print(tf.__version__)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0-rc1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Unt-RSGfGvPe",
        "colab_type": "code",
        "outputId": "14d4af2d-3b96-4008-d645-4b283407aa51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 709
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4c/a0/32e3a4501ef480f7ea01aac329a716132f32f7911ef1c2fac228acc57ca7/transformers-2.6.0-py3-none-any.whl (540kB)\n",
            "\u001b[K     |████████████████████████████████| 542kB 2.7MB/s \n",
            "\u001b[?25hCollecting tokenizers==0.5.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
            "\u001b[K     |████████████████████████████████| 3.7MB 15.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.12.27)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.38.0)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 27.2MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 45.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.5)\n",
            "Requirement already satisfied: botocore<1.16.0,>=1.15.27 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.15.27)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.27->boto3->transformers) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.27->boto3->transformers) (2.8.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=1063da7db8a0cb41e525b5dc810ba6f2befcbda6a72dd83b7754199b7e52a9d7\n",
            "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.38 sentencepiece-0.1.85 tokenizers-0.5.2 transformers-2.6.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2KrW7tZOGw-q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "import re"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cjfCF_hb8-Tc",
        "colab_type": "text"
      },
      "source": [
        "Так как в определении значения большую роль играет контекст, применим две модификации Bert: Albert и RoBERTa."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j8MQHlJ5UPtb",
        "colab_type": "text"
      },
      "source": [
        "## Albert\n",
        "\n",
        "Применим предобученные модели base и large"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eD3lyPo3OCrv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/DeepLearning/Bert/twitter/train.csv', encoding='utf-8')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Eq_viHijYIjU",
        "colab_type": "text"
      },
      "source": [
        "Let's extract the sentences and labels of our training set as numpy ``ndarrays``."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NMpwZtYYYF-F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Get the lists of sentences and their labels.\n",
        "sentences = df['text'].values\n",
        "labels = df['target'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lmfHHN5uI8jU",
        "colab_type": "text"
      },
      "source": [
        "### Albert base"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iRu07udPYL2i",
        "colab_type": "code",
        "outputId": "992e6b2e-525e-4d53-b53f-c180ae1304b4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68,
          "referenced_widgets": [
            "791d852cbc9942d985d0e2d9d1226c2e",
            "6e73786b549e4e3d806f4b53e32b7970",
            "6fd266ef8db846abb81876f3651cbbe4",
            "d5f65c01a27d4b0c93dc9b0031dbe8cf",
            "9f1cca834d7f4f8384fffe3941656e83",
            "56615bf1e55f4abc83a189ff3811e968",
            "5a552c41025449abbc3c0101eb561b7b",
            "adcd3c554bc6412cbf7f48ac4f6ad13d"
          ]
        }
      },
      "source": [
        "from transformers import AlbertTokenizer\n",
        "\n",
        "tokenizer = AlbertTokenizer.from_pretrained(\"albert-base-v2\", do_lower_case=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "791d852cbc9942d985d0e2d9d1226c2e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=760289, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ATUigwHiYpip",
        "colab_type": "text"
      },
      "source": [
        "Let's apply the tokenizer to one sentence just to see the output."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "neeY5VPpYmBj",
        "colab_type": "code",
        "outputId": "1f58e603-b549-41a0-b999-b97f5ef155e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 93
        }
      },
      "source": [
        "print(' Original: ', sentences[0])\n",
        "\n",
        "print('Tokenized: ', tokenizer.tokenize(sentences[0]))\n",
        "\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Original:  Our Deeds are the Reason of this #earthquake May ALLAH Forgive us all\n",
            "Tokenized:  ['▁our', '▁deeds', '▁are', '▁the', '▁reason', '▁of', '▁this', '▁#', 'earth', 'qua', 'ke', '▁may', '▁', 'allah', '▁forgive', '▁us', '▁all']\n",
            "Token IDs:  [318, 18055, 50, 14, 1215, 16, 48, 6926, 12322, 5418, 1048, 123, 13, 12209, 8591, 182, 65]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4CHXvSVqYv4u",
        "colab_type": "text"
      },
      "source": [
        "When we actually convert all of our sentences, we'll use the ``tokenize.encode`` function to handle both steps, rather than calling tokenize and ``convert_tokens_to_ids`` separately.\n",
        "\n",
        "Before we can do that, though, we need to talk about some of BERT's formatting requirements."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WNERGEeUYr1o",
        "colab_type": "code",
        "outputId": "43596558-6e65-4e6c-a589-22a21bffb1fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "\n",
        "input_ids = []\n",
        "\n",
        "for sent in sentences:\n",
        "    encoded_sent = tokenizer.encode(\n",
        "                        sent,\n",
        "                        add_special_tokens = True,\n",
        "                  )\n",
        "    \n",
        "    input_ids.append(encoded_sent)\n",
        "\n",
        "print('Original: ', sentences[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  Our Deeds are the Reason of this #earthquake May ALLAH Forgive us all\n",
            "Token IDs: [2, 318, 18055, 50, 14, 1215, 16, 48, 6926, 12322, 5418, 1048, 123, 13, 12209, 8591, 182, 65, 3]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UB1Tqt8UbPpS",
        "colab_type": "code",
        "outputId": "aad102ce-da70-4457-e088-51d867091d2e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        }
      },
      "source": [
        "print('Max sentence length: ', max([len(sen) for sen in input_ids]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Max sentence length:  100\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "43Z1dBfebsUb",
        "colab_type": "code",
        "outputId": "fd280fd2-4597-4c0f-953b-adf517d6fd1d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 129
        }
      },
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "MAX_LEN = 85\n",
        "\n",
        "print('\\nPadding/truncating all sentences to %d values...' % MAX_LEN)\n",
        "\n",
        "print('\\nPadding token: \"{:}\", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))\n",
        "\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", \n",
        "                          value=0, truncating=\"post\", padding=\"post\")\n",
        "\n",
        "print('\\nDone.')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Padding/truncating all sentences to 85 values...\n",
            "\n",
            "Padding token: \"<pad>\", ID: 0\n",
            "\n",
            "Done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8rp85Qhrb4ME",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "attention_masks = []\n",
        "\n",
        "for sent in input_ids:\n",
        "    \n",
        "    att_mask = [int(token_id > 0) for token_id in sent]\n",
        "    \n",
        "    attention_masks.append(att_mask)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Y8YBZitcBiF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, \n",
        "                                                            random_state=2018, test_size=0.1)\n",
        "train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,\n",
        "                                             random_state=2018, test_size=0.1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7r6CaQSRcThg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 32"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HIfujKYo2rNU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_gen():\n",
        "  for input_id, attention_mask, label_id in zip(train_inputs, train_masks, train_labels ):\n",
        "    yield(\n",
        "        {                        \n",
        "            \"input_ids\": input_id,\n",
        "            \"attention_mask\": attention_mask,\n",
        "        }, label_id\n",
        "    )\n",
        "\n",
        "def validation_gen():\n",
        "  for input_id, attention_mask, label_id in zip(validation_inputs, validation_masks, validation_labels):\n",
        "    yield(\n",
        "        {                        \n",
        "            \"input_ids\": input_id,\n",
        "            \"attention_mask\": attention_mask,\n",
        "        }, label_id\n",
        "    )\n",
        "\n",
        "\n",
        "train_data = tf.data.Dataset.from_generator(\n",
        "    train_gen, \n",
        "    ({\"input_ids\": tf.int32, \"attention_mask\": tf.int32}, tf.int64 ),\n",
        "    ({ \n",
        "        \"input_ids\": tf.TensorShape([None]),\n",
        "        \"attention_mask\": tf.TensorShape([None]),\n",
        "      },\n",
        "    tf.TensorShape([]),)\n",
        ")\n",
        "\n",
        "validation_data = tf.data.Dataset.from_generator(\n",
        "    validation_gen, \n",
        "    ({\"input_ids\": tf.int32, \"attention_mask\": tf.int32}, tf.int64 ),\n",
        "    ({ \n",
        "        \"input_ids\": tf.TensorShape([None]),\n",
        "        \"attention_mask\": tf.TensorShape([None]),\n",
        "      },\n",
        "    tf.TensorShape([]),)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A3MopQsbxwcu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_data = train_data.shuffle(64).batch(BATCH_SIZE).repeat(-1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a7fBA4B9Kwmy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "validation_data = validation_data.shuffle(64).batch(BATCH_SIZE).repeat(-1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8pOxo9QNtWVl",
        "colab_type": "code",
        "outputId": "b6c0dcc7-1843-4f33-d9d3-950e80dea040",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118,
          "referenced_widgets": [
            "cfde8f558a6a49df8990724a4680ae73",
            "44cfa8a3e5174c3c94805078bf96b600",
            "64ecc98a854f485289eafba567293a3a",
            "67088a95587a4850a04d05c5415683bf",
            "6dc95ac4fcae4dd9aea1508ae933d8c2",
            "69cba05a7fca499b9fdddf428a71097c",
            "356cc86fe74b463a820946de3d47b27a",
            "8151f409ac0c48c8afbc1b3f319c7ee4",
            "ec5b90deac24461e898d2a7a27c6d0cd",
            "ea2a828efa904605ad6f51ce0fe5f701",
            "7628b947679b4d64873e6864eebbafa6",
            "5858c342636842009aa354cd76d5a22e",
            "a889ed80ec3d4c24b6f3c0aa41696bef",
            "b4b5f83d07f94f3b87f254f784be15bf",
            "7690fbfda2024d3da2acde467b297d26",
            "8f824adb0e844ad3838b09c89cab8f3d"
          ]
        }
      },
      "source": [
        "from transformers import TFAlbertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "model = TFAlbertForSequenceClassification.from_pretrained(\n",
        "    \"albert-base-v2\",\n",
        "    num_labels = 2, \n",
        "    output_attentions = False,\n",
        "    output_hidden_states = False,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cfde8f558a6a49df8990724a4680ae73",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=534, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ec5b90deac24461e898d2a7a27c6d0cd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=63048440, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IAz-AGCXwWjV",
        "colab_type": "code",
        "outputId": "f880eea4-893a-40ac-b615-f251bd883734",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"tf_albert_for_sequence_classification\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "albert (TFAlbertMainLayer)   multiple                  11683584  \n",
            "_________________________________________________________________\n",
            "dropout_42 (Dropout)         multiple                  0         \n",
            "_________________________________________________________________\n",
            "classifier (Dense)           multiple                  1538      \n",
            "=================================================================\n",
            "Total params: 11,685,122\n",
            "Trainable params: 11,685,122\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LxXNodX1xF2_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "opt = tf.keras.optimizers.Adam(learning_rate=2e-5, epsilon=1e-08)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VPrQtuHTyPxg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if len(np.unique(labels)) > 1:\n",
        "  loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zek3LbPy0hxL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "metric = tf.keras.metrics.SparseCategoricalAccuracy(\"accuracy\")\n",
        "model.compile(optimizer=opt, loss=loss, metrics=[metric])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ed3qOytr0zJB",
        "colab_type": "code",
        "outputId": "e3620de6-dcff-472c-c907-9def87389a62",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        }
      },
      "source": [
        "train_examples = len(train_inputs)\n",
        "valid_examples = len(validation_inputs)\n",
        "\n",
        "print('In training set: {}'.format(train_examples))\n",
        "print('In validation set: {}'.format(valid_examples))\n",
        "# Train and evaluate using tf.keras.Model.fit()\n",
        "train_steps = train_examples // BATCH_SIZE\n",
        "valid_steps = valid_examples // BATCH_SIZE\n",
        "\n",
        "print('Count of steps:\\n\\tTraining set: {} \\n\\tValidation set: {}'.format(train_steps, valid_steps))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "In training set: 6851\n",
            "In validation set: 762\n",
            "Count of steps:\n",
            "\tTraining set: 214 \n",
            "\tValidation set: 23\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bh7FGDZ51xDu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "EPOCHS = 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yTlMfjGn0q9A",
        "colab_type": "code",
        "outputId": "d80b3060-9d4d-4114-e9d6-0a55ee25edec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 409
        }
      },
      "source": [
        "history = model.fit(\n",
        "    train_data,\n",
        "    epochs=EPOCHS,\n",
        "    steps_per_epoch=train_steps,\n",
        "    validation_data=validation_data,\n",
        "    validation_steps=valid_steps\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train for 214 steps, validate for 23 steps\n",
            "Epoch 1/10\n",
            "214/214 [==============================] - 76s 354ms/step - loss: 0.2556 - accuracy: 0.8994 - val_loss: 0.3968 - val_accuracy: 0.8451\n",
            "Epoch 2/10\n",
            "214/214 [==============================] - 75s 351ms/step - loss: 0.2010 - accuracy: 0.9267 - val_loss: 0.4489 - val_accuracy: 0.8383\n",
            "Epoch 3/10\n",
            "214/214 [==============================] - 75s 353ms/step - loss: 0.1902 - accuracy: 0.9280 - val_loss: 0.5664 - val_accuracy: 0.8125\n",
            "Epoch 4/10\n",
            "214/214 [==============================] - 75s 352ms/step - loss: 0.1730 - accuracy: 0.9359 - val_loss: 0.5491 - val_accuracy: 0.8410\n",
            "Epoch 5/10\n",
            "214/214 [==============================] - 75s 352ms/step - loss: 0.1454 - accuracy: 0.9479 - val_loss: 0.5271 - val_accuracy: 0.8465\n",
            "Epoch 6/10\n",
            "214/214 [==============================] - 76s 353ms/step - loss: 0.1094 - accuracy: 0.9614 - val_loss: 0.5866 - val_accuracy: 0.8397\n",
            "Epoch 7/10\n",
            "214/214 [==============================] - 75s 353ms/step - loss: 0.0898 - accuracy: 0.9683 - val_loss: 0.6383 - val_accuracy: 0.8234\n",
            "Epoch 8/10\n",
            "214/214 [==============================] - 75s 352ms/step - loss: 0.1285 - accuracy: 0.9510 - val_loss: 0.6117 - val_accuracy: 0.8288\n",
            "Epoch 9/10\n",
            "214/214 [==============================] - 75s 351ms/step - loss: 0.1021 - accuracy: 0.9620 - val_loss: 0.6012 - val_accuracy: 0.8342\n",
            "Epoch 10/10\n",
            "214/214 [==============================] - 75s 352ms/step - loss: 0.0859 - accuracy: 0.9676 - val_loss: 0.6578 - val_accuracy: 0.8193\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GzzfN0AR6nHs",
        "colab_type": "text"
      },
      "source": [
        "Фактическое количество эпох = 20, после 10 эпохи (на валидации = 0.8465) качество не улучшается. Предсказание меток сделано после 10 эпох."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cQgF4S1mLfcN",
        "colab_type": "code",
        "outputId": "bdb3faf4-6122-4ad6-f0b5-e58f4197355b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        " model.evaluate(validation_data, steps=valid_steps)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "23/23 [==============================] - 4s 191ms/step - loss: 0.4050 - accuracy: 0.8410\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.40503597259521484, 0.8410326242446899]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m4VgEW4btNAi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test = pd.read_csv('/content/gdrive/My Drive/test_tweets.csv', encoding='utf-8')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "goD6iE23tM94",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sentences = test['text'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i9kDFW-3tM8T",
        "colab_type": "code",
        "outputId": "84f159b2-3c11-4463-d852-453cdbfc45b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "input_ids = []\n",
        "\n",
        "for sent in sentences:\n",
        "    encoded_sent =  tf.constant(tokenizer.encode(\n",
        "                        sent,                     \n",
        "                        add_special_tokens = True,\n",
        "                  ))[None, :]\n",
        "    \n",
        "    input_ids.append(encoded_sent)\n",
        "\n",
        "print('Original: ', sentences[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  Just happened a terrible car crash\n",
            "Token IDs: tf.Tensor([[   2  114 1190   21 5803  349 4597    3]], shape=(1, 8), dtype=int32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RHOfrUf-tM6D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "outputs = []\n",
        "n=0\n",
        "for i in input_ids:\n",
        "    n+=1\n",
        "    outputs.append(np.argmax(model(i)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bH2BZHdutM2o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results = pd.DataFrame()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HFV1VWlQtM0a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results['id'] = test['id']\n",
        "results['target'] = outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EH-CZt9PtMxt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results.to_csv('/content/gdrive/My Drive/albert_sub.csv',index=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pajcekSmKUQu",
        "colab_type": "text"
      },
      "source": [
        "### Albert Large"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pAoyuE9wKYoX",
        "colab_type": "code",
        "outputId": "bdaa798d-a0aa-441e-9bcb-bc1589b54e84",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "bd1e739a726b47498e30a75e0defb267",
            "5ceba88d246a43778da8db56f4efcc45",
            "760acbcd5114495ba7aac139f6e8307c",
            "b6e3951bf566435380d4634edd50bb3e",
            "d75dade5d6e94d7c90bf30083d79c3eb",
            "52c678735565411885e4bd266e92d52a",
            "2267b59989ad42bc8639e340f99fdfa1",
            "28e8b53b76674dcda96b7f619e632d15"
          ]
        }
      },
      "source": [
        "from transformers import AlbertTokenizer\n",
        "\n",
        "tokenizer = AlbertTokenizer.from_pretrained(\"albert-large-v1\", do_lower_case=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bd1e739a726b47498e30a75e0defb267",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=760289, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hKOeSbhdKygu",
        "colab_type": "code",
        "outputId": "e8936bee-a834-42bd-9c63-837c7c31f3c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 117,
          "referenced_widgets": [
            "3256b00ff4074fdc9c0b0cc702f587a9",
            "417d263cef464258a46427742110098b",
            "eb770f9a41574b49a99467b29bf2fa01",
            "f1d2b63b0a23487abb745a8b57d07369",
            "110f8f0f6f8f4a82bef03d69f9c5e264",
            "5951f2c1ad7140f8a184a309489060f0",
            "5712645714654e6a953f61bd58184523",
            "e1486f399b174ede8f5e4072a368ac3d",
            "319812155fac4fa7ae7a67de2e014141",
            "2fd4ecec7f0946f885aaea89b5bf06c3",
            "340d139d2b044c7e813ed63a923a25ba",
            "aa7869ccc4e447c79a428ac30acda73e",
            "6b7c9a68853d4ee198f5d021af96e8e0",
            "7424791a8e614eb7921b0af2e2ce2f07",
            "d11a8c7c70204f7a97b258abd6ca5c0f",
            "b5cf001b80b047fca258772018934d36"
          ]
        }
      },
      "source": [
        "from transformers import TFAlbertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "model = TFAlbertForSequenceClassification.from_pretrained(\n",
        "    \"albert-large-v1\",\n",
        "    num_labels = 2, \n",
        "    output_attentions = False,\n",
        "    output_hidden_states = False,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3256b00ff4074fdc9c0b0cc702f587a9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=535, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "319812155fac4fa7ae7a67de2e014141",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=87181048, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "3158a06a-5c5e-4c02-e672-8bff3c966f24",
        "id": "3M6jbatILN0k",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "input_ids = []\n",
        "\n",
        "for sent in sentences:\n",
        "    encoded_sent = tokenizer.encode(\n",
        "                        sent,\n",
        "                        add_special_tokens = True,\n",
        "                  )\n",
        "    \n",
        "    input_ids.append(encoded_sent)\n",
        "\n",
        "print('Original: ', sentences[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  Our Deeds are the Reason of this #earthquake May ALLAH Forgive us all\n",
            "Token IDs: [2, 318, 18055, 50, 14, 1215, 16, 48, 6926, 12322, 5418, 1048, 123, 13, 12209, 8591, 182, 65, 3]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "8527e6ed-f878-46f9-9e71-0ddc93ecded8",
        "id": "5h0y2NMhLN1O",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        }
      },
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "MAX_LEN = 85\n",
        "\n",
        "print('\\nPadding/truncating all sentences to %d values...' % MAX_LEN)\n",
        "\n",
        "print('\\nPadding token: \"{:}\", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))\n",
        "\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", \n",
        "                          value=0, truncating=\"post\", padding=\"post\")\n",
        "\n",
        "print('\\nDone.')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Padding/truncating all sentences to 85 values...\n",
            "\n",
            "Padding token: \"<pad>\", ID: 0\n",
            "\n",
            "Done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bkUEGVp-LN1f",
        "colab": {}
      },
      "source": [
        "attention_masks = []\n",
        "\n",
        "for sent in input_ids:\n",
        "    att_mask = [int(token_id > 0) for token_id in sent]\n",
        "    \n",
        "    attention_masks.append(att_mask)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FHvfW7A5SgRW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "type_masks = []\n",
        "\n",
        "for sent in input_ids:\n",
        "\n",
        "    mask = [0]*MAX_LEN\n",
        "    type_masks.append(mask)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ZkeGZ2YRLN1t",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, \n",
        "                                                            random_state=42, test_size=0.1)\n",
        "train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,\n",
        "                                             random_state=42, test_size=0.1)\n",
        "train_type, validation_type, _, _ = train_test_split(type_masks, labels,\n",
        "                                             random_state=42, test_size=0.1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "uJPt12tILN13",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 32"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qL4At4MoLN1-",
        "colab": {}
      },
      "source": [
        "def train_gen():\n",
        "  for input_id, attention_mask, label_id, token_type_ids in zip(train_inputs, train_masks, train_labels, train_type):\n",
        "    yield(\n",
        "        {                        \n",
        "            \"input_ids\": input_id,\n",
        "            \"attention_mask\": attention_mask,\n",
        "            \"token_type_ids\": token_type_ids,\n",
        "        }, label_id\n",
        "    )\n",
        "\n",
        "def validation_gen():\n",
        "  for input_id, attention_mask, label_id, token_type_ids in zip(validation_inputs, validation_masks, validation_labels, validation_type):\n",
        "    yield(\n",
        "        {                        \n",
        "            \"input_ids\": input_id,\n",
        "            \"attention_mask\": attention_mask,\n",
        "            \"token_type_ids\": token_type_ids,\n",
        "        }, label_id\n",
        "    )\n",
        "\n",
        "train_data = tf.data.Dataset.from_generator(\n",
        "    train_gen, \n",
        "    ({\"input_ids\": tf.int32, \"attention_mask\": tf.int32, \"token_type_ids\": tf.int32}, tf.int64 ),\n",
        "    ({ \n",
        "        \"input_ids\": tf.TensorShape([None]),\n",
        "        \"attention_mask\": tf.TensorShape([None]),\n",
        "        \"token_type_ids\": tf.TensorShape([None]),\n",
        "      },\n",
        "    tf.TensorShape([]),)\n",
        ")\n",
        "\n",
        "validation_data = tf.data.Dataset.from_generator(\n",
        "    validation_gen, \n",
        "    ({\"input_ids\": tf.int32, \"attention_mask\": tf.int32, \"token_type_ids\": tf.int32}, tf.int64 ),\n",
        "    ({ \n",
        "        \"input_ids\": tf.TensorShape([None]),\n",
        "        \"attention_mask\": tf.TensorShape([None]),\n",
        "        \"token_type_ids\": tf.TensorShape([None]),\n",
        "      },\n",
        "    tf.TensorShape([]),)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "p5GiXLqULN2G",
        "colab": {}
      },
      "source": [
        "train_data = train_data.shuffle(64).batch(BATCH_SIZE).repeat(-1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "njbaKMUULN2N",
        "colab": {}
      },
      "source": [
        "validation_data = validation_data.shuffle(64).batch(BATCH_SIZE).repeat(-1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "208689ed-64c8-483e-c0ee-63627bd7a0dd",
        "id": "UxG8xQp2LN2V",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 271
        }
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"tf_albert_for_sequence_classification_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "albert (TFAlbertMainLayer)   multiple                  17683968  \n",
            "_________________________________________________________________\n",
            "dropout_11 (Dropout)         multiple                  0         \n",
            "_________________________________________________________________\n",
            "classifier (Dense)           multiple                  2050      \n",
            "=================================================================\n",
            "Total params: 17,686,018\n",
            "Trainable params: 17,686,018\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JNuHhv4kLN2c",
        "colab": {}
      },
      "source": [
        "opt = tf.keras.optimizers.Adam(learning_rate=2e-5, epsilon=1e-08)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "463WWIRwLN2m",
        "colab": {}
      },
      "source": [
        "if len(np.unique(labels)) > 1:\n",
        "  loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "BSWOPvp5LN2t",
        "colab": {}
      },
      "source": [
        "metric = tf.keras.metrics.SparseCategoricalAccuracy(\"accuracy\")\n",
        "model.compile(optimizer=opt, loss=loss, metrics=[metric])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "21102e82-c860-4014-a649-c1c38486fd0f",
        "id": "Ap77vxlELN24",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        }
      },
      "source": [
        "train_examples = len(train_inputs)\n",
        "valid_examples = len(validation_inputs)\n",
        "\n",
        "print('In training set: {}'.format(train_examples))\n",
        "print('In validation set: {}'.format(valid_examples))\n",
        "train_steps = train_examples // BATCH_SIZE\n",
        "valid_steps = valid_examples // BATCH_SIZE\n",
        "\n",
        "print('Count of steps:\\n\\tTraining set: {} \\n\\tValidation set: {}'.format(train_steps, valid_steps))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "In training set: 6851\n",
            "In validation set: 762\n",
            "Count of steps:\n",
            "\tTraining set: 214 \n",
            "\tValidation set: 23\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "BoWePvYCLN2_",
        "colab": {}
      },
      "source": [
        "EPOCHS = 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i0Dxe7Xu9U9M",
        "colab_type": "text"
      },
      "source": [
        "Так же, как и в первом случае, фактическое количество эпох = 20, максимальное качество получаем на 11, однако оно хуже лучшего по модели base."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "1e35291f-d51c-4c15-85d0-bae099e21124",
        "id": "raROFXJpLN3G",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 399
        }
      },
      "source": [
        "model.fit(\n",
        "    train_data,\n",
        "    epochs=EPOCHS,\n",
        "    steps_per_epoch=train_steps,\n",
        "    validation_data=validation_data,\n",
        "    validation_steps=valid_steps\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "214/214 [==============================] - 221s 1s/step - loss: 0.3909 - accuracy: 0.8375 - val_loss: 0.4368 - val_accuracy: 0.8193\n",
            "Epoch 2/10\n",
            "214/214 [==============================] - 221s 1s/step - loss: 0.3599 - accuracy: 0.8554 - val_loss: 0.4664 - val_accuracy: 0.8057\n",
            "Epoch 3/10\n",
            "214/214 [==============================] - 220s 1s/step - loss: 0.4410 - accuracy: 0.8086 - val_loss: 0.4755 - val_accuracy: 0.8043\n",
            "Epoch 4/10\n",
            "214/214 [==============================] - 221s 1s/step - loss: 0.3540 - accuracy: 0.8556 - val_loss: 0.5024 - val_accuracy: 0.7989\n",
            "Epoch 5/10\n",
            "214/214 [==============================] - 221s 1s/step - loss: 0.3221 - accuracy: 0.8711 - val_loss: 0.5024 - val_accuracy: 0.8003\n",
            "Epoch 6/10\n",
            "214/214 [==============================] - 221s 1s/step - loss: 0.2982 - accuracy: 0.8836 - val_loss: 0.5142 - val_accuracy: 0.7989\n",
            "Epoch 7/10\n",
            "214/214 [==============================] - 221s 1s/step - loss: 0.2773 - accuracy: 0.8984 - val_loss: 0.5285 - val_accuracy: 0.7935\n",
            "Epoch 8/10\n",
            "214/214 [==============================] - 221s 1s/step - loss: 0.2611 - accuracy: 0.9032 - val_loss: 0.5188 - val_accuracy: 0.7894\n",
            "Epoch 9/10\n",
            "214/214 [==============================] - 221s 1s/step - loss: 0.2564 - accuracy: 0.9032 - val_loss: 0.5940 - val_accuracy: 0.7908\n",
            "Epoch 10/10\n",
            "214/214 [==============================] - 220s 1s/step - loss: 0.2260 - accuracy: 0.9195 - val_loss: 0.6241 - val_accuracy: 0.7908\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f29a715ab70>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "odeKMQR_HeGD",
        "colab_type": "text"
      },
      "source": [
        "## RoBERTa Large\n",
        "\n",
        "RoBERTa base дает схожие с Albert large значения качества, поэтому сразу применим модель large."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R5S9i06-HfrC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from transformers import RobertaTokenizer\n",
        "from transformers import TFRobertaForSequenceClassification"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "teeoux2FPn8c",
        "colab_type": "code",
        "outputId": "0b6a1fd5-4c09-4872-a299-d163f30bf9b7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218,
          "referenced_widgets": [
            "c4d48e7604b3491d86b4eae738de53db",
            "fc015023ebcd491c9296e18e0a464389",
            "ba5f50bec2404f1d8c1553787c1233cd",
            "345918c2c56a472ea65b754551de8f2e",
            "5db08d4f2cac4f9aaec8f50c397bc1bb",
            "a844628eb4b642718b9e72b3a8ba59ff",
            "5f1820c392e545609ac013b90aeeac96",
            "67e425b2fda44e51848798e3046fafe8",
            "e101d5909ef24d6685b50dd9075cd3a5",
            "861c33c149a649ada58255e6c55c3d13",
            "fcfd6939a1d140d1819312b22c12246b",
            "0c0f0dcab70943e4acd48db536dd0931",
            "e5adc77bd5dd439d979516b7d9a1578d",
            "08c94cb1d9464ed1ae4c7a9a1cdbb5ca",
            "d80d9edeb14040e294a624ac532ba2c9",
            "64a86efec80d42439c3e1d0f8fc006f7",
            "b55753c0bee14c1592445e73e1586214",
            "2d41c2f13d08471b89e79adf2b950cfd",
            "55afc3097bee496f8bfe593e2b9abc2a",
            "f4daa31329864f48b4336d70cf76330d",
            "23fe778feb6b40a1831bea82542eec10",
            "206454c229c7408d8c506000bf21ba84",
            "7ea630e29a53490cb21a78365c571b34",
            "e8e44f294ed64b7a86d30033eb215598",
            "349a991fce5042dd9bd5ea81a0d78906",
            "c589a0243a2549e58615f0e0118bf67a",
            "fed87a132cff405d8f0f95abccc878e1",
            "a02b269ffc9b4c9eb212bd08f8cc7053",
            "a6c68da98ac549648e6d99f66585ee5a",
            "48bbe146d1cf4b1eb8fd1c1118bd3494",
            "8c23f3016a604464b8418fb99f080c25",
            "bcaefdf33b33444198ac86a193ed23f9"
          ]
        }
      },
      "source": [
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-large', do_lower_case=True)\n",
        "model = TFRobertaForSequenceClassification.from_pretrained('roberta-large', num_labels = 2, output_attentions = False, output_hidden_states = False)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c4d48e7604b3491d86b4eae738de53db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=898823, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e101d5909ef24d6685b50dd9075cd3a5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=456318, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b55753c0bee14c1592445e73e1586214",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=525, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "349a991fce5042dd9bd5ea81a0d78906",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=1634375628, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZU1eP2vaSwXN",
        "colab_type": "text"
      },
      "source": [
        "### tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gD_DslAYUee-",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/DeepLearning/Bert/twitter/train.csv', encoding='utf-8')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "QrvPxqj_Uefh",
        "colab": {}
      },
      "source": [
        "sentences = df['text'].values\n",
        "labels = df['target'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "04e27b85-ed9b-4930-87cb-62cd870e08df",
        "id": "GrPy1hHIS0XV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    encoded_sent = tokenizer.encode(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                  )\n",
        "    \n",
        "    # Add the encoded sentence to the list.\n",
        "    input_ids.append(encoded_sent)\n",
        "\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print('Original: ', sentences[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  Our Deeds are the Reason of this #earthquake May ALLAH Forgive us all\n",
            "Token IDs: [0, 84, 28519, 32, 5, 1219, 9, 42, 849, 25581, 2253, 5113, 189, 70, 895, 20184, 201, 70, 2]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "db556074-970d-4020-d6da-123fe5ac2afd",
        "id": "n2lTVWljS0Xu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 144
        }
      },
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "MAX_LEN = 85\n",
        "\n",
        "print('\\nPadding/truncating all sentences to %d values...' % MAX_LEN)\n",
        "\n",
        "print('\\nPadding token: \"{:}\", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))\n",
        "\n",
        "# Pad our input tokens with value 0.\n",
        "# \"post\" indicates that we want to pad and truncate at the end of the sequence,\n",
        "# as opposed to the beginning.\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", \n",
        "                          value=0, truncating=\"post\", padding=\"post\")\n",
        "\n",
        "print('\\nDone.')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Padding/truncating all sentences to 85 values...\n",
            "\n",
            "Padding token: \"<pad>\", ID: 1\n",
            "\n",
            "Done.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JF7vtD3yS0X-",
        "colab": {}
      },
      "source": [
        "# Create attention masks\n",
        "attention_masks = []\n",
        "\n",
        "# For each sentence...\n",
        "for sent in input_ids:\n",
        "    \n",
        "    # Create the attention mask.\n",
        "    #   - If a token ID is 0, then it's padding, set the mask to 0.\n",
        "    #   - If a token ID is > 0, then it's a real token, set the mask to 1.\n",
        "    att_mask = [int(token_id > 0) for token_id in sent]\n",
        "    \n",
        "    # Store the attention mask for this sentence.\n",
        "    attention_masks.append(att_mask)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sFLjyeI7S0YH",
        "colab": {}
      },
      "source": [
        "# Use train_test_split to split our data into train and validation sets for\n",
        "# training\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Use 90% for training and 10% for validation.\n",
        "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, \n",
        "                                                            random_state=42, test_size=0.1)\n",
        "# Do the same for the masks.\n",
        "train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,\n",
        "                                             random_state=42, test_size=0.1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4bXOiRMiS0YQ",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 32"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "VJKsrUT5S0YX",
        "colab": {}
      },
      "source": [
        "def train_gen():\n",
        "  for input_id, attention_mask, label_id in zip(train_inputs, train_masks, train_labels ):\n",
        "    yield(\n",
        "        {                        \n",
        "            \"input_ids\": input_id,\n",
        "            \"attention_mask\": attention_mask,\n",
        "        }, label_id\n",
        "    )\n",
        "\n",
        "def validation_gen():\n",
        "  for input_id, attention_mask, label_id in zip(validation_inputs, validation_masks, validation_labels):\n",
        "    yield(\n",
        "        {                        \n",
        "            \"input_ids\": input_id,\n",
        "            \"attention_mask\": attention_mask,\n",
        "        }, label_id\n",
        "    )\n",
        "\n",
        "\n",
        "train_data = tf.data.Dataset.from_generator(\n",
        "    train_gen, \n",
        "    ({\"input_ids\": tf.int32, \"attention_mask\": tf.int32}, tf.int64 ),\n",
        "    ({ \n",
        "        \"input_ids\": tf.TensorShape([None]),\n",
        "        \"attention_mask\": tf.TensorShape([None]),\n",
        "      },\n",
        "    tf.TensorShape([]),)\n",
        ")\n",
        "\n",
        "validation_data = tf.data.Dataset.from_generator(\n",
        "    validation_gen, \n",
        "    ({\"input_ids\": tf.int32, \"attention_mask\": tf.int32}, tf.int64 ),\n",
        "    ({ \n",
        "        \"input_ids\": tf.TensorShape([None]),\n",
        "        \"attention_mask\": tf.TensorShape([None]),\n",
        "      },\n",
        "    tf.TensorShape([]),)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "wEJ1K659S0Yg",
        "colab": {}
      },
      "source": [
        "train_data = train_data.shuffle(64).batch(BATCH_SIZE).repeat(-1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "h_ORM_T2S0Yo",
        "colab": {}
      },
      "source": [
        "validation_data = validation_data.shuffle(64).batch(BATCH_SIZE).repeat(-1)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Iu43ikLkSyv4",
        "colab_type": "text"
      },
      "source": [
        "### model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "b3fd4189-3344-441c-c5fa-3ef948168d16",
        "id": "h7ffRVV4Hg3X",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        }
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"tf_roberta_for_sequence_classification\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "roberta (TFRobertaMainLayer) multiple                  355359744 \n",
            "_________________________________________________________________\n",
            "classifier (TFRobertaClassif multiple                  1051650   \n",
            "=================================================================\n",
            "Total params: 356,411,394\n",
            "Trainable params: 356,411,394\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bZhC6qy0Hg3o",
        "colab": {}
      },
      "source": [
        "opt = tf.keras.optimizers.Adam(learning_rate=2e-5, epsilon=1e-08)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "aT2N8YLMHg3x",
        "colab": {}
      },
      "source": [
        "if len(np.unique(labels)) > 1:\n",
        "  loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6KcO1Kv9hs0i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "TveXD1vBHg32",
        "colab": {}
      },
      "source": [
        "metric = tf.keras.metrics.SparseCategoricalAccuracy(\"accuracy\")\n",
        "model.compile(optimizer=opt, loss=loss, metrics=[metric])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "c0d306e6-65f3-4149-b776-56be46c56d75",
        "id": "O9AWhRyyHg3-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        }
      },
      "source": [
        "train_examples = len(train_inputs)\n",
        "valid_examples = len(validation_inputs)\n",
        "\n",
        "print('In training set: {}'.format(train_examples))\n",
        "print('In validation set: {}'.format(valid_examples))\n",
        "# Train and evaluate using tf.keras.Model.fit()\n",
        "train_steps = train_examples // BATCH_SIZE\n",
        "valid_steps = valid_examples // BATCH_SIZE\n",
        "\n",
        "print('Count of steps:\\n\\tTraining set: {} \\n\\tValidation set: {}'.format(train_steps, valid_steps))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "In training set: 6851\n",
            "In validation set: 762\n",
            "Count of steps:\n",
            "\tTraining set: 214 \n",
            "\tValidation set: 23\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7qpkhrJ4Hg4E",
        "colab": {}
      },
      "source": [
        "EPOCHS = 2"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "5b5b3bfd-13e0-474c-a546-93f233186c20",
        "id": "vtDr-UY4Hg4M",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164
        }
      },
      "source": [
        "model.fit(\n",
        "    train_data,\n",
        "    epochs=EPOCHS,\n",
        "    steps_per_epoch=train_steps,\n",
        "    validation_data=validation_data,\n",
        "    validation_steps=valid_steps\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_roberta_for_sequence_classification/roberta/pooler/dense/kernel:0', 'tf_roberta_for_sequence_classification/roberta/pooler/dense/bias:0'] when minimizing the loss.\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_roberta_for_sequence_classification/roberta/pooler/dense/kernel:0', 'tf_roberta_for_sequence_classification/roberta/pooler/dense/bias:0'] when minimizing the loss.\n",
            "214/214 [==============================] - 233s 1s/step - loss: 0.4798 - accuracy: 0.7821 - val_loss: 0.3921 - val_accuracy: 0.8383\n",
            "Epoch 2/2\n",
            "214/214 [==============================] - 228s 1s/step - loss: 0.3896 - accuracy: 0.8403 - val_loss: 0.3849 - val_accuracy: 0.8451\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f73ba414a90>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zlfZoMxw913d",
        "colab_type": "text"
      },
      "source": [
        "После 2 эпохи качество падает до 0.6 и не поднимается выше полученного на второй. Предскажем значения для тестовой выборки."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YphEtm_Kv4am",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test = pd.read_csv('/content/gdrive/My Drive/Colab Notebooks/DeepLearning/Bert/twitter/test.csv', encoding='utf-8')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "1hD1Pk4p0HZM",
        "colab": {}
      },
      "source": [
        "sentences = test['text'].values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "outputId": "491785f1-ac48-4019-e24c-b89f3a455ca1",
        "id": "toHmIlgT0HZl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "\n",
        "for sent in sentences:\n",
        "    encoded_sent =  tf.constant(tokenizer.encode(\n",
        "                        sent,                     \n",
        "                        add_special_tokens = True,\n",
        "                  ))[None, :]\n",
        "    \n",
        "    input_ids.append(encoded_sent)\n",
        "\n",
        "print('Original: ', sentences[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  Just happened a terrible car crash\n",
            "Token IDs: tf.Tensor([[   0   95 1102   10 6587  512 2058    2]], shape=(1, 8), dtype=int32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X0nkyJQg_zPT",
        "colab_type": "code",
        "outputId": "b2753e52-655e-4f96-e931-ac38463eeba7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "outputs = []\n",
        "n=0\n",
        "for i in input_ids:\n",
        "    n+=1\n",
        "    print(n, end='-')\n",
        "    outputs.append(np.argmax(model(i)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1-2-3-4-5-6-7-8-9-10-11-12-13-14-15-16-17-18-19-20-21-22-23-24-25-26-27-28-29-30-31-32-33-34-35-36-37-38-39-40-41-42-43-44-45-46-47-48-49-50-51-52-53-54-55-56-57-58-59-60-61-62-63-64-65-66-67-68-69-70-71-72-73-74-75-76-77-78-79-80-81-82-83-84-85-86-87-88-89-90-91-92-93-94-95-96-97-98-99-100-101-102-103-104-105-106-107-108-109-110-111-112-113-114-115-116-117-118-119-120-121-122-123-124-125-126-127-128-129-130-131-132-133-134-135-136-137-138-139-140-141-142-143-144-145-146-147-148-149-150-151-152-153-154-155-156-157-158-159-160-161-162-163-164-165-166-167-168-169-170-171-172-173-174-175-176-177-178-179-180-181-182-183-184-185-186-187-188-189-190-191-192-193-194-195-196-197-198-199-200-201-202-203-204-205-206-207-208-209-210-211-212-213-214-215-216-217-218-219-220-221-222-223-224-225-226-227-228-229-230-231-232-233-234-235-236-237-238-239-240-241-242-243-244-245-246-247-248-249-250-251-252-253-254-255-256-257-258-259-260-261-262-263-264-265-266-267-268-269-270-271-272-273-274-275-276-277-278-279-280-281-282-283-284-285-286-287-288-289-290-291-292-293-294-295-296-297-298-299-300-301-302-303-304-305-306-307-308-309-310-311-312-313-314-315-316-317-318-319-320-321-322-323-324-325-326-327-328-329-330-331-332-333-334-335-336-337-338-339-340-341-342-343-344-345-346-347-348-349-350-351-352-353-354-355-356-357-358-359-360-361-362-363-364-365-366-367-368-369-370-371-372-373-374-375-376-377-378-379-380-381-382-383-384-385-386-387-388-389-390-391-392-393-394-395-396-397-398-399-400-401-402-403-404-405-406-407-408-409-410-411-412-413-414-415-416-417-418-419-420-421-422-423-424-425-426-427-428-429-430-431-432-433-434-435-436-437-438-439-440-441-442-443-444-445-446-447-448-449-450-451-452-453-454-455-456-457-458-459-460-461-462-463-464-465-466-467-468-469-470-471-472-473-474-475-476-477-478-479-480-481-482-483-484-485-486-487-488-489-490-491-492-493-494-495-496-497-498-499-500-501-502-503-504-505-506-507-508-509-510-511-512-513-514-515-516-517-518-519-520-521-522-523-524-525-526-527-528-529-530-531-532-533-534-535-536-537-538-539-540-541-542-543-544-545-546-547-548-549-550-551-552-553-554-555-556-557-558-559-560-561-562-563-564-565-566-567-568-569-570-571-572-573-574-575-576-577-578-579-580-581-582-583-584-585-586-587-588-589-590-591-592-593-594-595-596-597-598-599-600-601-602-603-604-605-606-607-608-609-610-611-612-613-614-615-616-617-618-619-620-621-622-623-624-625-626-627-628-629-630-631-632-633-634-635-636-637-638-639-640-641-642-643-644-645-646-647-648-649-650-651-652-653-654-655-656-657-658-659-660-661-662-663-664-665-666-667-668-669-670-671-672-673-674-675-676-677-678-679-680-681-682-683-684-685-686-687-688-689-690-691-692-693-694-695-696-697-698-699-700-701-702-703-704-705-706-707-708-709-710-711-712-713-714-715-716-717-718-719-720-721-722-723-724-725-726-727-728-729-730-731-732-733-734-735-736-737-738-739-740-741-742-743-744-745-746-747-748-749-750-751-752-753-754-755-756-757-758-759-760-761-762-763-764-765-766-767-768-769-770-771-772-773-774-775-776-777-778-779-780-781-782-783-784-785-786-787-788-789-790-791-792-793-794-795-796-797-798-799-800-801-802-803-804-805-806-807-808-809-810-811-812-813-814-815-816-817-818-819-820-821-822-823-824-825-826-827-828-829-830-831-832-833-834-835-836-837-838-839-840-841-842-843-844-845-846-847-848-849-850-851-852-853-854-855-856-857-858-859-860-861-862-863-864-865-866-867-868-869-870-871-872-873-874-875-876-877-878-879-880-881-882-883-884-885-886-887-888-889-890-891-892-893-894-895-896-897-898-899-900-901-902-903-904-905-906-907-908-909-910-911-912-913-914-915-916-917-918-919-920-921-922-923-924-925-926-927-928-929-930-931-932-933-934-935-936-937-938-939-940-941-942-943-944-945-946-947-948-949-950-951-952-953-954-955-956-957-958-959-960-961-962-963-964-965-966-967-968-969-970-971-972-973-974-975-976-977-978-979-980-981-982-983-984-985-986-987-988-989-990-991-992-993-994-995-996-997-998-999-1000-1001-1002-1003-1004-1005-1006-1007-1008-1009-1010-1011-1012-1013-1014-1015-1016-1017-1018-1019-1020-1021-1022-1023-1024-1025-1026-1027-1028-1029-1030-1031-1032-1033-1034-1035-1036-1037-1038-1039-1040-1041-1042-1043-1044-1045-1046-1047-1048-1049-1050-1051-1052-1053-1054-1055-1056-1057-1058-1059-1060-1061-1062-1063-1064-1065-1066-1067-1068-1069-1070-1071-1072-1073-1074-1075-1076-1077-1078-1079-1080-1081-1082-1083-1084-1085-1086-1087-1088-1089-1090-1091-1092-1093-1094-1095-1096-1097-1098-1099-1100-1101-1102-1103-1104-1105-1106-1107-1108-1109-1110-1111-1112-1113-1114-1115-1116-1117-1118-1119-1120-1121-1122-1123-1124-1125-1126-1127-1128-1129-1130-1131-1132-1133-1134-1135-1136-1137-1138-1139-1140-1141-1142-1143-1144-1145-1146-1147-1148-1149-1150-1151-1152-1153-1154-1155-1156-1157-1158-1159-1160-1161-1162-1163-1164-1165-1166-1167-1168-1169-1170-1171-1172-1173-1174-1175-1176-1177-1178-1179-1180-1181-1182-1183-1184-1185-1186-1187-1188-1189-1190-1191-1192-1193-1194-1195-1196-1197-1198-1199-1200-1201-1202-1203-1204-1205-1206-1207-1208-1209-1210-1211-1212-1213-1214-1215-1216-1217-1218-1219-1220-1221-1222-1223-1224-1225-1226-1227-1228-1229-1230-1231-1232-1233-1234-1235-1236-1237-1238-1239-1240-1241-1242-1243-1244-1245-1246-1247-1248-1249-1250-1251-1252-1253-1254-1255-1256-1257-1258-1259-1260-1261-1262-1263-1264-1265-1266-1267-1268-1269-1270-1271-1272-1273-1274-1275-1276-1277-1278-1279-1280-1281-1282-1283-1284-1285-1286-1287-1288-1289-1290-1291-1292-1293-1294-1295-1296-1297-1298-1299-1300-1301-1302-1303-1304-1305-1306-1307-1308-1309-1310-1311-1312-1313-1314-1315-1316-1317-1318-1319-1320-1321-1322-1323-1324-1325-1326-1327-1328-1329-1330-1331-1332-1333-1334-1335-1336-1337-1338-1339-1340-1341-1342-1343-1344-1345-1346-1347-1348-1349-1350-1351-1352-1353-1354-1355-1356-1357-1358-1359-1360-1361-1362-1363-1364-1365-1366-1367-1368-1369-1370-1371-1372-1373-1374-1375-1376-1377-1378-1379-1380-1381-1382-1383-1384-1385-1386-1387-1388-1389-1390-1391-1392-1393-1394-1395-1396-1397-1398-1399-1400-1401-1402-1403-1404-1405-1406-1407-1408-1409-1410-1411-1412-1413-1414-1415-1416-1417-1418-1419-1420-1421-1422-1423-1424-1425-1426-1427-1428-1429-1430-1431-1432-1433-1434-1435-1436-1437-1438-1439-1440-1441-1442-1443-1444-1445-1446-1447-1448-1449-1450-1451-1452-1453-1454-1455-1456-1457-1458-1459-1460-1461-1462-1463-1464-1465-1466-1467-1468-1469-1470-1471-1472-1473-1474-1475-1476-1477-1478-1479-1480-1481-1482-1483-1484-1485-1486-1487-1488-1489-1490-1491-1492-1493-1494-1495-1496-1497-1498-1499-1500-1501-1502-1503-1504-1505-1506-1507-1508-1509-1510-1511-1512-1513-1514-1515-1516-1517-1518-1519-1520-1521-1522-1523-1524-1525-1526-1527-1528-1529-1530-1531-1532-1533-1534-1535-1536-1537-1538-1539-1540-1541-1542-1543-1544-1545-1546-1547-1548-1549-1550-1551-1552-1553-1554-1555-1556-1557-1558-1559-1560-1561-1562-1563-1564-1565-1566-1567-1568-1569-1570-1571-1572-1573-1574-1575-1576-1577-1578-1579-1580-1581-1582-1583-1584-1585-1586-1587-1588-1589-1590-1591-1592-1593-1594-1595-1596-1597-1598-1599-1600-1601-1602-1603-1604-1605-1606-1607-1608-1609-1610-1611-1612-1613-1614-1615-1616-1617-1618-1619-1620-1621-1622-1623-1624-1625-1626-1627-1628-1629-1630-1631-1632-1633-1634-1635-1636-1637-1638-1639-1640-1641-1642-1643-1644-1645-1646-1647-1648-1649-1650-1651-1652-1653-1654-1655-1656-1657-1658-1659-1660-1661-1662-1663-1664-1665-1666-1667-1668-1669-1670-1671-1672-1673-1674-1675-1676-1677-1678-1679-1680-1681-1682-1683-1684-1685-1686-1687-1688-1689-1690-1691-1692-1693-1694-1695-1696-1697-1698-1699-1700-1701-1702-1703-1704-1705-1706-1707-1708-1709-1710-1711-1712-1713-1714-1715-1716-1717-1718-1719-1720-1721-1722-1723-1724-1725-1726-1727-1728-1729-1730-1731-1732-1733-1734-1735-1736-1737-1738-1739-1740-1741-1742-1743-1744-1745-1746-1747-1748-1749-1750-1751-1752-1753-1754-1755-1756-1757-1758-1759-1760-1761-1762-1763-1764-1765-1766-1767-1768-1769-1770-1771-1772-1773-1774-1775-1776-1777-1778-1779-1780-1781-1782-1783-1784-1785-1786-1787-1788-1789-1790-1791-1792-1793-1794-1795-1796-1797-1798-1799-1800-1801-1802-1803-1804-1805-1806-1807-1808-1809-1810-1811-1812-1813-1814-1815-1816-1817-1818-1819-1820-1821-1822-1823-1824-1825-1826-1827-1828-1829-1830-1831-1832-1833-1834-1835-1836-1837-1838-1839-1840-1841-1842-1843-1844-1845-1846-1847-1848-1849-1850-1851-1852-1853-1854-1855-1856-1857-1858-1859-1860-1861-1862-1863-1864-1865-1866-1867-1868-1869-1870-1871-1872-1873-1874-1875-1876-1877-1878-1879-1880-1881-1882-1883-1884-1885-1886-1887-1888-1889-1890-1891-1892-1893-1894-1895-1896-1897-1898-1899-1900-1901-1902-1903-1904-1905-1906-1907-1908-1909-1910-1911-1912-1913-1914-1915-1916-1917-1918-1919-1920-1921-1922-1923-1924-1925-1926-1927-1928-1929-1930-1931-1932-1933-1934-1935-1936-1937-1938-1939-1940-1941-1942-1943-1944-1945-1946-1947-1948-1949-1950-1951-1952-1953-1954-1955-1956-1957-1958-1959-1960-1961-1962-1963-1964-1965-1966-1967-1968-1969-1970-1971-1972-1973-1974-1975-1976-1977-1978-1979-1980-1981-1982-1983-1984-1985-1986-1987-1988-1989-1990-1991-1992-1993-1994-1995-1996-1997-1998-1999-2000-2001-2002-2003-2004-2005-2006-2007-2008-2009-2010-2011-2012-2013-2014-2015-2016-2017-2018-2019-2020-2021-2022-2023-2024-2025-2026-2027-2028-2029-2030-2031-2032-2033-2034-2035-2036-2037-2038-2039-2040-2041-2042-2043-2044-2045-2046-2047-2048-2049-2050-2051-2052-2053-2054-2055-2056-2057-2058-2059-2060-2061-2062-2063-2064-2065-2066-2067-2068-2069-2070-2071-2072-2073-2074-2075-2076-2077-2078-2079-2080-2081-2082-2083-2084-2085-2086-2087-2088-2089-2090-2091-2092-2093-2094-2095-2096-2097-2098-2099-2100-2101-2102-2103-2104-2105-2106-2107-2108-2109-2110-2111-2112-2113-2114-2115-2116-2117-2118-2119-2120-2121-2122-2123-2124-2125-2126-2127-2128-2129-2130-2131-2132-2133-2134-2135-2136-2137-2138-2139-2140-2141-2142-2143-2144-2145-2146-2147-2148-2149-2150-2151-2152-2153-2154-2155-2156-2157-2158-2159-2160-2161-2162-2163-2164-2165-2166-2167-2168-2169-2170-2171-2172-2173-2174-2175-2176-2177-2178-2179-2180-2181-2182-2183-2184-2185-2186-2187-2188-2189-2190-2191-2192-2193-2194-2195-2196-2197-2198-2199-2200-2201-2202-2203-2204-2205-2206-2207-2208-2209-2210-2211-2212-2213-2214-2215-2216-2217-2218-2219-2220-2221-2222-2223-2224-2225-2226-2227-2228-2229-2230-2231-2232-2233-2234-2235-2236-2237-2238-2239-2240-2241-2242-2243-2244-2245-2246-2247-2248-2249-2250-2251-2252-2253-2254-2255-2256-2257-2258-2259-2260-2261-2262-2263-2264-2265-2266-2267-2268-2269-2270-2271-2272-2273-2274-2275-2276-2277-2278-2279-2280-2281-2282-2283-2284-2285-2286-2287-2288-2289-2290-2291-2292-2293-2294-2295-2296-2297-2298-2299-2300-2301-2302-2303-2304-2305-2306-2307-2308-2309-2310-2311-2312-2313-2314-2315-2316-2317-2318-2319-2320-2321-2322-2323-2324-2325-2326-2327-2328-2329-2330-2331-2332-2333-2334-2335-2336-2337-2338-2339-2340-2341-2342-2343-2344-2345-2346-2347-2348-2349-2350-2351-2352-2353-2354-2355-2356-2357-2358-2359-2360-2361-2362-2363-2364-2365-2366-2367-2368-2369-2370-2371-2372-2373-2374-2375-2376-2377-2378-2379-2380-2381-2382-2383-2384-2385-2386-2387-2388-2389-2390-2391-2392-2393-2394-2395-2396-2397-2398-2399-2400-2401-2402-2403-2404-2405-2406-2407-2408-2409-2410-2411-2412-2413-2414-2415-2416-2417-2418-2419-2420-2421-2422-2423-2424-2425-2426-2427-2428-2429-2430-2431-2432-2433-2434-2435-2436-2437-2438-2439-2440-2441-2442-2443-2444-2445-2446-2447-2448-2449-2450-2451-2452-2453-2454-2455-2456-2457-2458-2459-2460-2461-2462-2463-2464-2465-2466-2467-2468-2469-2470-2471-2472-2473-2474-2475-2476-2477-2478-2479-2480-2481-2482-2483-2484-2485-2486-2487-2488-2489-2490-2491-2492-2493-2494-2495-2496-2497-2498-2499-2500-2501-2502-2503-2504-2505-2506-2507-2508-2509-2510-2511-2512-2513-2514-2515-2516-2517-2518-2519-2520-2521-2522-2523-2524-2525-2526-2527-2528-2529-2530-2531-2532-2533-2534-2535-2536-2537-2538-2539-2540-2541-2542-2543-2544-2545-2546-2547-2548-2549-2550-2551-2552-2553-2554-2555-2556-2557-2558-2559-2560-2561-2562-2563-2564-2565-2566-2567-2568-2569-2570-2571-2572-2573-2574-2575-2576-2577-2578-2579-2580-2581-2582-2583-2584-2585-2586-2587-2588-2589-2590-2591-2592-2593-2594-2595-2596-2597-2598-2599-2600-2601-2602-2603-2604-2605-2606-2607-2608-2609-2610-2611-2612-2613-2614-2615-2616-2617-2618-2619-2620-2621-2622-2623-2624-2625-2626-2627-2628-2629-2630-2631-2632-2633-2634-2635-2636-2637-2638-2639-2640-2641-2642-2643-2644-2645-2646-2647-2648-2649-2650-2651-2652-2653-2654-2655-2656-2657-2658-2659-2660-2661-2662-2663-2664-2665-2666-2667-2668-2669-2670-2671-2672-2673-2674-2675-2676-2677-2678-2679-2680-2681-2682-2683-2684-2685-2686-2687-2688-2689-2690-2691-2692-2693-2694-2695-2696-2697-2698-2699-2700-2701-2702-2703-2704-2705-2706-2707-2708-2709-2710-2711-2712-2713-2714-2715-2716-2717-2718-2719-2720-2721-2722-2723-2724-2725-2726-2727-2728-2729-2730-2731-2732-2733-2734-2735-2736-2737-2738-2739-2740-2741-2742-2743-2744-2745-2746-2747-2748-2749-2750-2751-2752-2753-2754-2755-2756-2757-2758-2759-2760-2761-2762-2763-2764-2765-2766-2767-2768-2769-2770-2771-2772-2773-2774-2775-2776-2777-2778-2779-2780-2781-2782-2783-2784-2785-2786-2787-2788-2789-2790-2791-2792-2793-2794-2795-2796-2797-2798-2799-2800-2801-2802-2803-2804-2805-2806-2807-2808-2809-2810-2811-2812-2813-2814-2815-2816-2817-2818-2819-2820-2821-2822-2823-2824-2825-2826-2827-2828-2829-2830-2831-2832-2833-2834-2835-2836-2837-2838-2839-2840-2841-2842-2843-2844-2845-2846-2847-2848-2849-2850-2851-2852-2853-2854-2855-2856-2857-2858-2859-2860-2861-2862-2863-2864-2865-2866-2867-2868-2869-2870-2871-2872-2873-2874-2875-2876-2877-2878-2879-2880-2881-2882-2883-2884-2885-2886-2887-2888-2889-2890-2891-2892-2893-2894-2895-2896-2897-2898-2899-2900-2901-2902-2903-2904-2905-2906-2907-2908-2909-2910-2911-2912-2913-2914-2915-2916-2917-2918-2919-2920-2921-2922-2923-2924-2925-2926-2927-2928-2929-2930-2931-2932-2933-2934-2935-2936-2937-2938-2939-2940-2941-2942-2943-2944-2945-2946-2947-2948-2949-2950-2951-2952-2953-2954-2955-2956-2957-2958-2959-2960-2961-2962-2963-2964-2965-2966-2967-2968-2969-2970-2971-2972-2973-2974-2975-2976-2977-2978-2979-2980-2981-2982-2983-2984-2985-2986-2987-2988-2989-2990-2991-2992-2993-2994-2995-2996-2997-2998-2999-3000-3001-3002-3003-3004-3005-3006-3007-3008-3009-3010-3011-3012-3013-3014-3015-3016-3017-3018-3019-3020-3021-3022-3023-3024-3025-3026-3027-3028-3029-3030-3031-3032-3033-3034-3035-3036-3037-3038-3039-3040-3041-3042-3043-3044-3045-3046-3047-3048-3049-3050-3051-3052-3053-3054-3055-3056-3057-3058-3059-3060-3061-3062-3063-3064-3065-3066-3067-3068-3069-3070-3071-3072-3073-3074-3075-3076-3077-3078-3079-3080-3081-3082-3083-3084-3085-3086-3087-3088-3089-3090-3091-3092-3093-3094-3095-3096-3097-3098-3099-3100-3101-3102-3103-3104-3105-3106-3107-3108-3109-3110-3111-3112-3113-3114-3115-3116-3117-3118-3119-3120-3121-3122-3123-3124-3125-3126-3127-3128-3129-3130-3131-3132-3133-3134-3135-3136-3137-3138-3139-3140-3141-3142-3143-3144-3145-3146-3147-3148-3149-3150-3151-3152-3153-3154-3155-3156-3157-3158-3159-3160-3161-3162-3163-3164-3165-3166-3167-3168-3169-3170-3171-3172-3173-3174-3175-3176-3177-3178-3179-3180-3181-3182-3183-3184-3185-3186-3187-3188-3189-3190-3191-3192-3193-3194-3195-3196-3197-3198-3199-3200-3201-3202-3203-3204-3205-3206-3207-3208-3209-3210-3211-3212-3213-3214-3215-3216-3217-3218-3219-3220-3221-3222-3223-3224-3225-3226-3227-3228-3229-3230-3231-3232-3233-3234-3235-3236-3237-3238-3239-3240-3241-3242-3243-3244-3245-3246-3247-3248-3249-3250-3251-3252-3253-3254-3255-3256-3257-3258-3259-3260-3261-3262-3263-"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7w5N-kW1_6xZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results = pd.DataFrame()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OfDMorbWAJMD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results['id'] = test['id']\n",
        "results['target'] = outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0YQpjl74DdB-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "results.to_csv('/content/gdrive/My Drive/Colab Notebooks/DeepLearning/Bert/robertalarge2.csv',index=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gz_PFQmL-GSw",
        "colab_type": "text"
      },
      "source": [
        "Оценка на тесте для RoBERTa оказывается выше Albert (0.81799 vs 0.79856)"
      ]
    }
  ]
}