LLM_015_summarize_pdf.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyPN6jAMMcd6OqCAShuDwYgp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "bd7c2a6db7ad436eac7094b3d4a7437a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fb6dff530d8f4f358c1b0f4590a12016",
              "IPY_MODEL_de591b80f2404b92b89a08ba3686827c",
              "IPY_MODEL_81ff954f42554d39af0fa85be4352b34"
            ],
            "layout": "IPY_MODEL_8e6961ae2df04dca879d633060803514"
          }
        },
        "fb6dff530d8f4f358c1b0f4590a12016": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4ccff7f5cff9418ba8b9ade37f7280c3",
            "placeholder": "​",
            "style": "IPY_MODEL_0e4cd16abb3441c29510db6928f9b317",
            "value": "config.json: 100%"
          }
        },
        "de591b80f2404b92b89a08ba3686827c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3d0a4ce68b9425b90a3928ba24be928",
            "max": 1585,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_62e76dc7dd504b9ca08d5c8eebc3b927",
            "value": 1585
          }
        },
        "81ff954f42554d39af0fa85be4352b34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2356a6a4300c416ca2913c22e82c272b",
            "placeholder": "​",
            "style": "IPY_MODEL_9bdaabdc47ef469aa8833553b7fe9594",
            "value": " 1.58k/1.58k [00:00&lt;00:00, 40.9kB/s]"
          }
        },
        "8e6961ae2df04dca879d633060803514": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ccff7f5cff9418ba8b9ade37f7280c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0e4cd16abb3441c29510db6928f9b317": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c3d0a4ce68b9425b90a3928ba24be928": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "62e76dc7dd504b9ca08d5c8eebc3b927": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2356a6a4300c416ca2913c22e82c272b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9bdaabdc47ef469aa8833553b7fe9594": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "eddc3a0532ed4d9796765fdce32a93ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_28dfa797feff4ee8acb73def954136fd",
              "IPY_MODEL_8302fd0033a8453e9179d2c62dd5cf49",
              "IPY_MODEL_14195764d6fd47579d08a20a3852c46a"
            ],
            "layout": "IPY_MODEL_39d35a69ebf14942951bf6372b9b2b95"
          }
        },
        "28dfa797feff4ee8acb73def954136fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1cf548d362b849bebb634b0a8718b050",
            "placeholder": "​",
            "style": "IPY_MODEL_e06c911ece8f4cd3912912f6e5380f46",
            "value": "model.safetensors: 100%"
          }
        },
        "8302fd0033a8453e9179d2c62dd5cf49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ccbca7d01f0b44e7953c685807b989c3",
            "max": 1625222120,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_93ee407726b449eb92b3d442621c0f33",
            "value": 1625222120
          }
        },
        "14195764d6fd47579d08a20a3852c46a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6749d29d71b4011acc0529d00db8c77",
            "placeholder": "​",
            "style": "IPY_MODEL_43e79b84965543ef8c95e1554518df6b",
            "value": " 1.63G/1.63G [00:23&lt;00:00, 103MB/s]"
          }
        },
        "39d35a69ebf14942951bf6372b9b2b95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1cf548d362b849bebb634b0a8718b050": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e06c911ece8f4cd3912912f6e5380f46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ccbca7d01f0b44e7953c685807b989c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93ee407726b449eb92b3d442621c0f33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f6749d29d71b4011acc0529d00db8c77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "43e79b84965543ef8c95e1554518df6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "af8cb2fa2c0742a2a0747c071ef0985f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6cb7c756c0cb4dc7806d35dc551dee89",
              "IPY_MODEL_113323c304ec4c12aac466d92f437be0",
              "IPY_MODEL_761e7b51c4cb4bc790b3f59e24536e1f"
            ],
            "layout": "IPY_MODEL_d1d2cfec8fbc4c06b2e01db1e145093a"
          }
        },
        "6cb7c756c0cb4dc7806d35dc551dee89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eaac52ed305c44f0acdd1e5a678cc8ca",
            "placeholder": "​",
            "style": "IPY_MODEL_46534ff8d8824f1c8dd83f0caf324042",
            "value": "generation_config.json: 100%"
          }
        },
        "113323c304ec4c12aac466d92f437be0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc6b170f33e042be84adf7bc5034c746",
            "max": 363,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c01b8dfd1ec94edd8f4ac44004a4f75f",
            "value": 363
          }
        },
        "761e7b51c4cb4bc790b3f59e24536e1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c94392933f0410d96f5c86fee7850cc",
            "placeholder": "​",
            "style": "IPY_MODEL_dedbd3d001db452a84d9211624754665",
            "value": " 363/363 [00:00&lt;00:00, 6.08kB/s]"
          }
        },
        "d1d2cfec8fbc4c06b2e01db1e145093a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eaac52ed305c44f0acdd1e5a678cc8ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46534ff8d8824f1c8dd83f0caf324042": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cc6b170f33e042be84adf7bc5034c746": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c01b8dfd1ec94edd8f4ac44004a4f75f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6c94392933f0410d96f5c86fee7850cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dedbd3d001db452a84d9211624754665": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d1eedfc975694315ab2ca460faa05117": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_40d3848625f74ae3a433fbe2afb68522",
              "IPY_MODEL_295617dfa8e94a69847056cb3c8ba585",
              "IPY_MODEL_7a1d00886a724b5b84908e60d768e9b4"
            ],
            "layout": "IPY_MODEL_37c38b9f240c4306a4e67c7c45112b39"
          }
        },
        "40d3848625f74ae3a433fbe2afb68522": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_74c38d4f54684b1cbfd2a286f85cb743",
            "placeholder": "​",
            "style": "IPY_MODEL_4c197e30d6a14669a1889f924b66939d",
            "value": "vocab.json: 100%"
          }
        },
        "295617dfa8e94a69847056cb3c8ba585": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8296b65193b1401595f43fb7c09930e3",
            "max": 898823,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8749d6d4b4794637ba0db844c0599d27",
            "value": 898823
          }
        },
        "7a1d00886a724b5b84908e60d768e9b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bef039dacf0c4a0ab332bb08ce041fce",
            "placeholder": "​",
            "style": "IPY_MODEL_a4b2bd43dd6f46c7b43e32171370aa28",
            "value": " 899k/899k [00:00&lt;00:00, 6.78MB/s]"
          }
        },
        "37c38b9f240c4306a4e67c7c45112b39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74c38d4f54684b1cbfd2a286f85cb743": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c197e30d6a14669a1889f924b66939d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8296b65193b1401595f43fb7c09930e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8749d6d4b4794637ba0db844c0599d27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bef039dacf0c4a0ab332bb08ce041fce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a4b2bd43dd6f46c7b43e32171370aa28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b849b95305e9478899f28b9361c0b8f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_487407bb1b034f96bd487111a18c343a",
              "IPY_MODEL_04c72dd459cd44f4a826d1ce669961c3",
              "IPY_MODEL_a35b0f151e014faea8b17f9d0172e3c7"
            ],
            "layout": "IPY_MODEL_56f43235b2e34b2c951aa36f353874ae"
          }
        },
        "487407bb1b034f96bd487111a18c343a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b3563b8a26264ae097e1cd22f10eca45",
            "placeholder": "​",
            "style": "IPY_MODEL_189b886926c74c8a9db31f44ab6dc685",
            "value": "merges.txt: 100%"
          }
        },
        "04c72dd459cd44f4a826d1ce669961c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eb0a88ea677d4d3b8b9ed5af0979ebab",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_02149c3eb9594b27b15b23751adb2cd9",
            "value": 456318
          }
        },
        "a35b0f151e014faea8b17f9d0172e3c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e66f46384ce6456290cfa89ee539b6df",
            "placeholder": "​",
            "style": "IPY_MODEL_9e021066aa7b486ca2e0bbb91428ae7a",
            "value": " 456k/456k [00:00&lt;00:00, 3.67MB/s]"
          }
        },
        "56f43235b2e34b2c951aa36f353874ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b3563b8a26264ae097e1cd22f10eca45": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "189b886926c74c8a9db31f44ab6dc685": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "eb0a88ea677d4d3b8b9ed5af0979ebab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02149c3eb9594b27b15b23751adb2cd9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e66f46384ce6456290cfa89ee539b6df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e021066aa7b486ca2e0bbb91428ae7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2b0e5ee9f5d345d794118ce208993e7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7f386d30aeb64c6fa6601ef01798dfae",
              "IPY_MODEL_47b35db46a2243b4b4e1fed468fe65f2",
              "IPY_MODEL_503fdc20cb8b4984b8ed01f9733cae94"
            ],
            "layout": "IPY_MODEL_dbbfbd08079e4604be014909af00f9fa"
          }
        },
        "7f386d30aeb64c6fa6601ef01798dfae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a38b1e432e848e38e602a5700b79923",
            "placeholder": "​",
            "style": "IPY_MODEL_2d2529c179bb466bacf7ebd43e6d7914",
            "value": "tokenizer.json: 100%"
          }
        },
        "47b35db46a2243b4b4e1fed468fe65f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e44e0a63f66c4cc6aaf115e33a427b9a",
            "max": 1355863,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9eba11be532d4d26b4429e31705599dd",
            "value": 1355863
          }
        },
        "503fdc20cb8b4984b8ed01f9733cae94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_322c2016f6474782ba144abb807e65cf",
            "placeholder": "​",
            "style": "IPY_MODEL_c8e48185ce9c435492706d2621532e66",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 4.16MB/s]"
          }
        },
        "dbbfbd08079e4604be014909af00f9fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a38b1e432e848e38e602a5700b79923": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2d2529c179bb466bacf7ebd43e6d7914": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e44e0a63f66c4cc6aaf115e33a427b9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9eba11be532d4d26b4429e31705599dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "322c2016f6474782ba144abb807e65cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8e48185ce9c435492706d2621532e66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/micah-shull/LLMs/blob/main/LLM_015_summarize_pdf.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "WgTBtRpemYbR"
      },
      "outputs": [],
      "source": [
        "# !pip install transformers datasets\n",
        "# !pip install python-dotenv"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Environment Setup"
      ],
      "metadata": {
        "id": "pHbiDXqanFff"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Necessary imports\n",
        "import os\n",
        "from dotenv import load_dotenv\n",
        "from transformers import pipeline\n",
        "from datasets import load_dataset\n",
        "\n",
        "# Load environment variables\n",
        "load_dotenv('/content/huggingface_api_key.env')\n",
        "api_key = os.getenv(\"HUGGINGFACE_API_KEY\")\n",
        "# os.environ[\"HUGGINGFACEHUB_API_TOKEN\"] = api_key\n",
        "os.environ[\"HF_TOKEN\"] = api_key\n",
        "\n",
        "# Initialize Hugging Face sentiment-analysis pipeline\n",
        "sentiment_pipeline = pipeline(\"sentiment-analysis\", model=\"distilbert-base-uncased-finetuned-sst-2-english\")"
      ],
      "metadata": {
        "id": "tya6p-kznEs8"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 1: Extract Text from the PDF\n",
        "To process the PDF text with an LLM, we first need to extract the text. Python libraries like **PyMuPDF (fitz)**, **PyPDF2**, or **pdfplumber** are great for this.\n",
        "\n",
        "Certainly! Here’s a breakdown of each step to include in your documentation.\n",
        "\n",
        "---\n",
        "\n",
        "### Steps to Read and Extract Text from a PDF Hosted Online\n",
        "\n",
        "When working with PDFs hosted online, we need to first download the PDF and then process it to extract the text. Here’s a step-by-step explanation of the process:\n",
        "\n",
        "#### Step 1: Download the PDF File Using `requests`\n",
        "\n",
        "The `requests` library in Python allows us to fetch content from a URL. In this case, we use `requests.get(pdf_url)` to download the PDF file from a specified URL.\n",
        "\n",
        "- **Code**: `response = requests.get(pdf_url)`\n",
        "- **Explanation**: This sends an HTTP GET request to the provided URL (`pdf_url`), which should point to a PDF file.\n",
        "- **Status Check**: `response.raise_for_status()` will raise an error if the request was unsuccessful (e.g., if the file is not found or the server returns an error). This step ensures that the download was successful before proceeding.\n",
        "\n",
        "#### Step 2: Open the PDF from Memory with `pdfplumber`\n",
        "\n",
        "Once the PDF content is downloaded, we need a way to process it without saving it to disk. We can use `io.BytesIO` to handle the PDF content directly in memory.\n",
        "\n",
        "- **Code**: `pdfplumber.open(io.BytesIO(response.content))`\n",
        "- **Explanation**: `response.content` contains the raw bytes of the PDF. Wrapping it in `io.BytesIO` creates an in-memory file-like object, allowing `pdfplumber` to open and process it as if it were a local file.\n",
        "- **Usage**: `with pdfplumber.open(io.BytesIO(response.content)) as pdf` opens the PDF, making it accessible page-by-page.\n",
        "\n",
        "#### Step 3: Extract Text from Each Page\n",
        "\n",
        "Once the PDF is open, we can iterate through its pages and extract text.\n",
        "\n",
        "- **Code**: `for page in pdf.pages`\n",
        "- **Explanation**: This loop iterates through each page of the PDF.\n",
        "- **Text Extraction**: `page.extract_text()` extracts the text from each page. We concatenate the text from all pages to get a complete text representation of the document.\n",
        "  \n",
        "#### Step 4: Return or Print Extracted Text\n",
        "\n",
        "The extracted text is stored in a variable (`text`) and can either be returned by the function or printed for a quick preview.\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "1WGmObo1mm79"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# !pip install pdfplumber"
      ],
      "metadata": {
        "id": "bALr7eu3ofqj"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "import pdfplumber\n",
        "import io\n",
        "\n",
        "# Function to extract text from a PDF file\n",
        "def extract_text_from_pdf(pdf_url):\n",
        "    # Download the PDF from the URL\n",
        "    response = requests.get(pdf_url)\n",
        "    response.raise_for_status()  # Check if the request was successful\n",
        "\n",
        "    # Open the PDF from the downloaded bytes\n",
        "    text = \"\"\n",
        "    with pdfplumber.open(io.BytesIO(response.content)) as pdf:\n",
        "        for page in pdf.pages:\n",
        "            text += page.extract_text() + \"\\n\"\n",
        "    return text\n",
        "\n",
        "# Example usage\n",
        "pdf_url = \"https://sdplantatlas.org/ge_files/pdf/Yellow-rumped%20Warbler.pdf\"\n",
        "pdf_text = extract_text_from_pdf(pdf_url)\n",
        "print(pdf_text[:500])  # Display the first 500 characters of extracted text\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tJbyFUWQpEAU",
        "outputId": "8503a159-997f-4920-e893-187454077d20"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Wood Warblers — Family Parulidae 477\n",
            "Yellow-rumped Warbler Dendroica coronata\n",
            "The Yellow-rumped Warbler is probably San Diego\n",
            "County’s most abundant winter visitor. If the White-\n",
            "crowned Sparrow exceeds it, it is not by much.\n",
            "Eucalyptus groves and other exotic trees planted in\n",
            "developed areas suit the Yellow-rumped Warbler at\n",
            "least as much as natural sage scrub, chaparral, and\n",
            "woodland. The birds are strongly concentrated in\n",
            "the coastal lowland but are found almost through-\n",
            "out the county, lacki\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### PDF from Local Folder"
      ],
      "metadata": {
        "id": "77rkpcdmpXjb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Extract text from each page of the PDF\n",
        "def extract_text_from_pdf(pdf_path):\n",
        "    text = \"\"\n",
        "    with pdfplumber.open(pdf_path) as pdf:\n",
        "        for page in pdf.pages:\n",
        "            text += page.extract_text() + \"\\n\"\n",
        "    return text\n",
        "\n",
        "# Example usage\n",
        "pdf_path = \"//content/Yellow-rumped Warbler.pdf\"\n",
        "pdf_text = extract_text_from_pdf(pdf_path)\n",
        "print(pdf_text[:500])  # Display the first 500 characters of extracted text"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tk_he9Qkm7mY",
        "outputId": "a6d72ad4-1996-4d18-b40f-b5855f4ca71b"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Wood Warblers — Family Parulidae 477\n",
            "Yellow-rumped Warbler Dendroica coronata\n",
            "The Yellow-rumped Warbler is probably San Diego\n",
            "County’s most abundant winter visitor. If the White-\n",
            "crowned Sparrow exceeds it, it is not by much.\n",
            "Eucalyptus groves and other exotic trees planted in\n",
            "developed areas suit the Yellow-rumped Warbler at\n",
            "least as much as natural sage scrub, chaparral, and\n",
            "woodland. The birds are strongly concentrated in\n",
            "the coastal lowland but are found almost through-\n",
            "out the county, lacki\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 2: Split Long Text into Chunks\n",
        "\n",
        "\n",
        "This code is designed to **split a long text into smaller chunks** to ensure that each chunk is within a specified token limit (`max_length=512`). This is especially useful when working with models like BERT-based models, which typically have a **token limit of 512** (i.e., they can process only 512 tokens at a time). Let’s break down the code:\n",
        "\n",
        "### Code Explanation\n",
        "\n",
        "1. **Split Text into Words**: `words = text.split()`\n",
        "   - This splits the `text` string into a list of individual words based on spaces. Each word becomes an element in the `words` list.\n",
        "\n",
        "2. **Loop Through Words in Chunks**: `for i in range(0, len(words), max_length)`\n",
        "   - The `range(0, len(words), max_length)` statement allows us to loop over `words` in increments of `max_length`.\n",
        "   - The loop variable `i` starts at `0` and increments by `max_length` each time, representing the starting index of each new chunk of words.\n",
        "\n",
        "3. **Generate Chunks**: `yield ' '.join(words[i:i + max_length])`\n",
        "   - `words[i:i + max_length]` selects a slice of the list containing `max_length` words.\n",
        "   - `join()` combines the list of words in the slice into a single string, creating a text chunk.\n",
        "   - `yield` returns this chunk of text, pausing the function until the next chunk is requested.\n",
        "\n",
        "\n",
        "- **Purpose**: This line collects all generated chunks in a list by calling `split_text_into_chunks()` on `pdf_text`.\n",
        "- Each element in `chunks` is a text segment of up to `512` words (or fewer if it’s the last chunk).\n",
        "\n",
        "### Why This is Useful\n",
        "- **Model Token Limit**: LLMs, particularly transformer models like BERT or BART, have a limit on how many tokens (words or subwords) they can process at once. This code breaks the text into manageable segments.\n",
        "- **Ensuring Full Text Coverage**: By processing text in chunks, you can summarize, classify, or analyze the entire document without losing any information due to truncation.\n",
        "- **Memory Efficiency**: Using `yield` allows chunks to be generated one at a time, which is memory-efficient when dealing with large documents.\n",
        "\n",
        "This chunking approach is essential for large documents in NLP tasks, ensuring the model can handle each part of the text without exceeding input limits."
      ],
      "metadata": {
        "id": "ZLyIDT7zmpc5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def split_text_into_chunks(text, max_length=512):\n",
        "    words = text.split()\n",
        "    for i in range(0, len(words), max_length):\n",
        "        yield ' '.join(words[i:i + max_length])\n",
        "\n",
        "# Example usage\n",
        "chunks = list(split_text_into_chunks(pdf_text, max_length=512))"
      ],
      "metadata": {
        "id": "yeRtAyD3m4VO"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 3: Summarize Each Chunk with Hugging Face Transformers\n",
        "Hugging Face provides pre-trained summarization models like **BART** or **T5**, which work well for summarizing text.\n"
      ],
      "metadata": {
        "id": "g1EyyaMimr1v"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "# Load the summarization pipeline\n",
        "summarizer = pipeline(\"summarization\", model=\"facebook/bart-large-cnn\")\n",
        "\n",
        "# Summarize each chunk\n",
        "summaries = [summarizer(chunk)[0][\"summary_text\"] for chunk in chunks]\n",
        "\n",
        "# Combine the chunk summaries into a single document summary\n",
        "full_summary = \" \".join(summaries)\n",
        "print(full_summary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 246,
          "referenced_widgets": [
            "bd7c2a6db7ad436eac7094b3d4a7437a",
            "fb6dff530d8f4f358c1b0f4590a12016",
            "de591b80f2404b92b89a08ba3686827c",
            "81ff954f42554d39af0fa85be4352b34",
            "8e6961ae2df04dca879d633060803514",
            "4ccff7f5cff9418ba8b9ade37f7280c3",
            "0e4cd16abb3441c29510db6928f9b317",
            "c3d0a4ce68b9425b90a3928ba24be928",
            "62e76dc7dd504b9ca08d5c8eebc3b927",
            "2356a6a4300c416ca2913c22e82c272b",
            "9bdaabdc47ef469aa8833553b7fe9594",
            "eddc3a0532ed4d9796765fdce32a93ca",
            "28dfa797feff4ee8acb73def954136fd",
            "8302fd0033a8453e9179d2c62dd5cf49",
            "14195764d6fd47579d08a20a3852c46a",
            "39d35a69ebf14942951bf6372b9b2b95",
            "1cf548d362b849bebb634b0a8718b050",
            "e06c911ece8f4cd3912912f6e5380f46",
            "ccbca7d01f0b44e7953c685807b989c3",
            "93ee407726b449eb92b3d442621c0f33",
            "f6749d29d71b4011acc0529d00db8c77",
            "43e79b84965543ef8c95e1554518df6b",
            "af8cb2fa2c0742a2a0747c071ef0985f",
            "6cb7c756c0cb4dc7806d35dc551dee89",
            "113323c304ec4c12aac466d92f437be0",
            "761e7b51c4cb4bc790b3f59e24536e1f",
            "d1d2cfec8fbc4c06b2e01db1e145093a",
            "eaac52ed305c44f0acdd1e5a678cc8ca",
            "46534ff8d8824f1c8dd83f0caf324042",
            "cc6b170f33e042be84adf7bc5034c746",
            "c01b8dfd1ec94edd8f4ac44004a4f75f",
            "6c94392933f0410d96f5c86fee7850cc",
            "dedbd3d001db452a84d9211624754665",
            "d1eedfc975694315ab2ca460faa05117",
            "40d3848625f74ae3a433fbe2afb68522",
            "295617dfa8e94a69847056cb3c8ba585",
            "7a1d00886a724b5b84908e60d768e9b4",
            "37c38b9f240c4306a4e67c7c45112b39",
            "74c38d4f54684b1cbfd2a286f85cb743",
            "4c197e30d6a14669a1889f924b66939d",
            "8296b65193b1401595f43fb7c09930e3",
            "8749d6d4b4794637ba0db844c0599d27",
            "bef039dacf0c4a0ab332bb08ce041fce",
            "a4b2bd43dd6f46c7b43e32171370aa28",
            "b849b95305e9478899f28b9361c0b8f0",
            "487407bb1b034f96bd487111a18c343a",
            "04c72dd459cd44f4a826d1ce669961c3",
            "a35b0f151e014faea8b17f9d0172e3c7",
            "56f43235b2e34b2c951aa36f353874ae",
            "b3563b8a26264ae097e1cd22f10eca45",
            "189b886926c74c8a9db31f44ab6dc685",
            "eb0a88ea677d4d3b8b9ed5af0979ebab",
            "02149c3eb9594b27b15b23751adb2cd9",
            "e66f46384ce6456290cfa89ee539b6df",
            "9e021066aa7b486ca2e0bbb91428ae7a",
            "2b0e5ee9f5d345d794118ce208993e7a",
            "7f386d30aeb64c6fa6601ef01798dfae",
            "47b35db46a2243b4b4e1fed468fe65f2",
            "503fdc20cb8b4984b8ed01f9733cae94",
            "dbbfbd08079e4604be014909af00f9fa",
            "0a38b1e432e848e38e602a5700b79923",
            "2d2529c179bb466bacf7ebd43e6d7914",
            "e44e0a63f66c4cc6aaf115e33a427b9a",
            "9eba11be532d4d26b4429e31705599dd",
            "322c2016f6474782ba144abb807e65cf",
            "c8e48185ce9c435492706d2621532e66"
          ]
        },
        "id": "5xQgqxU0m1rd",
        "outputId": "d9348c48-e88a-4974-8e6b-d626c5b3f2d7"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/1.58k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd7c2a6db7ad436eac7094b3d4a7437a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/1.63G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "eddc3a0532ed4d9796765fdce32a93ca"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/363 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "af8cb2fa2c0742a2a0747c071ef0985f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d1eedfc975694315ab2ca460faa05117"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b849b95305e9478899f28b9361c0b8f0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2b0e5ee9f5d345d794118ce208993e7a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The Yellow-rumped Warbler is probably San Diego County’s most abundant winter visitor. The birds are strongly concentrated in the coastal lowland but are found almost through- out the county. The warblers are now of annual occurrence in the Cuyamaca Mountains, with up to eight, including five singing males, on Middle Peak. Audubon’s Warblers usually nest in the outer branches of P. K. Nelson but much less common in native desert the middle levels of conifers. It is rare to absent in the least-vegetated tracts of 2004 was in a burned canyon live oak retaining toasted desert and lacking entirely from pinyon–juniper zone leaves. Most records are from the coastal lowland, summits of San Diego County’s peaks. The Myrtle Warbler too is usu- southeastern Arizona. The dominant subspecies of the Yellow- high as 30 in Tijuana River valley 17 December 1977. Dates for the Myrtle range from 5 October (Townsend, 1837)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Format Summary\n",
        "\n",
        "To make your summary more readable, you can format it by breaking it into shorter paragraphs or adding line breaks. Here are a few techniques for improving readability:\n",
        "\n",
        "### 1. **Text Wrapping**\n",
        "   - Use the `textwrap` module to wrap long lines of text into a specified width (e.g., 80 characters), making it easier to read without scrolling horizontally.\n",
        "\n",
        "### 2. **Adding Paragraph Breaks**\n",
        "   - You can split the summary into paragraphs if it’s long. Adding breaks every few sentences improves readability.\n",
        "\n",
        "\n",
        "### Explanation\n",
        "\n",
        "- **`textwrap.fill(sentence, width=width)`**: Wraps each sentence to a specified width, breaking lines to fit within the limit.\n",
        "- **Adding Line Breaks**: Every 3 sentences, a blank line is added to create paragraph breaks.\n",
        "\n",
        "### Result\n",
        "This will print the summary in a structured, legible format with line breaks and neatly wrapped lines, making it easier to read in a console or output window. Adjust `width` or the number of sentences per paragraph as needed to customize the format."
      ],
      "metadata": {
        "id": "v2Z5lNbntUyk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import textwrap\n",
        "\n",
        "def format_summary(summary_text, width=80):\n",
        "    # Split text into sentences (optional: based on periods)\n",
        "    sentences = summary_text.split(\". \")\n",
        "\n",
        "    # Group sentences into paragraphs\n",
        "    formatted_text = \"\"\n",
        "    for i, sentence in enumerate(sentences, 1):\n",
        "        # Wrap each sentence to the specified width\n",
        "        wrapped_sentence = textwrap.fill(sentence, width=width)\n",
        "        formatted_text += wrapped_sentence + \".\\n\"\n",
        "\n",
        "        # Add a blank line every 3 sentences for readability\n",
        "        if i % 3 == 0:\n",
        "            formatted_text += \"\\n\"\n",
        "\n",
        "    return formatted_text\n",
        "\n",
        "# Example usage\n",
        "formatted_summary = format_summary(full_summary, width=80)\n",
        "print(formatted_summary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HJQ5AMBftCIo",
        "outputId": "383bb648-1393-4b54-8620-52382dc798d0"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The Yellow-rumped Warbler is probably San Diego County’s most abundant winter\n",
            "visitor.\n",
            "The birds are strongly concentrated in the coastal lowland but are found almost\n",
            "through- out the county.\n",
            "The warblers are now of annual occurrence in the Cuyamaca Mountains, with up to\n",
            "eight, including five singing males, on Middle Peak.\n",
            "\n",
            "Audubon’s Warblers usually nest in the outer branches of P.\n",
            "K.\n",
            "Nelson but much less common in native desert the middle levels of conifers.\n",
            "\n",
            "It is rare to absent in the least-vegetated tracts of 2004 was in a burned\n",
            "canyon live oak retaining toasted desert and lacking entirely from\n",
            "pinyon–juniper zone leaves.\n",
            "Most records are from the coastal lowland, summits of San Diego County’s peaks.\n",
            "The Myrtle Warbler too is usu- southeastern Arizona.\n",
            "\n",
            "The dominant subspecies of the Yellow- high as 30 in Tijuana River valley 17\n",
            "December 1977.\n",
            "Dates for the Myrtle range from 5 October (Townsend, 1837).\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "### Step 4: Optional - Save the Summary to a Text File\n",
        "\n",
        "\n",
        "### Explanation of Each Step\n",
        "1. **Extract Text**: We use `pdfplumber` to extract text from each page in the PDF.\n",
        "2. **Split Text**: The text is split into chunks to fit within the LLM’s input limit.\n",
        "3. **Summarize**: Each chunk is summarized, and all summaries are combined for a final output.\n",
        "4. **Save**: The final summary is saved to a file if needed.\n",
        "\n",
        "### Key Considerations\n",
        "- **Accuracy**: For best results, you may want to experiment with different summarization models like `t5-small`, `facebook/bart-large-cnn`, or `google/pegasus-xsum`.\n",
        "- **Document Structure**: If the PDF has distinct sections, summarizing each section separately can improve the coherence of the summary.\n",
        "\n"
      ],
      "metadata": {
        "id": "wTDZJBnkmgQF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"summary.txt\", \"w\") as file:\n",
        "    file.write(full_summary)\n",
        "\n",
        "print(\"Summary saved to summary.txt\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xAsx0i2Rmv3l",
        "outputId": "3caad4cd-9f06-4ea0-f3c1-8c9ababc500b"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Summary saved to summary.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Modify Summary Length\n",
        "\n",
        "You can adjust the summary length by setting parameters in the Hugging Face `pipeline` when using models like BART or T5 for summarization. Specifically, you can use the **`min_length`** and **`max_length`** parameters to control the minimum and maximum number of tokens in the summary.\n",
        "\n",
        "### Parameters Explained\n",
        "- **`min_length`**: Sets the minimum number of tokens for the summary. The model will try not to generate a summary shorter than this length.\n",
        "- **`max_length`**: Sets the maximum number of tokens for the summary, allowing you to restrict the length of the output.\n",
        "\n",
        "### Considerations\n",
        "- **Balance**: Choose `min_length` and `max_length` based on your needs. A higher `max_length` produces a longer, more detailed summary, while a lower one provides a concise overview.\n",
        "- **Truncation**: Setting `truncation=True` ensures that if the input text is too long, it will be truncated to fit within the model’s token limit (typically 512 tokens for BART).\n",
        "\n",
        "This approach allows you to control the summary length dynamically, making it possible to adjust the length based on the document type or your summarization needs."
      ],
      "metadata": {
        "id": "5L53Htm8rHSi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "# Load the summarization pipeline\n",
        "summarizer = pipeline(\"summarization\", model=\"facebook/bart-large-cnn\")\n",
        "\n",
        "# Set min_length and max_length for a custom summary length\n",
        "def summarize_text(text, min_len=50, max_len=150):\n",
        "    summary = summarizer(text, min_length=min_len, max_length=max_len, truncation=True)\n",
        "    return summary[0]['summary_text']\n",
        "\n",
        "# Example usage\n",
        "text = \"Your long PDF text or extracted text goes here...\"\n",
        "print(summarize_text(text, min_len=50, max_len=150))"
      ],
      "metadata": {
        "id": "-iX5Im48rDbL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Full Document Context\n",
        "\n",
        "When a large document is broken into chunks for an LLM, each chunk is processed independently, which means the model doesn’t have direct access to the full document context across chunks. Here’s how LLMs and summarization techniques address this limitation and some strategies to retain contextual connection:\n",
        "\n",
        "### 1. **Chunk-by-Chunk Summarization**\n",
        "   - **Independent Summaries**: Each chunk is treated as a standalone piece of text. The model generates a summary for each chunk separately, but this lacks a direct contextual link between chunks.\n",
        "   - **Combining Summaries**: After summarizing each chunk, the individual summaries can be concatenated into a new “summary document” and then summarized again. This is often called a **hierarchical summarization approach** and helps retain a higher-level summary of the entire document.\n",
        "\n",
        "### 2. **Hierarchical Summarization Approach**\n",
        "   - **Stage 1**: Summarize each chunk independently to get a \"mini-summary\" for each section.\n",
        "   - **Stage 2**: Combine all mini-summaries and run them through the model again to produce a cohesive summary that captures the document’s main points.\n",
        "   - **Why This Helps**: Although each chunk’s summary doesn’t know the content of other chunks directly, the second-stage summary provides a synthesized view by “summarizing the summaries.”\n",
        "\n",
        "### 3. **Embedding-Based Approaches for Semantic Connection**\n",
        "   - Using an **embedding model** (e.g., Sentence-BERT) allows you to create a vector representation of each chunk. These embeddings capture semantic similarities between chunks without needing a sequential context.\n",
        "   - **Clustering or Topic Modeling**: Once you have embeddings for each chunk, you can use clustering or topic modeling to find related themes across chunks. This provides a sense of document cohesion and can guide which key points to include in the final summary.\n",
        "\n",
        "### 4. **Memory-Augmented Transformers**\n",
        "   - Some advanced transformer models, like **Longformer** or **Reformer**, are designed to handle longer documents by keeping track of more extensive context. These models have specialized architectures allowing them to attend to a greater range of tokens, providing better context preservation over long documents.\n",
        "\n",
        "### 5. **Using Overlapping Chunks**\n",
        "   - By creating overlapping chunks (where, for instance, the last 50 tokens of one chunk are repeated at the beginning of the next), you allow some degree of context carry-over between chunks.\n",
        "   - This method lets the model “see” some of the previous content in the next chunk, though it’s still a limited solution.\n",
        "\n",
        "### Practical Implementation\n",
        "If you’re using a standard transformer model (like BERT or BART), the **hierarchical summarization approach** is often the most practical:\n",
        "- **Step 1**: Summarize each chunk independently.\n",
        "- **Step 2**: Concatenate the summaries.\n",
        "- **Step 3**: Run the concatenated summary through the model again to create a cohesive summary.\n",
        "\n",
        "Here’s a code snippet showing how you might implement hierarchical summarization:\n",
        "\n",
        "\n",
        "### Summary\n",
        "- **Chunk-by-chunk summarization** provides summaries for each part of the document.\n",
        "- **Hierarchical summarization** combines those summaries for a cohesive final summary.\n",
        "- While standard models can’t retain document-wide context directly, these techniques give you a practical way to retain key points from across the document.\n",
        "\n",
        "This process won’t capture nuanced context as well as reading the entire document at once, but it’s a robust workaround when handling documents that exceed model token limits."
      ],
      "metadata": {
        "id": "rQosKcozsyuX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "# Load summarization pipeline\n",
        "summarizer = pipeline(\"summarization\", model=\"facebook/bart-large-cnn\")\n",
        "\n",
        "# Summarize each chunk\n",
        "chunk_summaries = [summarizer(chunk, max_length=100, min_length=30)[0]['summary_text'] for chunk in chunks]\n",
        "\n",
        "# Combine summaries\n",
        "combined_summary_text = \" \".join(chunk_summaries)\n",
        "\n",
        "# Run a second summarization on the combined text\n",
        "final_summary = summarizer(combined_summary_text, max_length=150, min_length=50)[0]['summary_text']\n",
        "\n",
        "print(\"Final Summary of Document:\", final_summary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pqdrIwxMs6-J",
        "outputId": "87d494c0-6d75-4122-85e6-01675dd17a92"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Final Summary of Document: The Yellow-rumped Warbler is probably San Diego County’s most abundant winter visitor. The birds are strongly concentrated in the coastal lowland but are found almost through- out the county. Sightings of adults carrying insects are as early as 27 May.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "formatted_summary = format_summary(final_summary, width=80)\n",
        "print(formatted_summary)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8XE0YFWdu2C-",
        "outputId": "cf99676f-f5a3-46a7-826f-5c3cc744c179"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The Yellow-rumped Warbler is probably San Diego County’s most abundant winter\n",
            "visitor.\n",
            "The birds are strongly concentrated in the coastal lowland but are found almost\n",
            "through- out the county.\n",
            "Sightings of adults carrying insects are as early as 27 May..\n",
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cCdK_6zZu2TM"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}