practice_examples/09_Huggingface_GPT_BERT_NER.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c328ca5f11154fde8c41d79ab1852833": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a002354522bb414d8b70d892f115f802",
              "IPY_MODEL_bebca33ce0f24e8f901bcc05bcd8d912",
              "IPY_MODEL_19abf642508648dca573027b473d498e"
            ],
            "layout": "IPY_MODEL_f937ed88502c44759a6edad58c992279"
          }
        },
        "a002354522bb414d8b70d892f115f802": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_49057805aba241f1aec3ddccf1894e98",
            "placeholder": "​",
            "style": "IPY_MODEL_92a39ab5195c4993aa162ddc91e53ff4",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "bebca33ce0f24e8f901bcc05bcd8d912": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d515a3cd1add4eb7abe357c1bf55fc6b",
            "max": 762,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_256809315e9146a6bb9d31e130eca9ae",
            "value": 762
          }
        },
        "19abf642508648dca573027b473d498e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_584152188167453cb62322c3941af32c",
            "placeholder": "​",
            "style": "IPY_MODEL_8cbf85da59694b3386549c7d298fd78d",
            "value": " 762/762 [00:00&lt;00:00, 17.5kB/s]"
          }
        },
        "f937ed88502c44759a6edad58c992279": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "49057805aba241f1aec3ddccf1894e98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "92a39ab5195c4993aa162ddc91e53ff4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d515a3cd1add4eb7abe357c1bf55fc6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "256809315e9146a6bb9d31e130eca9ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "584152188167453cb62322c3941af32c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8cbf85da59694b3386549c7d298fd78d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fb3e66710abe49699a546f2e058eaa9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1ec8ae697f1d4ff5bf56cee2c593def7",
              "IPY_MODEL_90e42039d4044708bedf351b1f73472f",
              "IPY_MODEL_a133102a926a463ca28bbd7b44c5cf59"
            ],
            "layout": "IPY_MODEL_a74bfc52ae9b4617b499f03cfb4e5b4f"
          }
        },
        "1ec8ae697f1d4ff5bf56cee2c593def7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0fb8060fe9e34bca9e0f23bde339146a",
            "placeholder": "​",
            "style": "IPY_MODEL_61a159b7d94548009a17d99fba1fea7f",
            "value": "Downloading (…)olve/main/vocab.json: 100%"
          }
        },
        "90e42039d4044708bedf351b1f73472f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_191bb11498444a81beb1ba5c2d9e0081",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6dbecf38e4e74ce09b7bdbc15a5068f4",
            "value": 1042301
          }
        },
        "a133102a926a463ca28bbd7b44c5cf59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3d9aceb106a484482f3529325224f56",
            "placeholder": "​",
            "style": "IPY_MODEL_c586855f7e6a4f38b98a181b49b47935",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 21.8MB/s]"
          }
        },
        "a74bfc52ae9b4617b499f03cfb4e5b4f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0fb8060fe9e34bca9e0f23bde339146a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61a159b7d94548009a17d99fba1fea7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "191bb11498444a81beb1ba5c2d9e0081": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6dbecf38e4e74ce09b7bdbc15a5068f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c3d9aceb106a484482f3529325224f56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c586855f7e6a4f38b98a181b49b47935": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "abfe0448f2d54151bddf5b11f1356c69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_be5bbc4cd08243218f5961a15a8785ad",
              "IPY_MODEL_edb7ee0d59924c37bda59cde70dc23e0",
              "IPY_MODEL_cbbe0c16319d4f1584447dff0d6af6e8"
            ],
            "layout": "IPY_MODEL_7317718a8e99436099467c4681e546d0"
          }
        },
        "be5bbc4cd08243218f5961a15a8785ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f14224bcbe1c4841a981cf0ab4890fd9",
            "placeholder": "​",
            "style": "IPY_MODEL_5950085705b34a1a9de30b101395aeec",
            "value": "Downloading (…)olve/main/merges.txt: 100%"
          }
        },
        "edb7ee0d59924c37bda59cde70dc23e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ca11ee007fc140f38b628cd26f12c08a",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_172c0852a6e940eaa36adfbc8875ecac",
            "value": 456318
          }
        },
        "cbbe0c16319d4f1584447dff0d6af6e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_41e49ba6d73749bc86db17d8018090df",
            "placeholder": "​",
            "style": "IPY_MODEL_08f818f450954622b861e4332c52713d",
            "value": " 456k/456k [00:00&lt;00:00, 10.4MB/s]"
          }
        },
        "7317718a8e99436099467c4681e546d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f14224bcbe1c4841a981cf0ab4890fd9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5950085705b34a1a9de30b101395aeec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ca11ee007fc140f38b628cd26f12c08a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "172c0852a6e940eaa36adfbc8875ecac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "41e49ba6d73749bc86db17d8018090df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08f818f450954622b861e4332c52713d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14305cb0ce90460c87f66079fc361ed1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9f0251c8b9dc4bb9b37ec67439c773e4",
              "IPY_MODEL_2b74dde64e494727a183117e3b08aeb1",
              "IPY_MODEL_4561ec826c45430eba36a34b57197d94"
            ],
            "layout": "IPY_MODEL_e0fe925d15714628b93c4a961793ef21"
          }
        },
        "9f0251c8b9dc4bb9b37ec67439c773e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b10505db83f04605a0727c3f3ea005f4",
            "placeholder": "​",
            "style": "IPY_MODEL_549e113cb47741edb46114476bba836b",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "2b74dde64e494727a183117e3b08aeb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_178d0e30dba04932b3a486b2ad401012",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_14ceb4a1f94f4abbb1a2a3cca1ebd42b",
            "value": 1355256
          }
        },
        "4561ec826c45430eba36a34b57197d94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0e66bc4799044a1b8418c33cdc02c8c2",
            "placeholder": "​",
            "style": "IPY_MODEL_b20359b584ab4e80b0e19a54dfaa7819",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 26.9MB/s]"
          }
        },
        "e0fe925d15714628b93c4a961793ef21": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b10505db83f04605a0727c3f3ea005f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "549e113cb47741edb46114476bba836b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "178d0e30dba04932b3a486b2ad401012": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "14ceb4a1f94f4abbb1a2a3cca1ebd42b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0e66bc4799044a1b8418c33cdc02c8c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b20359b584ab4e80b0e19a54dfaa7819": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9b41376ffd26472a943c4b965e8c38df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_03372176a485416290a3030ca81a428a",
              "IPY_MODEL_d89862dc220d457cb33248988ccb82e8",
              "IPY_MODEL_737657d9913c48a1910ff57f0a70b1cc"
            ],
            "layout": "IPY_MODEL_6342986af02642efba797afef354d890"
          }
        },
        "03372176a485416290a3030ca81a428a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_163947545e3f418cbdf10ac30d765a3d",
            "placeholder": "​",
            "style": "IPY_MODEL_515c9036f32147048d8f4a68098bba8a",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "d89862dc220d457cb33248988ccb82e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4e07c34d61a440718cd553cb8f9be481",
            "max": 352833716,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b0be43c2b03c428b9a0125cb7995123d",
            "value": 352833716
          }
        },
        "737657d9913c48a1910ff57f0a70b1cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b98c86110029412e8ba66a3019313216",
            "placeholder": "​",
            "style": "IPY_MODEL_46d71c5005d544c49ea658c667adc77f",
            "value": " 353M/353M [00:01&lt;00:00, 200MB/s]"
          }
        },
        "6342986af02642efba797afef354d890": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "163947545e3f418cbdf10ac30d765a3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "515c9036f32147048d8f4a68098bba8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4e07c34d61a440718cd553cb8f9be481": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b0be43c2b03c428b9a0125cb7995123d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b98c86110029412e8ba66a3019313216": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46d71c5005d544c49ea658c667adc77f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "16d4e0936f1943dc9a1e338325a4b2c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1840e2e163974b59b2dde29ef42c7d20",
              "IPY_MODEL_3f5b5c546a34444881847d0261d50718",
              "IPY_MODEL_440c186bd2244d62aa02b653fb885f8e"
            ],
            "layout": "IPY_MODEL_bbc294f2fe7541459a512833fbcec733"
          }
        },
        "1840e2e163974b59b2dde29ef42c7d20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fbc2d6f0743d4fe3bfb1716d39ad3dd5",
            "placeholder": "​",
            "style": "IPY_MODEL_5cc35b6cf8c44fbca741183c7ac6bc35",
            "value": "Downloading (…)neration_config.json: 100%"
          }
        },
        "3f5b5c546a34444881847d0261d50718": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ac3a169c14b948069bff12b98c3db129",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fd6eca7b8f984214a553311e4ee47fe4",
            "value": 124
          }
        },
        "440c186bd2244d62aa02b653fb885f8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e4fd65f201845c19061a9afac861937",
            "placeholder": "​",
            "style": "IPY_MODEL_6d4f6f1e52a842bd962c8602a65825e1",
            "value": " 124/124 [00:00&lt;00:00, 2.11kB/s]"
          }
        },
        "bbc294f2fe7541459a512833fbcec733": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbc2d6f0743d4fe3bfb1716d39ad3dd5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5cc35b6cf8c44fbca741183c7ac6bc35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ac3a169c14b948069bff12b98c3db129": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd6eca7b8f984214a553311e4ee47fe4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3e4fd65f201845c19061a9afac861937": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d4f6f1e52a842bd962c8602a65825e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "48677b1c79e0430b9b51fb6869bc50cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_77cb5bcf90a04d4da37fcb97dcadd58d",
              "IPY_MODEL_0c271299eb1545cb81c8ddb456df2cc8",
              "IPY_MODEL_f2fcc80f8c2d40ce9ee546adfb4fcee7"
            ],
            "layout": "IPY_MODEL_44a0f774507e430e9e3e10ee10f6df1d"
          }
        },
        "77cb5bcf90a04d4da37fcb97dcadd58d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f70145bac54745318408932e3d47f8af",
            "placeholder": "​",
            "style": "IPY_MODEL_a5da0b72d8294aedbaa89ecc8c20f74a",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "0c271299eb1545cb81c8ddb456df2cc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f55d96e3ff84350847a928190b8139c",
            "max": 29,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_628ac92ac029498a8a90d813a3a0cf3b",
            "value": 29
          }
        },
        "f2fcc80f8c2d40ce9ee546adfb4fcee7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ab8a641fd14248589ead87232ed5cb8d",
            "placeholder": "​",
            "style": "IPY_MODEL_0b799bac44db4eabb7320c2a2d85c8ad",
            "value": " 29.0/29.0 [00:00&lt;00:00, 1.26kB/s]"
          }
        },
        "44a0f774507e430e9e3e10ee10f6df1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f70145bac54745318408932e3d47f8af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5da0b72d8294aedbaa89ecc8c20f74a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8f55d96e3ff84350847a928190b8139c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "628ac92ac029498a8a90d813a3a0cf3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ab8a641fd14248589ead87232ed5cb8d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b799bac44db4eabb7320c2a2d85c8ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "36b95bd1e9bd409ebc82c02d208c8e2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c56820bd06b540c8b5647fbcb18780c8",
              "IPY_MODEL_1273a7c789ff4216abd8619d84774497",
              "IPY_MODEL_747da68f204240cf870a57a0ea9e865b"
            ],
            "layout": "IPY_MODEL_4e98d57070de4c68835805ff89bc58bc"
          }
        },
        "c56820bd06b540c8b5647fbcb18780c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b7e8cd49a034c0ea91c37db1f8dacd8",
            "placeholder": "​",
            "style": "IPY_MODEL_e66ea9047be54a5da548d2037567795c",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "1273a7c789ff4216abd8619d84774497": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8d3d4fd7558941a4a1f9820776557991",
            "max": 213450,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_58f062d674ec43a9b23ae7677262e1bb",
            "value": 213450
          }
        },
        "747da68f204240cf870a57a0ea9e865b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5da33df9f7cf4cdba3cc3b09a1e0a0c0",
            "placeholder": "​",
            "style": "IPY_MODEL_94f526c2e79242d18e588b343b657a6f",
            "value": " 213k/213k [00:00&lt;00:00, 10.2MB/s]"
          }
        },
        "4e98d57070de4c68835805ff89bc58bc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b7e8cd49a034c0ea91c37db1f8dacd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e66ea9047be54a5da548d2037567795c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8d3d4fd7558941a4a1f9820776557991": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58f062d674ec43a9b23ae7677262e1bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5da33df9f7cf4cdba3cc3b09a1e0a0c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94f526c2e79242d18e588b343b657a6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d38577c1068041a6a8f3742f3685c523": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_422f3a877b1d4164b4dbeeae094e1b0e",
              "IPY_MODEL_c1e0d5e1ef9f439cb49a88701ec6630b",
              "IPY_MODEL_12ff357205384726b03fa1f8194c5d56"
            ],
            "layout": "IPY_MODEL_653000dd65f04d7ebf4aec140168de9a"
          }
        },
        "422f3a877b1d4164b4dbeeae094e1b0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a8d65f2e705e4a78bb1a2b30a1cfa6b7",
            "placeholder": "​",
            "style": "IPY_MODEL_a90a5408fbaa45f58c9d6181918a0801",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "c1e0d5e1ef9f439cb49a88701ec6630b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4bc76cf7b76142f7810319935fc0ec2c",
            "max": 435797,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_482d4a9aa5964253a59bb3cf8638142d",
            "value": 435797
          }
        },
        "12ff357205384726b03fa1f8194c5d56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a606a1a07d904bf293160bced14a9b2d",
            "placeholder": "​",
            "style": "IPY_MODEL_68aa1f19151a4a7f903a77a5313423ab",
            "value": " 436k/436k [00:00&lt;00:00, 25.8MB/s]"
          }
        },
        "653000dd65f04d7ebf4aec140168de9a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a8d65f2e705e4a78bb1a2b30a1cfa6b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a90a5408fbaa45f58c9d6181918a0801": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4bc76cf7b76142f7810319935fc0ec2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "482d4a9aa5964253a59bb3cf8638142d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a606a1a07d904bf293160bced14a9b2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68aa1f19151a4a7f903a77a5313423ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "33b09a1367f846a0ac6d1901f943a42d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_01fa6dae96314b9383ba693f2ba5f55f",
              "IPY_MODEL_2f167b681bdc4f1285dbe16782a8f9b9",
              "IPY_MODEL_9d34fb699e1f4b5da69b38d8c73f8765"
            ],
            "layout": "IPY_MODEL_f7313e0425474b8b822d415756bcf0c0"
          }
        },
        "01fa6dae96314b9383ba693f2ba5f55f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f3ce1afcf17d47b1824e0e8e470310c0",
            "placeholder": "​",
            "style": "IPY_MODEL_a93e01ce92b84c068f11a7346c80d1b7",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "2f167b681bdc4f1285dbe16782a8f9b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_891067e31e6541699a8aa597b33562a7",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_66ec4b30836349edbf63a260f51a985e",
            "value": 570
          }
        },
        "9d34fb699e1f4b5da69b38d8c73f8765": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_824235bba40445238a4bbf18475bb171",
            "placeholder": "​",
            "style": "IPY_MODEL_08f0bddb402146e1b3016969b0d3d5b9",
            "value": " 570/570 [00:00&lt;00:00, 42.9kB/s]"
          }
        },
        "f7313e0425474b8b822d415756bcf0c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3ce1afcf17d47b1824e0e8e470310c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a93e01ce92b84c068f11a7346c80d1b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "891067e31e6541699a8aa597b33562a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66ec4b30836349edbf63a260f51a985e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "824235bba40445238a4bbf18475bb171": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08f0bddb402146e1b3016969b0d3d5b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cca18c68475644aea3dc8f08dfab9f9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_100388d0561b4d9985b5d468a5958cbf",
              "IPY_MODEL_a20c758ce3f345b39821edc56b17cee2",
              "IPY_MODEL_f83777b234394b36ac708958906cda7a"
            ],
            "layout": "IPY_MODEL_630aff84c9ae4151933bc1ca31781607"
          }
        },
        "100388d0561b4d9985b5d468a5958cbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_37d7d0e0cd51407f89312398e06b5887",
            "placeholder": "​",
            "style": "IPY_MODEL_559f052776a448198c908aa1a4ba016c",
            "value": "Downloading tf_model.h5: 100%"
          }
        },
        "a20c758ce3f345b39821edc56b17cee2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dfc6823ec6eb45bd87e370f3bcc19994",
            "max": 526681800,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ea179e2f8d9f407eb82994d9c1192e6e",
            "value": 526681800
          }
        },
        "f83777b234394b36ac708958906cda7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f0b9e6a92ede464d8e91f0c4786db2af",
            "placeholder": "​",
            "style": "IPY_MODEL_7879630787db424eb0475dbbe1d0e05b",
            "value": " 527M/527M [00:03&lt;00:00, 255MB/s]"
          }
        },
        "630aff84c9ae4151933bc1ca31781607": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37d7d0e0cd51407f89312398e06b5887": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "559f052776a448198c908aa1a4ba016c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dfc6823ec6eb45bd87e370f3bcc19994": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea179e2f8d9f407eb82994d9c1192e6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f0b9e6a92ede464d8e91f0c4786db2af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7879630787db424eb0475dbbe1d0e05b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# [HuggingFace](https://huggingface.co/) \n",
        "<img src=\"https://huggingface.co/front/assets/huggingface_logo-noborder.svg\">\n",
        "\n",
        "- Repository for mainly Transformer-based models\n",
        "- Comes with a high-level python API"
      ],
      "metadata": {
        "id": "ezyrdhLQSnQR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "id": "9q9C_p3VTa4q",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0c86fba0-2b0f-4e95-b0d6-f469c52cea64"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.29.2-py3-none-any.whl (7.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m71.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m42.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.14.1 tokenizers-0.13.3 transformers-4.29.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModelForCausalLM"
      ],
      "metadata": {
        "id": "so4n7vIxTXZR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"distilgpt2\")\n",
        "model = AutoModelForCausalLM.from_pretrained(\"distilgpt2\", pad_token_id = tokenizer.eos_token_id)"
      ],
      "metadata": {
        "id": "kc1rd1TYT3d-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 209,
          "referenced_widgets": [
            "c328ca5f11154fde8c41d79ab1852833",
            "a002354522bb414d8b70d892f115f802",
            "bebca33ce0f24e8f901bcc05bcd8d912",
            "19abf642508648dca573027b473d498e",
            "f937ed88502c44759a6edad58c992279",
            "49057805aba241f1aec3ddccf1894e98",
            "92a39ab5195c4993aa162ddc91e53ff4",
            "d515a3cd1add4eb7abe357c1bf55fc6b",
            "256809315e9146a6bb9d31e130eca9ae",
            "584152188167453cb62322c3941af32c",
            "8cbf85da59694b3386549c7d298fd78d",
            "fb3e66710abe49699a546f2e058eaa9e",
            "1ec8ae697f1d4ff5bf56cee2c593def7",
            "90e42039d4044708bedf351b1f73472f",
            "a133102a926a463ca28bbd7b44c5cf59",
            "a74bfc52ae9b4617b499f03cfb4e5b4f",
            "0fb8060fe9e34bca9e0f23bde339146a",
            "61a159b7d94548009a17d99fba1fea7f",
            "191bb11498444a81beb1ba5c2d9e0081",
            "6dbecf38e4e74ce09b7bdbc15a5068f4",
            "c3d9aceb106a484482f3529325224f56",
            "c586855f7e6a4f38b98a181b49b47935",
            "abfe0448f2d54151bddf5b11f1356c69",
            "be5bbc4cd08243218f5961a15a8785ad",
            "edb7ee0d59924c37bda59cde70dc23e0",
            "cbbe0c16319d4f1584447dff0d6af6e8",
            "7317718a8e99436099467c4681e546d0",
            "f14224bcbe1c4841a981cf0ab4890fd9",
            "5950085705b34a1a9de30b101395aeec",
            "ca11ee007fc140f38b628cd26f12c08a",
            "172c0852a6e940eaa36adfbc8875ecac",
            "41e49ba6d73749bc86db17d8018090df",
            "08f818f450954622b861e4332c52713d",
            "14305cb0ce90460c87f66079fc361ed1",
            "9f0251c8b9dc4bb9b37ec67439c773e4",
            "2b74dde64e494727a183117e3b08aeb1",
            "4561ec826c45430eba36a34b57197d94",
            "e0fe925d15714628b93c4a961793ef21",
            "b10505db83f04605a0727c3f3ea005f4",
            "549e113cb47741edb46114476bba836b",
            "178d0e30dba04932b3a486b2ad401012",
            "14ceb4a1f94f4abbb1a2a3cca1ebd42b",
            "0e66bc4799044a1b8418c33cdc02c8c2",
            "b20359b584ab4e80b0e19a54dfaa7819",
            "9b41376ffd26472a943c4b965e8c38df",
            "03372176a485416290a3030ca81a428a",
            "d89862dc220d457cb33248988ccb82e8",
            "737657d9913c48a1910ff57f0a70b1cc",
            "6342986af02642efba797afef354d890",
            "163947545e3f418cbdf10ac30d765a3d",
            "515c9036f32147048d8f4a68098bba8a",
            "4e07c34d61a440718cd553cb8f9be481",
            "b0be43c2b03c428b9a0125cb7995123d",
            "b98c86110029412e8ba66a3019313216",
            "46d71c5005d544c49ea658c667adc77f",
            "16d4e0936f1943dc9a1e338325a4b2c0",
            "1840e2e163974b59b2dde29ef42c7d20",
            "3f5b5c546a34444881847d0261d50718",
            "440c186bd2244d62aa02b653fb885f8e",
            "bbc294f2fe7541459a512833fbcec733",
            "fbc2d6f0743d4fe3bfb1716d39ad3dd5",
            "5cc35b6cf8c44fbca741183c7ac6bc35",
            "ac3a169c14b948069bff12b98c3db129",
            "fd6eca7b8f984214a553311e4ee47fe4",
            "3e4fd65f201845c19061a9afac861937",
            "6d4f6f1e52a842bd962c8602a65825e1"
          ]
        },
        "outputId": "006b16ad-37f9-46b9-f27f-0b76e13a7587"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/762 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c328ca5f11154fde8c41d79ab1852833"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fb3e66710abe49699a546f2e058eaa9e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "abfe0448f2d54151bddf5b11f1356c69"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "14305cb0ce90460c87f66079fc361ed1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/353M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9b41376ffd26472a943c4b965e8c38df"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "16d4e0936f1943dc9a1e338325a4b2c0"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"Summer is\"\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "tokens = tokenizer(text, return_tensors=\"pt\")\n",
        "tokens"
      ],
      "metadata": {
        "id": "cQzi7eseUEk7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9528d8b4-d82d-478b-f484-12e8c3c74fca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': tensor([[33560,   318]]), 'attention_mask': tensor([[1, 1]])}"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "completion = model.generate(tokens[\"input_ids\"], max_length=64, do_sample=True, temperature=0.5, top_p=1.0)"
      ],
      "metadata": {
        "id": "ygxeRY7DUjDJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d7e1692d-c15c-4b44-869e-0a06abaaba8d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1255: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation)\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(completion)"
      ],
      "metadata": {
        "id": "cm_aqjV-XNLH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b2dec10b-c876-4e3c-e177-35aaf9090e0e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[33560,   318,   257,   845,  2968, 11984,  2168,   290,   257,  2168,\n",
            "          9593,  1811,  1180,  3435,    13,   198,   198,   198,   198,   198,\n",
            "           198,   198,   198,   198,   198,   198,   198,   198,   198,   198,\n",
            "           198,   198,   198,   198,   198,   198,   198,   198,   198,   198,\n",
            "           198,   198,   198,   198,   198,   198,   198,   198,   198,   198,\n",
            "           198,   198,   198,   198,   198,   198,   198,   198,   198,   198,\n",
            "           198,   198,   198,   198]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.decode(completion[0])"
      ],
      "metadata": {
        "id": "S76fDog4VLXo",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "outputId": "3767a9b2-7259-42f5-c975-1262880e543b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Summer is a very popular anime series and a series featuring several different characters.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lftoken = tokenizer.encode(\"\\n\")[0]"
      ],
      "metadata": {
        "id": "e2bKUiFGYgZv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(10):\n",
        "  text = \"Summer is\"\n",
        "  tokens = tokenizer(text, return_tensors=\"pt\")\n",
        "\n",
        "  completion = model.generate(tokens[\"input_ids\"], max_length=64, do_sample=True, temperature=0.5, top_p=1.0, eos_token_id=lftoken)\n",
        "  print(tokenizer.decode(completion[0]))"
      ],
      "metadata": {
        "id": "c90bIzDKW9EJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "212f345d-2d9c-4012-c591-759ed1490e76"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Summer is a very well-established, well-established, well-established, well-established, well-established, well-established, well-established, well-established, well-established, well-established, well-established, well-established, well-established, well-established, well-established,\n",
            "Summer is an excellent example of how we can work together to improve the lives of people who are struggling with substance abuse.\n",
            "\n",
            "Summer is a very well-known, very unique game that is a very well-known, very unique game that is a very well-known, very unique game that is a very well-known, very unique game that is a very well-known, very unique game that is a very well-known, very\n",
            "Summer is an open source project that is designed to help people build and maintain their own web application. The project is open source, and it will work on the server side of the project, and is open source.\n",
            "\n",
            "Summer is a popular music video game for the PlayStation 4 and Xbox One. It is also a popular video game for the PlayStation 3 and Xbox 360.\n",
            "\n",
            "Summer is a great place to get a good look at the history of the town. It was built in the 18th century and was built in 1867. It was built in the 1880s and is now a part of the city's historic district.\n",
            "\n",
            "Summer is a fun time to play with friends.\n",
            "\n",
            "Summer is a game about finding the balance between the two worlds.\n",
            "\n",
            "Summer is a new story of how the West is beginning to feel like a new world.\n",
            "\n",
            "Summer is a very good way to get your head around the issues of the Internet and Internet.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "for i in range(10):\n",
        "  text = \"Bob went to the cinema\"\n",
        "  tokens = tokenizer(text, return_tensors=\"pt\")\n",
        "\n",
        "  completion = model.generate(tokens[\"input_ids\"], max_length=64, do_sample=True, temperature=0.2, top_p=1.0, eos_token_id=lftoken, repetition_penalty=1e2)\n",
        "  print(tokenizer.decode(completion[0]))"
      ],
      "metadata": {
        "id": "gKiSsYG0YnqX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "355f2185-2dc1-45d9-de68-66a37fccb0ca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Bob went to the cinema and saw a lot of people who were there, but I think that's what happened.\n",
            "\n",
            "Bob went to the cinema and saw a movie. I was like, 'Oh my God! This is what you're doing!' And he said: \"I'm not going anywhere.\"\n",
            "\n",
            "Bob went to the cinema and saw a lot of people who were going through this.\n",
            "\n",
            "Bob went to the cinema and saw a lot of people. I think it was very interesting that he had this idea for me, but also because there were so many other things in life which we could do with him.\"\n",
            "\n",
            "Bob went to the cinema and found a movie.\n",
            "\n",
            "Bob went to the cinema and saw a movie called “The Last Man on Earth”. He said, \"I think it was an amazing film.\"\n",
            "\n",
            "Bob went to the cinema and saw a lot of people.\n",
            "\n",
            "Bob went to the cinema and saw a movie, but he didn't see it. He was just sitting there watching this film with his wife who had been in her car for some time now because she thought that would be funny.\"\n",
            "\n",
            "Bob went to the cinema, and he was a bit surprised.\n",
            "\n",
            "Bob went to the cinema and saw a movie, but I was just not able do that.\n",
            "\n",
            "CPU times: user 7.7 s, sys: 23.8 ms, total: 7.72 s\n",
            "Wall time: 9.8 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(10):\n",
        "  text = \"I\\'ve just seen a movie which was about a robot fighting in a city. The genre of the film is Action\\n\"+\\\n",
        "          \"I\\'ve just seen a movie which was about English lords arguing. The genre of the film is Drama\\n\"+\\\n",
        "          \"I\\'ve just seen a movie which was about the Wild West and bandits. The genre of the film is\"\n",
        "  tokens = tokenizer(text, return_tensors=\"pt\")\n",
        "\n",
        "  completion = model.generate(tokens[\"input_ids\"], max_new_tokens=3, do_sample=True, temperature=0.05, top_p=1.0, eos_token_id=lftoken, repetition_penalty=1e2)\n",
        "  print(tokenizer.decode(completion[0]))"
      ],
      "metadata": {
        "id": "IB-jzQCKaplL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3c2a7631-aee9-4234-9467-89cd8d807852"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n",
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n",
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n",
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n",
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n",
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n",
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n",
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n",
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n",
            "I've just seen a movie which was about a robot fighting in a city. The genre of the film is Action\n",
            "I've just seen a movie which was about English lords arguing. The genre of the film is Drama\n",
            "I've just seen a movie which was about the Wild West and bandits. The genre of the film is Comedy<|endoftext|>The\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "#Batched version\n",
        "\n",
        "text = [\"Bob went to the cinema\"]*10\n",
        "tokens = tokenizer(text, return_tensors=\"pt\")\n",
        "\n",
        "completion = model.generate(tokens[\"input_ids\"], max_length=64, do_sample=True, temperature=0.2, top_p=1.0, eos_token_id=lftoken, repetition_penalty=1e2)\n",
        "for i in range(10):\n",
        "  print(tokenizer.decode(completion[i]).replace(\"<|endoftext|>\",\"\"))"
      ],
      "metadata": {
        "id": "LCYWt9pRbJ8a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d88b2d2b-f10b-4694-bfa4-d4de3dd8886b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Bob went to the cinema and saw a bunch of people who were there, but they didn't know what was going on.\n",
            "\n",
            "Bob went to the cinema and saw a movie called The Last Man on screen. He was very impressed with it, but he didn't like what I said about him or anything else that could be construed as an endorsement of his views.\"\n",
            "\n",
            "Bob went to the cinema and saw a movie that was about an old man who had been murdered. He said he thought it would be funny, but then suddenly there were people saying 'I'm not going anywhere'. And I think they're right - we've got no idea what's happening here.\"\n",
            "\n",
            "Bob went to the cinema and saw a movie, but he didn't see it. He was just looking at his own life.\"\n",
            "\n",
            "Bob went to the cinema and saw a young man who was wearing his own shirt.\n",
            "\n",
            "Bob went to the cinema and saw a movie, but it was not.\n",
            "\n",
            "Bob went to the cinema and saw a lot of people. I think it was great for him, but also he had his own way in life.\"Theresource is now on sale at auction!\n",
            "\n",
            "Bob went to the cinema and saw a lot of people coming up with ideas for new movies. I think it was really interesting that they were able come across something like this, but then we got into some stuff about how much money is going on in Hollywood right now.\"\n",
            "\n",
            "Bob went to the cinema and saw a movie called “The Great American Dream�Depth. It was an amazing film, but it wasn't really that great for me because I didn´t know what would happen if my dad died or something like this happened in America.\"\n",
            "\n",
            "Bob went to the cinema and saw a movie called \"The Man Who Loved Me\" in which he was asked if it would be good for him.\n",
            "\n",
            "CPU times: user 5.95 s, sys: 28.1 ms, total: 5.98 s\n",
            "Wall time: 7.12 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Models are reusable and retrainable\n",
        "\n",
        "## Using BERT for NER tagging\n",
        "\n",
        "### BERT\n",
        "<img src=\"https://1.bp.blogspot.com/-RLAbr6kPNUo/W9is5FwUXmI/AAAAAAAADeU/5y9466Zoyoc96vqLjbruLK8i_t8qEdHnQCLcBGAs/s1600/image3.png\">\n",
        "\n",
        "- Transformer Encoder Blocks (unmasked attention)\n",
        "- Two special tokens **[CLS]** and **[SEP]**\n",
        "- Trained with MLM (Masked Language Modelling) and NSP (Next Sentence Prediction)\n",
        "\n",
        "### MLM\n",
        "<img src=\"https://jalammar.github.io/images/BERT-language-modeling-masked-lm.png\" width=80%>\n",
        "\n",
        "### NSP\n",
        "<img src=\"https://jalammar.github.io/images/bert-next-sentence-prediction.png\" width=80%>\n",
        "\n",
        "\n",
        "\n",
        "Nice visualization page: https://jalammar.github.io/illustrated-bert/\n"
      ],
      "metadata": {
        "id": "eHWtLnTlZLTK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# NER (Named Entity Recognition)\n",
        "\n",
        "Assigning a label to each token corresponding to named entities.   \n",
        "Topics vary based on use-case. Most common: GEO, ORG, PER, MISC.\n",
        "### In this lab   \n",
        "- geo = Geographical Entity\n",
        "- org = Organization\n",
        "- per = Person\n",
        "- gpe = Geopolitical Entity\n",
        "- tim = Time indicator\n",
        "- art = Artifact\n",
        "- eve = Event\n",
        "- nat = Natural Phenomenon\n",
        "\n",
        "### IBO annotation\n",
        "- I: inside\n",
        "- B: beginning\n",
        "- O: out\n",
        "\n",
        "For example:   \n",
        "\"We have been in New York this september.\"   \n",
        "\" O O O O B-GEO I-GEO O B-TIM O\" (last one is for punct)\n",
        "\n",
        "### Problem: Sub-word tokens\n",
        "Example: \n",
        "- \"processing\" is in the vocabulary\n",
        "- \"parsing\" is OOV, regular n-gram components are used here [\"p\", \"ars\", \"ing\"]\n",
        "- Tags are assigned to the first sub-word token this time.\n",
        "- Other sub-word tokens are labeled with padding tokens"
      ],
      "metadata": {
        "id": "RPNgdtPif_8t"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenizer.tokenize(\"processing\"))\n",
        "print(tokenizer.tokenize(\"parsing\"))"
      ],
      "metadata": {
        "id": "4JWTy5l4YyLt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "10d7b2fc-7cb7-4845-c0b2-c97b6ca0714e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['processing']\n",
            "['p', 'ars', 'ing']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Loading NER dataset"
      ],
      "metadata": {
        "id": "45DyjJipuuEF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "jzHRyu83p7UT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def download_from_drive(link, filename):\n",
        "  id = link.split(\"/\")[-2]\n",
        "  !wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id={id}' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\\1\\n/p')&id={id}\" -O {\"/content/\"+filename} && rm -rf /tmp/cookies.txt\n",
        "\n",
        "\n",
        "download_from_drive(\"https://drive.google.com/file/d/1c6oLXggGNS5WdD9BIJK6K9H-cNnexZfD/view?usp=sharing\",\"NER.csv\")"
      ],
      "metadata": {
        "id": "fzvAHGbEvTw9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "93a148d6-aac4-412e-e402-32cbd1abd3b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-05-22 19:59:34--  https://docs.google.com/uc?export=download&confirm=&id=1c6oLXggGNS5WdD9BIJK6K9H-cNnexZfD\n",
            "Resolving docs.google.com (docs.google.com)... 172.253.122.138, 172.253.122.102, 172.253.122.101, ...\n",
            "Connecting to docs.google.com (docs.google.com)|172.253.122.138|:443... connected.\n",
            "HTTP request sent, awaiting response... 303 See Other\n",
            "Location: https://doc-00-1k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/ge670jurn95l8d7c9rcsa4agus3g1nmh/1684785525000/02440328774490686871/*/1c6oLXggGNS5WdD9BIJK6K9H-cNnexZfD?e=download&uuid=5cc6e2e4-5585-4cb8-9be3-0a9de26a3e04 [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2023-05-22 19:59:35--  https://doc-00-1k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/ge670jurn95l8d7c9rcsa4agus3g1nmh/1684785525000/02440328774490686871/*/1c6oLXggGNS5WdD9BIJK6K9H-cNnexZfD?e=download&uuid=5cc6e2e4-5585-4cb8-9be3-0a9de26a3e04\n",
            "Resolving doc-00-1k-docs.googleusercontent.com (doc-00-1k-docs.googleusercontent.com)... 142.251.167.132, 2607:f8b0:4004:c1d::84\n",
            "Connecting to doc-00-1k-docs.googleusercontent.com (doc-00-1k-docs.googleusercontent.com)|142.251.167.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 15208151 (15M) [text/csv]\n",
            "Saving to: ‘/content/NER.csv’\n",
            "\n",
            "/content/NER.csv    100%[===================>]  14.50M  --.-KB/s    in 0.06s   \n",
            "\n",
            "2023-05-22 19:59:35 (234 MB/s) - ‘/content/NER.csv’ saved [15208151/15208151]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"NER.csv\",encoding=\"windows-1251\")\n",
        "df.head(200)"
      ],
      "metadata": {
        "id": "62oz4ISdvguG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "outputId": "9f4a5945-3595-42cf-9722-4a330e318b2a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       Sentence #           Word  POS    Tag\n",
              "0     Sentence: 1      Thousands  NNS      O\n",
              "1             NaN             of   IN      O\n",
              "2             NaN  demonstrators  NNS      O\n",
              "3             NaN           have  VBP      O\n",
              "4             NaN        marched  VBN      O\n",
              "..            ...            ...  ...    ...\n",
              "195           NaN              .    .      O\n",
              "196  Sentence: 10        Iranian   JJ  B-gpe\n",
              "197           NaN      officials  NNS      O\n",
              "198           NaN            say  VBP      O\n",
              "199           NaN           they  PRP      O\n",
              "\n",
              "[200 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c5c4f323-b848-4390-aa10-fa0863738aed\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>195</th>\n",
              "      <td>NaN</td>\n",
              "      <td>.</td>\n",
              "      <td>.</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>196</th>\n",
              "      <td>Sentence: 10</td>\n",
              "      <td>Iranian</td>\n",
              "      <td>JJ</td>\n",
              "      <td>B-gpe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>197</th>\n",
              "      <td>NaN</td>\n",
              "      <td>officials</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>198</th>\n",
              "      <td>NaN</td>\n",
              "      <td>say</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>199</th>\n",
              "      <td>NaN</td>\n",
              "      <td>they</td>\n",
              "      <td>PRP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>200 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c5c4f323-b848-4390-aa10-fa0863738aed')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c5c4f323-b848-4390-aa10-fa0863738aed button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c5c4f323-b848-4390-aa10-fa0863738aed');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"Tag\"].value_counts()"
      ],
      "metadata": {
        "id": "TThB3joVX5wT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2417ef14-a17e-418a-8f81-35fff2b246ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "O        887908\n",
              "B-geo     37644\n",
              "B-tim     20333\n",
              "B-org     20143\n",
              "I-per     17251\n",
              "B-per     16990\n",
              "I-org     16784\n",
              "B-gpe     15870\n",
              "I-geo      7414\n",
              "I-tim      6528\n",
              "B-art       402\n",
              "B-eve       308\n",
              "I-art       297\n",
              "I-eve       253\n",
              "B-nat       201\n",
              "I-gpe       198\n",
              "I-nat        51\n",
              "Name: Tag, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Imbalanced dataset, we will have to deal with that later.\n",
        "\n",
        "Creating NER <--> class id mapping"
      ],
      "metadata": {
        "id": "zk06ffmdmCw0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "id_to_ner = dict(enumerate(df[\"Tag\"].unique()))"
      ],
      "metadata": {
        "id": "xKJ-n4FmYUVq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "id_to_ner"
      ],
      "metadata": {
        "id": "o-A0k7sDYrv3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "535296ea-3071-4b7c-a151-0d0bf4dd6edf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'O',\n",
              " 1: 'B-geo',\n",
              " 2: 'B-gpe',\n",
              " 3: 'B-per',\n",
              " 4: 'I-geo',\n",
              " 5: 'B-org',\n",
              " 6: 'I-org',\n",
              " 7: 'B-tim',\n",
              " 8: 'B-art',\n",
              " 9: 'I-art',\n",
              " 10: 'I-per',\n",
              " 11: 'I-gpe',\n",
              " 12: 'I-tim',\n",
              " 13: 'B-nat',\n",
              " 14: 'B-eve',\n",
              " 15: 'I-eve',\n",
              " 16: 'I-nat'}"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "id_to_ner[-100]=\"[PAD]\""
      ],
      "metadata": {
        "id": "wsM_2FvaYwHD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ner_to_id = {v: k for k, v in id_to_ner.items()}\n",
        "ner_to_id"
      ],
      "metadata": {
        "id": "A04k4WgpZFGN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0841e3a-4e65-42f7-fb48-9331ace1ef68"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'O': 0,\n",
              " 'B-geo': 1,\n",
              " 'B-gpe': 2,\n",
              " 'B-per': 3,\n",
              " 'I-geo': 4,\n",
              " 'B-org': 5,\n",
              " 'I-org': 6,\n",
              " 'B-tim': 7,\n",
              " 'B-art': 8,\n",
              " 'I-art': 9,\n",
              " 'I-per': 10,\n",
              " 'I-gpe': 11,\n",
              " 'I-tim': 12,\n",
              " 'B-nat': 13,\n",
              " 'B-eve': 14,\n",
              " 'I-eve': 15,\n",
              " 'I-nat': 16,\n",
              " '[PAD]': -100}"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Convert to sentence-based dataset.\n",
        "\n",
        "Forwardfilling NaN values will fill missing sentence labels until a new one is met"
      ],
      "metadata": {
        "id": "Pj0GClfymNvD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.fillna(method=\"ffill\")\n",
        "df"
      ],
      "metadata": {
        "id": "53rNgKM7xhlK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "outputId": "394404ab-8e9f-43ed-93c3-f7142651f759"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "              Sentence #           Word  POS Tag\n",
              "0            Sentence: 1      Thousands  NNS   O\n",
              "1            Sentence: 1             of   IN   O\n",
              "2            Sentence: 1  demonstrators  NNS   O\n",
              "3            Sentence: 1           have  VBP   O\n",
              "4            Sentence: 1        marched  VBN   O\n",
              "...                  ...            ...  ...  ..\n",
              "1048570  Sentence: 47959           they  PRP   O\n",
              "1048571  Sentence: 47959      responded  VBD   O\n",
              "1048572  Sentence: 47959             to   TO   O\n",
              "1048573  Sentence: 47959            the   DT   O\n",
              "1048574  Sentence: 47959         attack   NN   O\n",
              "\n",
              "[1048575 rows x 4 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a0b178dc-b704-4c12-bc32-2f76edda10d7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048570</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>they</td>\n",
              "      <td>PRP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048571</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>responded</td>\n",
              "      <td>VBD</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048572</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>to</td>\n",
              "      <td>TO</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048573</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>the</td>\n",
              "      <td>DT</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048574</th>\n",
              "      <td>Sentence: 47959</td>\n",
              "      <td>attack</td>\n",
              "      <td>NN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1048575 rows × 4 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a0b178dc-b704-4c12-bc32-2f76edda10d7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a0b178dc-b704-4c12-bc32-2f76edda10d7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a0b178dc-b704-4c12-bc32-2f76edda10d7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Group words and NER tags by sentences and drop the duplicate rows"
      ],
      "metadata": {
        "id": "zK4WF0svmWKU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sdf = df[[\"Word\",\"Tag\"]].groupby(df[\"Sentence #\"]).transform(lambda x: \" \".join(x))\n",
        "sdf"
      ],
      "metadata": {
        "id": "6xVShaRu0LjD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "outputId": "ae624e03-6797-42b0-aff4-aab38f0cffff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                      Word  \\\n",
              "0        Thousands of demonstrators have marched throug...   \n",
              "1        Thousands of demonstrators have marched throug...   \n",
              "2        Thousands of demonstrators have marched throug...   \n",
              "3        Thousands of demonstrators have marched throug...   \n",
              "4        Thousands of demonstrators have marched throug...   \n",
              "...                                                    ...   \n",
              "1048570    Indian forces said they responded to the attack   \n",
              "1048571    Indian forces said they responded to the attack   \n",
              "1048572    Indian forces said they responded to the attack   \n",
              "1048573    Indian forces said they responded to the attack   \n",
              "1048574    Indian forces said they responded to the attack   \n",
              "\n",
              "                                                       Tag  \n",
              "0        O O O O O O B-geo O O O O O B-geo O O O O O B-...  \n",
              "1        O O O O O O B-geo O O O O O B-geo O O O O O B-...  \n",
              "2        O O O O O O B-geo O O O O O B-geo O O O O O B-...  \n",
              "3        O O O O O O B-geo O O O O O B-geo O O O O O B-...  \n",
              "4        O O O O O O B-geo O O O O O B-geo O O O O O B-...  \n",
              "...                                                    ...  \n",
              "1048570                                B-gpe O O O O O O O  \n",
              "1048571                                B-gpe O O O O O O O  \n",
              "1048572                                B-gpe O O O O O O O  \n",
              "1048573                                B-gpe O O O O O O O  \n",
              "1048574                                B-gpe O O O O O O O  \n",
              "\n",
              "[1048575 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0ee30425-be50-46be-8b5d-aa87a0316a4d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Word</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O O O O O O B-geo O O O O O B-geo O O O O O B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O O O O O O B-geo O O O O O B-geo O O O O O B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O O O O O O B-geo O O O O O B-geo O O O O O B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O O O O O O B-geo O O O O O B-geo O O O O O B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O O O O O O B-geo O O O O O B-geo O O O O O B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048570</th>\n",
              "      <td>Indian forces said they responded to the attack</td>\n",
              "      <td>B-gpe O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048571</th>\n",
              "      <td>Indian forces said they responded to the attack</td>\n",
              "      <td>B-gpe O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048572</th>\n",
              "      <td>Indian forces said they responded to the attack</td>\n",
              "      <td>B-gpe O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048573</th>\n",
              "      <td>Indian forces said they responded to the attack</td>\n",
              "      <td>B-gpe O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048574</th>\n",
              "      <td>Indian forces said they responded to the attack</td>\n",
              "      <td>B-gpe O O O O O O O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1048575 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0ee30425-be50-46be-8b5d-aa87a0316a4d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0ee30425-be50-46be-8b5d-aa87a0316a4d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0ee30425-be50-46be-8b5d-aa87a0316a4d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sdf = sdf.drop_duplicates()\n",
        "sdf"
      ],
      "metadata": {
        "id": "A9UB13XoPLR_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "outputId": "014ab610-ac15-49e8-f4b7-6b79020efecb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                      Word  \\\n",
              "0        Thousands of demonstrators have marched throug...   \n",
              "24       Families of soldiers killed in the conflict jo...   \n",
              "54       They marched from the Houses of Parliament to ...   \n",
              "68       Police put the number of marchers at 10,000 wh...   \n",
              "83       The protest comes on the eve of the annual con...   \n",
              "...                                                    ...   \n",
              "1048501  Indian border security forces are accusing the...   \n",
              "1048521  Indian officials said no one was injured in Sa...   \n",
              "1048545  Two more landed in fields belonging to a nearb...   \n",
              "1048556  They say not all of the rockets exploded upon ...   \n",
              "1048567    Indian forces said they responded to the attack   \n",
              "\n",
              "                                                       Tag  \n",
              "0        O O O O O O B-geo O O O O O B-geo O O O O O B-...  \n",
              "24       O O O O O O O O O O O O O O O O O O B-per O O ...  \n",
              "54                     O O O O O O O O O O O B-geo I-geo O  \n",
              "68                           O O O O O O O O O O O O O O O  \n",
              "83       O O O O O O O O O O O B-geo O O B-org I-org O ...  \n",
              "...                                                    ...  \n",
              "1048501  B-gpe O O O O O O B-gpe O O O O O O O O O B-ge...  \n",
              "1048521  B-gpe O O O O O O O B-tim O O O O O O O O O O ...  \n",
              "1048545                              O O O O O O O O O O O  \n",
              "1048556                              O O O O O O O O O O O  \n",
              "1048567                                B-gpe O O O O O O O  \n",
              "\n",
              "[47610 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3baa62bc-c080-4953-9857-0d521bfd6e1b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Word</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O O O O O O B-geo O O O O O B-geo O O O O O B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>Families of soldiers killed in the conflict jo...</td>\n",
              "      <td>O O O O O O O O O O O O O O O O O O B-per O O ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>They marched from the Houses of Parliament to ...</td>\n",
              "      <td>O O O O O O O O O O O B-geo I-geo O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68</th>\n",
              "      <td>Police put the number of marchers at 10,000 wh...</td>\n",
              "      <td>O O O O O O O O O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>83</th>\n",
              "      <td>The protest comes on the eve of the annual con...</td>\n",
              "      <td>O O O O O O O O O O O B-geo O O B-org I-org O ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048501</th>\n",
              "      <td>Indian border security forces are accusing the...</td>\n",
              "      <td>B-gpe O O O O O O B-gpe O O O O O O O O O B-ge...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048521</th>\n",
              "      <td>Indian officials said no one was injured in Sa...</td>\n",
              "      <td>B-gpe O O O O O O O B-tim O O O O O O O O O O ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048545</th>\n",
              "      <td>Two more landed in fields belonging to a nearb...</td>\n",
              "      <td>O O O O O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048556</th>\n",
              "      <td>They say not all of the rockets exploded upon ...</td>\n",
              "      <td>O O O O O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1048567</th>\n",
              "      <td>Indian forces said they responded to the attack</td>\n",
              "      <td>B-gpe O O O O O O O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>47610 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3baa62bc-c080-4953-9857-0d521bfd6e1b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3baa62bc-c080-4953-9857-0d521bfd6e1b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3baa62bc-c080-4953-9857-0d521bfd6e1b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Resetting indices for better handling (optional)"
      ],
      "metadata": {
        "id": "Yl8PM8A0meGK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sdf = sdf.reset_index(drop=True)\n",
        "sdf"
      ],
      "metadata": {
        "id": "-9PzWT-nSZ6a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        },
        "outputId": "19c3dabf-6880-47d6-df65-4fd96e647934"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    Word  \\\n",
              "0      Thousands of demonstrators have marched throug...   \n",
              "1      Families of soldiers killed in the conflict jo...   \n",
              "2      They marched from the Houses of Parliament to ...   \n",
              "3      Police put the number of marchers at 10,000 wh...   \n",
              "4      The protest comes on the eve of the annual con...   \n",
              "...                                                  ...   \n",
              "47605  Indian border security forces are accusing the...   \n",
              "47606  Indian officials said no one was injured in Sa...   \n",
              "47607  Two more landed in fields belonging to a nearb...   \n",
              "47608  They say not all of the rockets exploded upon ...   \n",
              "47609    Indian forces said they responded to the attack   \n",
              "\n",
              "                                                     Tag  \n",
              "0      O O O O O O B-geo O O O O O B-geo O O O O O B-...  \n",
              "1      O O O O O O O O O O O O O O O O O O B-per O O ...  \n",
              "2                    O O O O O O O O O O O B-geo I-geo O  \n",
              "3                          O O O O O O O O O O O O O O O  \n",
              "4      O O O O O O O O O O O B-geo O O B-org I-org O ...  \n",
              "...                                                  ...  \n",
              "47605  B-gpe O O O O O O B-gpe O O O O O O O O O B-ge...  \n",
              "47606  B-gpe O O O O O O O B-tim O O O O O O O O O O ...  \n",
              "47607                              O O O O O O O O O O O  \n",
              "47608                              O O O O O O O O O O O  \n",
              "47609                                B-gpe O O O O O O O  \n",
              "\n",
              "[47610 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-869a9a99-badd-4326-bb3a-962f0d25b9b9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Word</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O O O O O O B-geo O O O O O B-geo O O O O O B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Families of soldiers killed in the conflict jo...</td>\n",
              "      <td>O O O O O O O O O O O O O O O O O O B-per O O ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>They marched from the Houses of Parliament to ...</td>\n",
              "      <td>O O O O O O O O O O O B-geo I-geo O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Police put the number of marchers at 10,000 wh...</td>\n",
              "      <td>O O O O O O O O O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The protest comes on the eve of the annual con...</td>\n",
              "      <td>O O O O O O O O O O O B-geo O O B-org I-org O ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47605</th>\n",
              "      <td>Indian border security forces are accusing the...</td>\n",
              "      <td>B-gpe O O O O O O B-gpe O O O O O O O O O B-ge...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47606</th>\n",
              "      <td>Indian officials said no one was injured in Sa...</td>\n",
              "      <td>B-gpe O O O O O O O B-tim O O O O O O O O O O ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47607</th>\n",
              "      <td>Two more landed in fields belonging to a nearb...</td>\n",
              "      <td>O O O O O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47608</th>\n",
              "      <td>They say not all of the rockets exploded upon ...</td>\n",
              "      <td>O O O O O O O O O O O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47609</th>\n",
              "      <td>Indian forces said they responded to the attack</td>\n",
              "      <td>B-gpe O O O O O O O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>47610 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-869a9a99-badd-4326-bb3a-962f0d25b9b9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-869a9a99-badd-4326-bb3a-962f0d25b9b9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-869a9a99-badd-4326-bb3a-962f0d25b9b9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load BertTokenizerFast for tokenization and TFBertForTokenClassification (TF means tensorflow compatibility here)"
      ],
      "metadata": {
        "id": "krmJH3p6mhJN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertTokenizerFast, TFBertForTokenClassification"
      ],
      "metadata": {
        "id": "cLf_-Kz70miz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Use the cased version, as upper-case is a clue for a large number of entities"
      ],
      "metadata": {
        "id": "_Q9JCViAmrmY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = BertTokenizerFast.from_pretrained(\"bert-base-cased\")"
      ],
      "metadata": {
        "id": "hA0yBI8cRtRe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "48677b1c79e0430b9b51fb6869bc50cb",
            "77cb5bcf90a04d4da37fcb97dcadd58d",
            "0c271299eb1545cb81c8ddb456df2cc8",
            "f2fcc80f8c2d40ce9ee546adfb4fcee7",
            "44a0f774507e430e9e3e10ee10f6df1d",
            "f70145bac54745318408932e3d47f8af",
            "a5da0b72d8294aedbaa89ecc8c20f74a",
            "8f55d96e3ff84350847a928190b8139c",
            "628ac92ac029498a8a90d813a3a0cf3b",
            "ab8a641fd14248589ead87232ed5cb8d",
            "0b799bac44db4eabb7320c2a2d85c8ad",
            "36b95bd1e9bd409ebc82c02d208c8e2b",
            "c56820bd06b540c8b5647fbcb18780c8",
            "1273a7c789ff4216abd8619d84774497",
            "747da68f204240cf870a57a0ea9e865b",
            "4e98d57070de4c68835805ff89bc58bc",
            "0b7e8cd49a034c0ea91c37db1f8dacd8",
            "e66ea9047be54a5da548d2037567795c",
            "8d3d4fd7558941a4a1f9820776557991",
            "58f062d674ec43a9b23ae7677262e1bb",
            "5da33df9f7cf4cdba3cc3b09a1e0a0c0",
            "94f526c2e79242d18e588b343b657a6f",
            "d38577c1068041a6a8f3742f3685c523",
            "422f3a877b1d4164b4dbeeae094e1b0e",
            "c1e0d5e1ef9f439cb49a88701ec6630b",
            "12ff357205384726b03fa1f8194c5d56",
            "653000dd65f04d7ebf4aec140168de9a",
            "a8d65f2e705e4a78bb1a2b30a1cfa6b7",
            "a90a5408fbaa45f58c9d6181918a0801",
            "4bc76cf7b76142f7810319935fc0ec2c",
            "482d4a9aa5964253a59bb3cf8638142d",
            "a606a1a07d904bf293160bced14a9b2d",
            "68aa1f19151a4a7f903a77a5313423ab",
            "33b09a1367f846a0ac6d1901f943a42d",
            "01fa6dae96314b9383ba693f2ba5f55f",
            "2f167b681bdc4f1285dbe16782a8f9b9",
            "9d34fb699e1f4b5da69b38d8c73f8765",
            "f7313e0425474b8b822d415756bcf0c0",
            "f3ce1afcf17d47b1824e0e8e470310c0",
            "a93e01ce92b84c068f11a7346c80d1b7",
            "891067e31e6541699a8aa597b33562a7",
            "66ec4b30836349edbf63a260f51a985e",
            "824235bba40445238a4bbf18475bb171",
            "08f0bddb402146e1b3016969b0d3d5b9"
          ]
        },
        "outputId": "d7eac55e-65fe-4288-beae-00a517d797ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "48677b1c79e0430b9b51fb6869bc50cb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/213k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "36b95bd1e9bd409ebc82c02d208c8e2b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/436k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d38577c1068041a6a8f3742f3685c523"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "33b09a1367f846a0ac6d1901f943a42d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Subword tokens are present, only the leading tokens should get a NER tag each word."
      ],
      "metadata": {
        "id": "Sjhwa1F9mqGf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text = sdf[\"Word\"][7]\n",
        "tokenizer.tokenize(text)"
      ],
      "metadata": {
        "id": "6p17dvtTR25O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "095ef14c-4d9d-44ab-edfa-05c13b5bd999"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['The',\n",
              " 'International',\n",
              " 'Atomic',\n",
              " 'Energy',\n",
              " 'Agency',\n",
              " 'is',\n",
              " 'to',\n",
              " 'hold',\n",
              " 'second',\n",
              " 'day',\n",
              " 'of',\n",
              " 'talks',\n",
              " 'in',\n",
              " 'Vienna',\n",
              " 'Wednesday',\n",
              " 'on',\n",
              " 'how',\n",
              " 'to',\n",
              " 'respond',\n",
              " 'to',\n",
              " 'Iran',\n",
              " \"'\",\n",
              " 's',\n",
              " 're',\n",
              " '##sumption',\n",
              " 'of',\n",
              " 'low',\n",
              " '-',\n",
              " 'level',\n",
              " 'uranium',\n",
              " 'conversion',\n",
              " '.']"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer(text.split(\" \"), is_split_into_words=True, add_special_tokens=True, \n",
        "          max_length=64, padding=\"max_length\", return_attention_mask=True, \n",
        "          return_offsets_mapping=True)\n",
        "  "
      ],
      "metadata": {
        "id": "fjHWj4pXX1FZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cb4a210c-cd49-4894-a65e-413ac45e8d61"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [101, 1109, 1570, 18662, 5514, 5571, 1110, 1106, 2080, 1248, 1285, 1104, 7430, 1107, 5337, 9031, 1113, 1293, 1106, 6297, 1106, 3398, 112, 188, 1231, 16937, 1104, 1822, 118, 1634, 16410, 7497, 119, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'offset_mapping': [(0, 0), (0, 3), (0, 13), (0, 6), (0, 6), (0, 6), (0, 2), (0, 2), (0, 4), (0, 6), (0, 3), (0, 2), (0, 5), (0, 2), (0, 6), (0, 9), (0, 2), (0, 3), (0, 2), (0, 7), (0, 2), (0, 4), (0, 1), (1, 2), (0, 2), (2, 10), (0, 2), (0, 3), (3, 4), (4, 9), (0, 7), (0, 10), (0, 1), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)]}"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf"
      ],
      "metadata": {
        "id": "dfrSTDEBhkro"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Assembling sequences.   \n",
        "[PAD] and non-leading subword tokens are denoted with NER token ID -100"
      ],
      "metadata": {
        "id": "PP4gxIMeC1Ik"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def prepare_training_data(text, ner=None):\n",
        "  ids, _, att, off = tokenizer(text.split(\" \"), is_split_into_words=True, add_special_tokens=True, \n",
        "                            max_length=64, padding=\"max_length\", truncation=True, return_attention_mask=True, \n",
        "                            return_offsets_mapping=True).values()\n",
        "  ids = np.array(ids)\n",
        "  att = np.array(att)\n",
        "  label = np.ones_like(ids, dtype=np.int32)*-100\n",
        "  if ner is not None:\n",
        "    for i, offset in enumerate(off):\n",
        "      if offset[0]==0 and offset[1]!=0:\n",
        "        label[i] = ner_to_id[ner.pop(0)]\n",
        "  \n",
        "  return ids, att, label"
      ],
      "metadata": {
        "id": "fRGHEyhBSKmQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prepare_training_data(\"This is a test sentence\",[\"O\",\"O\",\"O\",\"B-geo\",\"O\"])"
      ],
      "metadata": {
        "id": "CyOnHHxeUd9j",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a48f2734-bb50-4381-d7be-ec3c114fcd2c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([ 101, 1188, 1110,  170, 2774, 5650,  102,    0,    0,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "           0,    0,    0,    0,    0,    0,    0,    0,    0]),\n",
              " array([1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
              " array([-100,    0,    0,    0,    1,    0, -100, -100, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100], dtype=int32))"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prepare_training_data(\"This is a teszt sentence\",[\"O\",\"O\",\"O\",\"B-geo\",\"O\"])"
      ],
      "metadata": {
        "id": "krcPfM0XWZIV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "06e056c5-b079-4936-91fb-801b4240d3f7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([  101,  1188,  1110,   170, 21359, 21529,  1204,  5650,   102,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "            0]),\n",
              " array([1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
              " array([-100,    0,    0,    0,    1, -100, -100,    0, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "        -100, -100, -100, -100, -100, -100, -100, -100, -100], dtype=int32))"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def prepare_df(series):\n",
        "  series[\"ids\"], series[\"att\"], series[\"label\"] = prepare_training_data(series[\"Word\"],series[\"Tag\"].split(\" \"))\n",
        "  return series"
      ],
      "metadata": {
        "id": "0igw4lxmZv4r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sdf=sdf.apply(prepare_df,axis=1)\n",
        "sdf.head()"
      ],
      "metadata": {
        "id": "JdMhc-0Zbr2z",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "d920a863-cc49-4c88-98ec-d43c1595a230"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                Word  \\\n",
              "0  Thousands of demonstrators have marched throug...   \n",
              "1  Families of soldiers killed in the conflict jo...   \n",
              "2  They marched from the Houses of Parliament to ...   \n",
              "3  Police put the number of marchers at 10,000 wh...   \n",
              "4  The protest comes on the eve of the annual con...   \n",
              "\n",
              "                                                 Tag  \\\n",
              "0  O O O O O O B-geo O O O O O B-geo O O O O O B-...   \n",
              "1  O O O O O O O O O O O O O O O O O O B-per O O ...   \n",
              "2                O O O O O O O O O O O B-geo I-geo O   \n",
              "3                      O O O O O O O O O O O O O O O   \n",
              "4  O O O O O O O O O O O B-geo O O B-org I-org O ...   \n",
              "\n",
              "                                                 ids  \\\n",
              "0  [101, 26159, 1104, 8568, 4487, 5067, 1138, 963...   \n",
              "1  [101, 21860, 1104, 2803, 1841, 1107, 1103, 413...   \n",
              "2  [101, 1220, 9639, 1121, 1103, 16862, 1104, 290...   \n",
              "3  [101, 3284, 1508, 1103, 1295, 1104, 8943, 1468...   \n",
              "4  [101, 1109, 5641, 2502, 1113, 1103, 23491, 110...   \n",
              "\n",
              "                                                 att  \\\n",
              "0  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
              "1  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
              "2  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
              "3  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
              "4  [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...   \n",
              "\n",
              "                                               label  \n",
              "0  [-100, 0, 0, 0, -100, -100, 0, 0, 0, 1, 0, 0, ...  \n",
              "1  [-100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
              "2  [-100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, ...  \n",
              "3  [-100, 0, 0, 0, 0, 0, 0, -100, 0, 0, -100, -10...  \n",
              "4  [-100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5d3978cc-e542-428d-bc81-7211ff10761c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Word</th>\n",
              "      <th>Tag</th>\n",
              "      <th>ids</th>\n",
              "      <th>att</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Thousands of demonstrators have marched throug...</td>\n",
              "      <td>O O O O O O B-geo O O O O O B-geo O O O O O B-...</td>\n",
              "      <td>[101, 26159, 1104, 8568, 4487, 5067, 1138, 963...</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
              "      <td>[-100, 0, 0, 0, -100, -100, 0, 0, 0, 1, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Families of soldiers killed in the conflict jo...</td>\n",
              "      <td>O O O O O O O O O O O O O O O O O O B-per O O ...</td>\n",
              "      <td>[101, 21860, 1104, 2803, 1841, 1107, 1103, 413...</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
              "      <td>[-100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>They marched from the Houses of Parliament to ...</td>\n",
              "      <td>O O O O O O O O O O O B-geo I-geo O</td>\n",
              "      <td>[101, 1220, 9639, 1121, 1103, 16862, 1104, 290...</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
              "      <td>[-100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Police put the number of marchers at 10,000 wh...</td>\n",
              "      <td>O O O O O O O O O O O O O O O</td>\n",
              "      <td>[101, 3284, 1508, 1103, 1295, 1104, 8943, 1468...</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
              "      <td>[-100, 0, 0, 0, 0, 0, 0, -100, 0, 0, -100, -10...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The protest comes on the eve of the annual con...</td>\n",
              "      <td>O O O O O O O O O O O B-geo O O B-org I-org O ...</td>\n",
              "      <td>[101, 1109, 5641, 2502, 1113, 1103, 23491, 110...</td>\n",
              "      <td>[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...</td>\n",
              "      <td>[-100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5d3978cc-e542-428d-bc81-7211ff10761c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5d3978cc-e542-428d-bc81-7211ff10761c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5d3978cc-e542-428d-bc81-7211ff10761c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Splitting df into datasets for training, validation and testing"
      ],
      "metadata": {
        "id": "J5nu8ej3nF8o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ids_train, ids_val, ids_test = np.split(sdf[\"ids\"].sample(frac=1, random_state=1), [int(0.6*len(sdf)), int(0.8*len(sdf))])\n",
        "att_train, att_val, att_test = np.split(sdf[\"att\"].sample(frac=1, random_state=1), [int(0.6*len(sdf)), int(0.8*len(sdf))])\n",
        "Y_train, Y_val, Y_test = np.split(sdf[\"label\"].sample(frac=1, random_state=1), [int(0.6*len(sdf)), int(0.8*len(sdf))])\n",
        "\n",
        "print(ids_train, end=\"\\n\\n\")\n",
        "print(ids_train.to_numpy())"
      ],
      "metadata": {
        "id": "Qi6oxLj-jLgz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e96bd01e-6c58-4f89-f855-f49e05cd7819"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "29322    [101, 5459, 117, 1107, 14465, 117, 5229, 1104,...\n",
            "42836    [101, 1109, 19342, 1138, 4963, 1106, 4010, 497...\n",
            "13609    [101, 1109, 2061, 1585, 1110, 26943, 1158, 112...\n",
            "11034    [101, 1109, 4711, 2084, 112, 188, 7430, 1114, ...\n",
            "6405     [101, 1109, 1291, 3225, 6534, 10777, 1199, 122...\n",
            "                               ...                        \n",
            "25575    [101, 1448, 4229, 3300, 1867, 1103, 8602, 1108...\n",
            "32856    [101, 1109, 2084, 1110, 1103, 1148, 1809, 2301...\n",
            "24946    [101, 1109, 1244, 3854, 1867, 1120, 1655, 1955...\n",
            "17090    [101, 9578, 112, 188, 2084, 1144, 9497, 10299,...\n",
            "5859     [101, 2677, 1104, 1103, 1300, 12555, 1132, 635...\n",
            "Name: ids, Length: 28566, dtype: object\n",
            "\n",
            "[array([  101,  5459,   117,  1107, 14465,   117,  5229,  1104,  6638,\n",
            "         1104,  8276, 14104,   112,  1122,  1162,   172,  2879,  1596,\n",
            "        12556,  4426, 18518,  1161,  2393,   118, 20572,  1197,  7160,\n",
            "         1222,  3675,  1111,  2877,  1863,  1107,  5008,   119,   102,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0])\n",
            " array([  101,  1109, 19342,  1138,  4963,  1106,  4010,  4979,   117,\n",
            "         1497,   117,  1105,  4236, 17160,  1107,  1103,  8988,  6835,\n",
            "          119,   102,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0])\n",
            " array([  101,  1109,  2061,  1585,  1110, 26943,  1158,  1126,  8851,\n",
            "         4878,   118,  8988,  7966,  1107,  4498,   117,  3516,  1122,\n",
            "         1126, 11653,  2585,  1115,  2196,  1241,  3091,  1328,  1106,\n",
            "        16414,  1103,  3767,  1111,  3519,   119,   102,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0])                                                        ...\n",
            " array([  101,  1109,  1244,  3854,  1867,  1120,  1655,  1955,  1234,\n",
            "         1127,  1841,  1107,  1103,  8672,  5455,   117,  1134, 13746,\n",
            "         1835, 22052,  1222,  1103,  1764,  1433,   119,   102,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0])\n",
            " array([  101,  9578,   112,   188,  2084,  1144,  9497, 10299,   112,\n",
            "          188,  1433,  1105,  9283,  1107,  1103,  2466, 23612, 14703,\n",
            "         1197,  1805,  1106,  4161,   170,  8038, 15079, 14092,   118,\n",
            "         1783,   117,  1105, 14197,  1126,  1322,  1106,  1147,  1160,\n",
            "          118,  1214,  2987,  1594,   119,   102,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0])\n",
            " array([  101,  2677,  1104,  1103,  1300, 12555,  1132,  6351,  1106,\n",
            "         1129,  9651,   139, 20376,  2316,   119,   102,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0])                                                       ]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Convert the series of numpy arrays into a single array"
      ],
      "metadata": {
        "id": "OVQPd5YgDACG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def conv_np_stack(series):\n",
        "  return np.vstack(series.to_numpy())"
      ],
      "metadata": {
        "id": "rwDYwZNargN-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train = (conv_np_stack(ids_train), conv_np_stack(att_train))\n",
        "X_val = (conv_np_stack(ids_val), conv_np_stack(att_val))\n",
        "X_test = (conv_np_stack(ids_test), conv_np_stack(att_test))\n",
        "\n",
        "Y_train = np.vstack(Y_train)\n",
        "Y_val = np.vstack(Y_val)\n",
        "Y_test = np.vstack(Y_test)\n",
        "\n",
        "print(X_train[0].shape)"
      ],
      "metadata": {
        "id": "oKRz6mj7qMBj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3f2d9bb7-390b-4754-f107-47cf0e7a21d4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(28566, 64)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "7-wGKdeKihzv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Highly imbalanced dataset, let's assign class weights for loss calculation   \n",
        "(i.e.: the O class will have a low loss, the less frequent classes have high losses)"
      ],
      "metadata": {
        "id": "wAptCWyoDKQb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.utils import class_weight\n",
        "\n",
        "class_weights = class_weight.compute_class_weight('balanced',classes=np.unique(Y_train[np.bitwise_and(Y_train!=-100,Y_train!=0)]), y=Y_train[np.bitwise_and(Y_train!=-100,Y_train!=0)])\n",
        "class_weights = list(class_weights)\n",
        "class_weights.insert(0,1)\n",
        "class_weights = np.array(class_weights)\n",
        "class_weights"
      ],
      "metadata": {
        "id": "03SwbpKL4iJU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "931c0fe3-f166-4b03-e63f-56178343fa70"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([  1.        ,   0.26942918,   0.6332406 ,   0.58805323,\n",
              "         1.37346311,   0.50143392,   0.59174514,   0.49114558,\n",
              "        24.92665289,  31.25518135,   0.57713835,  51.12076271,\n",
              "         1.52175832,  46.76162791,  34.27414773,  44.0310219 ,\n",
              "       177.41911765])"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#sqrt for less agressive corrections\n",
        "class_weights = tf.convert_to_tensor(class_weights**(1/2))\n",
        "class_weights"
      ],
      "metadata": {
        "id": "mOqjzhKa55jl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f7521520-665e-45f2-dcc7-451c01cefa5e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(17,), dtype=float64, numpy=\n",
              "array([ 1.        ,  0.51906568,  0.79576416,  0.76684629,  1.17194843,\n",
              "        0.70811999,  0.76924973,  0.70081779,  4.9926599 ,  5.59063336,\n",
              "        0.75969622,  7.14987851,  1.23359569,  6.83824743,  5.85441267,\n",
              "        6.63558753, 13.31987679])>"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Custom Loss Function\n",
        "\n",
        "1. Use categorical crossentropy as base\n",
        "  - Sparse: The labels are specified as class IDs, not as vectors\n",
        "  - From logits (False): We did use softmax\n",
        "2. The NER token for PAD causes an error (there is no -100th element)\n",
        "  - Replace it with meaningful values (0 in this example)\n",
        "  - Mask the loss values (multiply by 0) coming from these tokens (as they are irrelevant)\n",
        "3. Apply class weights from above\n",
        "  - tf.gather() gathers all the weights specified by the tokenID values\n",
        "\n",
        "### Custom Accuracy Functions\n",
        "1. acc_all is the non-[PAD] token accuracy\n",
        "2. acc_NER is the non-[PAD], non-O token accuracy"
      ],
      "metadata": {
        "id": "Skmm0mbHDYvA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "@tf.function\n",
        "def loss_function(y_true, y_pred):\n",
        "  others = tf.equal(y_true, 0) # NER TAG: O\n",
        "  pads = tf.equal(y_true, -100) # NER TAG: [PAD]\n",
        "  rand_leave_negative_examples = tf.math.greater(tf.random.uniform(tf.shape(pads)),0.03)\n",
        "\n",
        "  #NOT(PAD | (O_tags & should be left out))\n",
        "  mask = tf.logical_not(tf.math.logical_or(pads, tf.logical_and(rand_leave_negative_examples,others)))\n",
        "  \n",
        "  y_nopad = tf.where(pads,0,y_true)\n",
        "  \n",
        "  loss = tf.keras.losses.sparse_categorical_crossentropy(y_nopad, y_pred, from_logits=False)\n",
        "  loss *= tf.cast(mask, loss.dtype)\n",
        "  class_weight = tf.cast(tf.gather(class_weights,y_nopad), loss.dtype)\n",
        "  loss *= class_weight\n",
        "  return tf.reduce_sum(loss)/(tf.reduce_sum(tf.cast(mask, loss.dtype))+1e-8)\n",
        "\n",
        "@tf.function\n",
        "def acc_all(real, pred):\n",
        "    pads = tf.math.equal(real, -100)\n",
        "    mask = tf.math.logical_not(pads)\n",
        "    \n",
        "    acc = tf.equal(real+100*tf.cast(pads, dtype=real.dtype), tf.cast(tf.argmax(pred, axis=-1),dtype=real.dtype))\n",
        "    acc = tf.cast(tf.logical_and(acc,mask), dtype=tf.float32)\n",
        "\n",
        "    return tf.reduce_sum(acc)/(tf.reduce_sum(tf.cast(mask,dtype=tf.float32))+1e-8)\n",
        "\n",
        "@tf.function\n",
        "def acc_NER(real, pred):\n",
        "    ignored = tf.logical_or(tf.math.equal(real, -100), tf.math.equal(real, 0))\n",
        "    mask = tf.math.logical_not(ignored)\n",
        "    \n",
        "    acc = tf.equal(real+100*tf.cast(ignored, dtype=real.dtype), tf.cast(tf.argmax(pred, axis=-1),dtype=real.dtype))\n",
        "    acc = tf.cast(tf.logical_and(acc,mask),dtype=tf.float32)\n",
        "\n",
        "    return tf.reduce_sum(acc)/(tf.reduce_sum(tf.cast(mask,dtype=tf.float32))+1e-8)\n"
      ],
      "metadata": {
        "id": "2Xmwfg6Obtw5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load model, specify the number of outputs (PAD is not needed)"
      ],
      "metadata": {
        "id": "FryWDRbqDFzc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = TFBertForTokenClassification.from_pretrained(\"bert-base-cased\", num_labels=len(ner_to_id)-1)"
      ],
      "metadata": {
        "id": "G-HSko1Qt3nj",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121,
          "referenced_widgets": [
            "cca18c68475644aea3dc8f08dfab9f9b",
            "100388d0561b4d9985b5d468a5958cbf",
            "a20c758ce3f345b39821edc56b17cee2",
            "f83777b234394b36ac708958906cda7a",
            "630aff84c9ae4151933bc1ca31781607",
            "37d7d0e0cd51407f89312398e06b5887",
            "559f052776a448198c908aa1a4ba016c",
            "dfc6823ec6eb45bd87e370f3bcc19994",
            "ea179e2f8d9f407eb82994d9c1192e6e",
            "f0b9e6a92ede464d8e91f0c4786db2af",
            "7879630787db424eb0475dbbe1d0e05b"
          ]
        },
        "outputId": "1350bc59-143b-4ab9-97d1-c46381c3035f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading tf_model.h5:   0%|          | 0.00/527M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cca18c68475644aea3dc8f08dfab9f9b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForTokenClassification.\n",
            "\n",
            "Some layers of TFBertForTokenClassification were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Using softmax as the last layer activation for proper classification   \n",
        "(from_logits=True in loss calculation is fine too)\n"
      ],
      "metadata": {
        "id": "OTgnNutPkXkW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.classifier.activation=tf.keras.activations.softmax"
      ],
      "metadata": {
        "id": "LouOZJ1Cj8xu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Pretraining the classifier only"
      ],
      "metadata": {
        "id": "wFvQMVMsknPs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.bert.trainable=False\n",
        "model.classifier.trainable=True"
      ],
      "metadata": {
        "id": "vdZBO6Lht7l5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(loss=loss_function, optimizer=tf.keras.optimizers.Adam(learning_rate=5e-5), metrics=[acc_NER,acc_all])"
      ],
      "metadata": {
        "id": "pAazauwtim5N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(X_train,Y_train,validation_data=(X_val,Y_val),batch_size=64,epochs=2)"
      ],
      "metadata": {
        "id": "n5Q9WBBPis1w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f408747d-631c-4420-c494-82e95809dbb2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/2\n",
            "447/447 [==============================] - 230s 473ms/step - loss: 1.8576 - acc_NER: 0.3942 - acc_all: 0.6979 - val_loss: 1.5242 - val_acc_NER: 0.5655 - val_acc_all: 0.8756\n",
            "Epoch 2/2\n",
            "447/447 [==============================] - 170s 381ms/step - loss: 1.3858 - acc_NER: 0.6288 - acc_all: 0.8735 - val_loss: 1.2097 - val_acc_NER: 0.6815 - val_acc_all: 0.8897\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f25ad4355d0>"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Fine-tuning the whole architecture"
      ],
      "metadata": {
        "id": "4RFhkHy4k6_C"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.bert.trainable=True\n",
        "model.classifier.trainable=True"
      ],
      "metadata": {
        "id": "9cdOJcuii4CH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(loss=loss_function, optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5), metrics=[acc_NER,acc_all])"
      ],
      "metadata": {
        "id": "1uHfPk6rADKx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.fit(X_train,Y_train,validation_data=(X_val,Y_val),batch_size=32,epochs=100)"
      ],
      "metadata": {
        "id": "KT765BhfABgn",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 429
        },
        "outputId": "1036388c-014d-429d-b277-e2e303f00d74"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/100\n",
            "893/893 [==============================] - 513s 515ms/step - loss: 0.5364 - acc_NER: 0.8176 - acc_all: 0.9244 - val_loss: 0.3980 - val_acc_NER: 0.8533 - val_acc_all: 0.9451\n",
            "Epoch 2/100\n",
            "893/893 [==============================] - 449s 503ms/step - loss: 0.3520 - acc_NER: 0.8599 - acc_all: 0.9394 - val_loss: 0.3619 - val_acc_NER: 0.8566 - val_acc_all: 0.9477\n",
            "Epoch 3/100\n",
            "463/893 [==============>...............] - ETA: 2:55 - loss: 0.2881 - acc_NER: 0.8763 - acc_all: 0.9473"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-66-54c825108203>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mY_train\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_val\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mY_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1683\u001b[0m                         ):\n\u001b[1;32m   1684\u001b[0m                             \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1685\u001b[0;31m                             \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1686\u001b[0m                             \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1687\u001b[0m                                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    892\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    893\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 894\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    895\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    896\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    924\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    925\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 926\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_no_variable_creation_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    927\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_variable_creation_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    141\u001b[0m       (concrete_function,\n\u001b[1;32m    142\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m--> 143\u001b[0;31m     return concrete_function._call_flat(\n\u001b[0m\u001b[1;32m    144\u001b[0m         filtered_flat_args, captured_inputs=concrete_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[1;32m    145\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1755\u001b[0m         and executing_eagerly):\n\u001b[1;32m   1756\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1757\u001b[0;31m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m   1758\u001b[0m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m   1759\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    379\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    380\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 381\u001b[0;31m           outputs = execute.execute(\n\u001b[0m\u001b[1;32m    382\u001b[0m               \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m               \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     50\u001b[0m   \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m     53\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[1;32m     54\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<img src=\"https://i.ytimg.com/vi/2CyP3hAOy0g/maxresdefault.jpg\" width=50%>"
      ],
      "metadata": {
        "id": "hjDsWmtqlHU3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def download_from_drive(link, filename):\n",
        "  id = link.split(\"/\")[-2]\n",
        "  !wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id={id}' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\\1\\n/p')&id={id}\" -O {\"/content/\"+filename+\".rar\"} && rm -rf /tmp/cookies.txt\n",
        "  !unrar x -Y {\"/content/\"+filename+\".rar\"} {\"/content/\"+filename+\"/\"}\n",
        "\n",
        "download_from_drive(\"https://drive.google.com/file/d/1yPG7yv2z-bbMY1PS6dVOr2YpgfZg3aBK/view?usp=sharing\",\"checkpoints\")"
      ],
      "metadata": {
        "id": "dWpM6FEclGqf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2ada53d3-e267-4f4e-9c1f-53007983140e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-05-22 20:28:56--  https://docs.google.com/uc?export=download&confirm=t&id=1yPG7yv2z-bbMY1PS6dVOr2YpgfZg3aBK\n",
            "Resolving docs.google.com (docs.google.com)... 172.253.115.101, 172.253.115.113, 172.253.115.138, ...\n",
            "Connecting to docs.google.com (docs.google.com)|172.253.115.101|:443... connected.\n",
            "HTTP request sent, awaiting response... 303 See Other\n",
            "Location: https://doc-0c-1k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/4csg9169ci783aek9d4j314rubb8i7li/1684787325000/02440328774490686871/*/1yPG7yv2z-bbMY1PS6dVOr2YpgfZg3aBK?e=download&uuid=3e41614f-6ba9-4c11-afd6-b6c5c86c57cd [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2023-05-22 20:28:56--  https://doc-0c-1k-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/4csg9169ci783aek9d4j314rubb8i7li/1684787325000/02440328774490686871/*/1yPG7yv2z-bbMY1PS6dVOr2YpgfZg3aBK?e=download&uuid=3e41614f-6ba9-4c11-afd6-b6c5c86c57cd\n",
            "Resolving doc-0c-1k-docs.googleusercontent.com (doc-0c-1k-docs.googleusercontent.com)... 142.251.167.132, 2607:f8b0:4004:c1d::84\n",
            "Connecting to doc-0c-1k-docs.googleusercontent.com (doc-0c-1k-docs.googleusercontent.com)|142.251.167.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 401394496 (383M) [application/rar]\n",
            "Saving to: ‘/content/checkpoints.rar’\n",
            "\n",
            "/content/checkpoint 100%[===================>] 382.80M  85.1MB/s    in 4.9s    \n",
            "\n",
            "2023-05-22 20:29:01 (78.3 MB/s) - ‘/content/checkpoints.rar’ saved [401394496/401394496]\n",
            "\n",
            "\n",
            "UNRAR 5.61 beta 1 freeware      Copyright (c) 1993-2018 Alexander Roshal\n",
            "\n",
            "\n",
            "Extracting from /content/checkpoints.rar\n",
            "\n",
            "Creating    /content/checkpoints                                      OK\n",
            "Creating    /content/checkpoints/NER_lab_trained                      OK\n",
            "Extracting  /content/checkpoints/NER_lab_trained/config.json             \b\b\b\b  0%\b\b\b\b\b  OK \n",
            "Extracting  /content/checkpoints/NER_lab_trained/tf_model.h5             \b\b\b\b  1%\b\b\b\b  2%\b\b\b\b  3%\b\b\b\b  4%\b\b\b\b  5%\b\b\b\b  6%\b\b\b\b  7%\b\b\b\b  8%\b\b\b\b  9%\b\b\b\b 10%\b\b\b\b 11%\b\b\b\b 12%\b\b\b\b 13%\b\b\b\b 14%\b\b\b\b 15%\b\b\b\b 16%\b\b\b\b 17%\b\b\b\b 18%\b\b\b\b 19%\b\b\b\b 20%\b\b\b\b 21%\b\b\b\b 22%\b\b\b\b 24%\b\b\b\b 25%\b\b\b\b 26%\b\b\b\b 27%\b\b\b\b 28%\b\b\b\b 29%\b\b\b\b 30%\b\b\b\b 31%\b\b\b\b 32%\b\b\b\b 33%\b\b\b\b 34%\b\b\b\b 35%\b\b\b\b 36%\b\b\b\b 37%\b\b\b\b 38%\b\b\b\b 39%\b\b\b\b 40%\b\b\b\b 41%\b\b\b\b 42%\b\b\b\b 43%\b\b\b\b 44%\b\b\b\b 45%\b\b\b\b 47%\b\b\b\b 48%\b\b\b\b 49%\b\b\b\b 50%\b\b\b\b 51%\b\b\b\b 52%\b\b\b\b 53%\b\b\b\b 54%\b\b\b\b 55%\b\b\b\b 56%\b\b\b\b 57%\b\b\b\b 58%\b\b\b\b 59%\b\b\b\b 60%\b\b\b\b 61%\b\b\b\b 62%\b\b\b\b 63%\b\b\b\b 64%\b\b\b\b 65%\b\b\b\b 66%\b\b\b\b 67%\b\b\b\b 68%\b\b\b\b 70%\b\b\b\b 71%\b\b\b\b 72%\b\b\b\b 73%\b\b\b\b 74%\b\b\b\b 75%\b\b\b\b 76%\b\b\b\b 77%\b\b\b\b 78%\b\b\b\b 79%\b\b\b\b 80%\b\b\b\b 81%\b\b\b\b 82%\b\b\b\b 83%\b\b\b\b 84%\b\b\b\b 85%\b\b\b\b 86%\b\b\b\b 87%\b\b\b\b 88%\b\b\b\b 89%\b\b\b\b 90%\b\b\b\b 91%\b\b\b\b 92%\b\b\b\b 94%\b\b\b\b 95%\b\b\b\b 96%\b\b\b\b 97%\b\b\b\b 98%\b\b\b\b 99%\b\b\b\b\b  OK \n",
            "All OK\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = TFBertForTokenClassification.from_pretrained(\"./checkpoints/NER_lab_trained\")"
      ],
      "metadata": {
        "id": "OI5l2c2jB2lQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b5cb0e25-162c-4300-98db-ec0ca19b4604"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForTokenClassification.\n",
            "\n",
            "All the layers of TFBertForTokenClassification were initialized from the model checkpoint at ./checkpoints/NER_lab_trained.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertForTokenClassification for predictions without further training.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(loss=loss_function, optimizer=tf.keras.optimizers.Adam(learning_rate=1e-5), metrics=[acc_NER,acc_all])"
      ],
      "metadata": {
        "id": "fya9U1QBq9sl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.evaluate(X_test, Y_test)"
      ],
      "metadata": {
        "id": "HVKJ5fdnokZg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e8b285da-5463-454a-d155-0ce7b2f8e5ca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "298/298 [==============================] - 51s 144ms/step - loss: 1.2995 - acc_NER: 0.8730 - acc_all: 0.9577\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1.2995046377182007, 0.8730359077453613, 0.9576652646064758]"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Making predictions using sub-word tokens"
      ],
      "metadata": {
        "id": "D5z6EgOenpZN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def predict_entities(text):\n",
        "  ids, att, _ = prepare_training_data(text)\n",
        "  #(64,)-->(1, 64)\n",
        "  return model.predict((np.expand_dims(ids,0),np.expand_dims(att,0)))"
      ],
      "metadata": {
        "id": "L0zu8gpLnm3Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predict_entities(\"This is a sentence about New York and Boston.\")"
      ],
      "metadata": {
        "id": "Mk5OAy_PojYd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1a976aed-cbd7-4d80-889f-a0bc7f69e65e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 4s 4s/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TFTokenClassifierOutput(loss=None, logits=array([[[ 7.394515  ,  1.836859  ,  0.2529405 , ..., -2.602069  ,\n",
              "         -2.8288252 , -2.2467585 ],\n",
              "        [10.4971695 ,  0.8446492 , -0.8676269 , ..., -3.836185  ,\n",
              "         -3.261433  , -2.6797745 ],\n",
              "        [11.0004835 ,  0.39116016, -1.0814829 , ..., -3.6836996 ,\n",
              "         -3.1966357 , -2.7357965 ],\n",
              "        ...,\n",
              "        [ 3.3465908 ,  1.0002524 , -0.07820229, ..., -1.1290853 ,\n",
              "         -1.7160007 , -1.1882938 ],\n",
              "        [ 4.7629185 ,  0.6839687 , -0.22230786, ..., -1.76533   ,\n",
              "         -1.9400254 , -1.4089925 ],\n",
              "        [ 5.1745715 ,  0.51804453,  0.13305889, ..., -1.6604767 ,\n",
              "         -1.81544   , -1.1674573 ]]], dtype=float32), hidden_states=None, attentions=None)"
            ]
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def predict_entities(text):\n",
        "  ids, att, _ = prepare_training_data(text)\n",
        "  attmask = np.dstack([att]*model.num_labels)\n",
        "  return model.predict((np.expand_dims(ids,0),np.expand_dims(att,0)))[\"logits\"][attmask==1].reshape(-1,model.num_labels)\n",
        "  "
      ],
      "metadata": {
        "id": "-GOzlTIwt2yX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"This is a sentence about New York a Boston.\"\n",
        "\n",
        "ner_ids = np.argmax(predict_entities(text), axis=-1)\n",
        "ner_ids"
      ],
      "metadata": {
        "id": "br-ZJtnHrxuk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "64be99da-283d-485c-920a-7184344f7790"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 77ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0, 0, 0, 0, 1, 4, 0, 1, 0, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokens = tokenizer.tokenize(text)\n",
        "ner_ids = np.argmax(predict_entities(text), axis=-1)\n",
        "for i, token in enumerate(tokens,1):\n",
        "  nertoken = id_to_ner[ner_ids[i]]\n",
        "  print(token, nertoken)"
      ],
      "metadata": {
        "id": "Z3eEazdOzsRo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "552fa416-4f6d-454e-dc4c-c64dc952c3b8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 83ms/step\n",
            "This O\n",
            "is O\n",
            "a O\n",
            "sentence O\n",
            "about O\n",
            "New B-geo\n",
            "York I-geo\n",
            "a O\n",
            "Boston B-geo\n",
            ". O\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def gather_entities(text):\n",
        "  entities = []\n",
        "  inent = False\n",
        "  ent_type = None\n",
        "  ent_tokens = []\n",
        "\n",
        "  tokens = tokenizer.tokenize(text)\n",
        "  ner_ids = np.argmax(predict_entities(text), axis=-1)\n",
        "\n",
        "  for i, token in enumerate(tokens,1):\n",
        "    nertoken = id_to_ner[ner_ids[i]]\n",
        "\n",
        "    if inent:\n",
        "      # Finalize entity by linking tokens together, taking care of sub-words too\n",
        "      if nertoken==\"O\" or i == len(tokens):\n",
        "        entities.append([\" \".join(ent_tokens).replace(\" ##\",\"\"), ent_type])\n",
        "        inent = False\n",
        "      elif nertoken[0]==\"B\" or (nertoken[0]==\"I\" and nertoken[2:]!=ent_type): \n",
        "        entities.append([\" \".join(ent_tokens).replace(\" ##\",\"\"),ent_type])\n",
        "        ent_tokens = [token]\n",
        "        ent_type = nertoken[2:]\n",
        "      elif nertoken[0]==\"I\" and nertoken[2:]==ent_type:\n",
        "        ent_tokens.append(token)\n",
        "\n",
        "    else:\n",
        "      if nertoken[0] in [\"B\",\"I\"]:\n",
        "        inent = True\n",
        "        ent_tokens = [token]\n",
        "        ent_type = nertoken[2:]\n",
        "\n",
        "  return entities"
      ],
      "metadata": {
        "id": "_MuLfnths6-E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "gather_entities(\"This phone was made by Apple.\")"
      ],
      "metadata": {
        "id": "KLLqHJgLxFI1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f4ce7419-4b12-4649-853a-a594c23f6ec1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 48ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Apple', 'org']]"
            ]
          },
          "metadata": {},
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gather_entities(\"Google Colab is a nice tool.\")"
      ],
      "metadata": {
        "id": "uGb8HCVcz5f3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6ec31b3c-19fe-4c76-b1ed-01e3ee2c047f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 113ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Google Colab', 'org']]"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gather_entities(\"Bill went to the local Police Department to file a case.\")"
      ],
      "metadata": {
        "id": "QoIzMlzY0WgV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "240d8f50-51c9-4a6c-db3f-738139d59d2e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 49ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Bill', 'per'], ['Police Department', 'org']]"
            ]
          },
          "metadata": {},
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gather_entities(\"Michael Jackson is my favorite singer.\")"
      ],
      "metadata": {
        "id": "XlCyOTti0MLK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a654f336-d134-4fa2-8173-ea5a0c47ed26"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 62ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Michael Jackson', 'per']]"
            ]
          },
          "metadata": {},
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "gather_entities(\"Christmas Eve is my favorite time of the year.\")"
      ],
      "metadata": {
        "id": "6kz_ZFyC0T4G",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1cfb7ff6-68f3-4f59-8843-9826aafdac7f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1/1 [==============================] - 0s 49ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['Christmas Eve', 'tim'], ['time', 'tim'], ['of', 'tim']]"
            ]
          },
          "metadata": {},
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "INKbfw6u1vgh"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}