Fine-tuning_RoBERTa_tokenizer.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyMvvvc6E7QkZzcFlg4G4hFJ"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","gpuClass":"standard","widgets":{"application/vnd.jupyter.widget-state+json":{"01e45c4f760f4647b95cc27fc043fb47":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_0efd5116f60e45759252aed4ef4e0f83","IPY_MODEL_9ef8f0e57ec8413cb9bdf3a90d7b69ac","IPY_MODEL_48926d984b9548488eeb06dbbbeb6cde"],"layout":"IPY_MODEL_223001bf2313466c974004ab25af53bf"}},"0efd5116f60e45759252aed4ef4e0f83":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6476b44fe7ff4c1d9a453963356d5089","placeholder":"​","style":"IPY_MODEL_144828376d0e4f1e8812453824a68a41","value":"Downloading: 100%"}},"9ef8f0e57ec8413cb9bdf3a90d7b69ac":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_8d2085169c3d4d7293d4e6b205f8957c","max":898823,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c825f64d011e4df0b39dd239824dfe39","value":898823}},"48926d984b9548488eeb06dbbbeb6cde":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b49a591618594729a2a804f825cc7089","placeholder":"​","style":"IPY_MODEL_bce7ca4446904450a29898ed9c9c9070","value":" 899k/899k [00:01&lt;00:00, 937kB/s]"}},"223001bf2313466c974004ab25af53bf":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6476b44fe7ff4c1d9a453963356d5089":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"144828376d0e4f1e8812453824a68a41":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8d2085169c3d4d7293d4e6b205f8957c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c825f64d011e4df0b39dd239824dfe39":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"b49a591618594729a2a804f825cc7089":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bce7ca4446904450a29898ed9c9c9070":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"e625f6e41c084cd38c0bed43d534d554":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7e0c581a98f6404d83404e47f931cc84","IPY_MODEL_45d8d1af927d4f52af010ba0cef08569","IPY_MODEL_b0c7271d75e94d088fa336b5aa9d0f0b"],"layout":"IPY_MODEL_44958a7b85354a4f861aeedf24cdd882"}},"7e0c581a98f6404d83404e47f931cc84":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c86d66aa9fe04c15ab85d7bb9ae84a6e","placeholder":"​","style":"IPY_MODEL_bb5577485f8b4612a5ea25ab499782d8","value":"Downloading: 100%"}},"45d8d1af927d4f52af010ba0cef08569":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_32332047d298465aaf867cf044c1a6a1","max":456318,"min":0,"orientation":"horizontal","style":"IPY_MODEL_81f96cbb64194ee091fe4ce227f431dd","value":456318}},"b0c7271d75e94d088fa336b5aa9d0f0b":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6cf08fc65f6e4c98bbcb4cd86094c72a","placeholder":"​","style":"IPY_MODEL_9c05b07059474fa8bf126fe924ecb643","value":" 456k/456k [00:01&lt;00:00, 484kB/s]"}},"44958a7b85354a4f861aeedf24cdd882":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c86d66aa9fe04c15ab85d7bb9ae84a6e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bb5577485f8b4612a5ea25ab499782d8":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"32332047d298465aaf867cf044c1a6a1":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"81f96cbb64194ee091fe4ce227f431dd":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"6cf08fc65f6e4c98bbcb4cd86094c72a":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9c05b07059474fa8bf126fe924ecb643":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2bd5f95f6306424798877317c750f8b9":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_0bc46dd19234490ab7d3bf0ed8c33ad2","IPY_MODEL_3fb3b96101654ef988b509c059430cd5","IPY_MODEL_a5789d274a284497a8feea7b670751ec"],"layout":"IPY_MODEL_bf6895831ee5481cadadea1674582dbb"}},"0bc46dd19234490ab7d3bf0ed8c33ad2":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_3fa2e83872594062a3d53aaf61d07165","placeholder":"​","style":"IPY_MODEL_d116e792247541d28c22e97af0972309","value":"Downloading: 100%"}},"3fb3b96101654ef988b509c059430cd5":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_a7156fdc27f042aab5674b9d21d133c9","max":481,"min":0,"orientation":"horizontal","style":"IPY_MODEL_3f2c4bf2487741f691fe7587b280a02e","value":481}},"a5789d274a284497a8feea7b670751ec":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_1aeffcdafe024821a6092423e5f4d466","placeholder":"​","style":"IPY_MODEL_0db8a7ce76c543bdb05bb82e3cf8cf4b","value":" 481/481 [00:00&lt;00:00, 6.88kB/s]"}},"bf6895831ee5481cadadea1674582dbb":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3fa2e83872594062a3d53aaf61d07165":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d116e792247541d28c22e97af0972309":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"a7156fdc27f042aab5674b9d21d133c9":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"3f2c4bf2487741f691fe7587b280a02e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"1aeffcdafe024821a6092423e5f4d466":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0db8a7ce76c543bdb05bb82e3cf8cf4b":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"94a2a7c840854eb79f44975b2dc7dfd3":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_20346b1b5ecf4550b89e10825695bb59","IPY_MODEL_55c752f0b2034402badab573dd190aa3","IPY_MODEL_ac16daa298f643ca9fe10993a2298218"],"layout":"IPY_MODEL_98a632051cf24b51ab7ba22aa154801f"}},"20346b1b5ecf4550b89e10825695bb59":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_470d65ac5bce49a38dc38e99600b3102","placeholder":"​","style":"IPY_MODEL_adc1a54046c745598c7d665e8cf570a1","value":"Downloading: 100%"}},"55c752f0b2034402badab573dd190aa3":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_d014a7425c27417ea1883e1ffba273a4","max":501200538,"min":0,"orientation":"horizontal","style":"IPY_MODEL_9a2f3d1f4f03497ab36890ff51ff212e","value":501200538}},"ac16daa298f643ca9fe10993a2298218":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_af65ce962cbc493abd8726c483892218","placeholder":"​","style":"IPY_MODEL_a048624a3b4445d19bb9e28926184bae","value":" 501M/501M [00:08&lt;00:00, 59.9MB/s]"}},"98a632051cf24b51ab7ba22aa154801f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"470d65ac5bce49a38dc38e99600b3102":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"adc1a54046c745598c7d665e8cf570a1":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d014a7425c27417ea1883e1ffba273a4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9a2f3d1f4f03497ab36890ff51ff212e":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"af65ce962cbc493abd8726c483892218":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a048624a3b4445d19bb9e28926184bae":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"markdown","source":["## 1 Importing the Pre-Processed Dataset\n","\n","As the dataset has been separated to 4 parts, we need to reread them from files:\n","\n","- X_train (training variables of the dataset)\n","- X_val (validation variables of the dataset)\n","- y_train (training labels of the dataset)\n","- y_val (validation labels of the dataset)"],"metadata":{"id":"gPJUvMopwS6U"}},{"cell_type":"code","source":["import pandas as pd"],"metadata":{"id":"lDJgCJwYwSYK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["X_train = pd.read_csv('X_train.csv')\n","X_val = pd.read_csv('X_val.csv')\n","y_train = pd.read_csv('y_train.csv')\n","y_val = pd.read_csv('y_val.csv')"],"metadata":{"id":"fyWvX_PVwXe7"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["Because we are only attempting to classify the ```True``` from the ```False``` by the text. Then we should select the variable \"text\" from X_train and X_val, and select the variable \"target\" from y_train and y_val."],"metadata":{"id":"m2EU3C9XDe9u"}},{"cell_type":"code","source":["train_text = X_train['cleaned_text'].to_list()\n","train_label = y_train['target'].to_list()\n","val_text = X_val['cleaned_text'].to_list()\n","val_label = y_val['target'].to_list()"],"metadata":{"id":"IjMN9ZcYwYQ5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["train_text[:5]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vbPqjNZ5Dg0e","executionInfo":{"status":"ok","timestamp":1669468138622,"user_tz":-480,"elapsed":480,"user":{"displayName":"Trenton Wong","userId":"06174541562312717890"}},"outputId":"f7539048-410e-483a-b9b0-142eff6ebc75"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['  jimmyfallon crush squirrel bone mortar pestl school  bio dept  realli sure whi worstsummerjob',\n"," ' mccainenl think spectacular look stonewal riot obliter white house ',\n"," 'can t bloodi wait   soni set date stephen king       the dark tower    stephenk thedarktow    bdisgust',\n"," 'protest ralli stone mountain  atleast they r burn build loot store like individu  protest ',\n"," ' rbcinsur quot websit   disaster  tri 3 browser  amp  3 machines  alway get  miss info  error due non exist drop down ']"]},"metadata":{},"execution_count":4}]},{"cell_type":"code","source":["train_label[:5]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vpCQagIsDjeu","executionInfo":{"status":"ok","timestamp":1669468147612,"user_tz":-480,"elapsed":2,"user":{"displayName":"Trenton Wong","userId":"06174541562312717890"}},"outputId":"e5132c7d-a201-4b0f-d4fe-1e24b209f010"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[0, 1, 0, 0, 0]"]},"metadata":{},"execution_count":5}]},{"cell_type":"markdown","source":["## 2 Tokenize the text by the pre-trained model RoBERTa\n","\n","In this notebook, we choose a pre-trained model named RoBERTa-base to tokenize the text from the dataset.\n","\n","Then, we should download the pre-trained model first:"],"metadata":{"id":"7FQblnxuDnyE"}},{"cell_type":"code","source":["!pip install transformers"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"hZcwBrl6Ew-Z","executionInfo":{"status":"ok","timestamp":1669468480325,"user_tz":-480,"elapsed":9994,"user":{"displayName":"Trenton Wong","userId":"06174541562312717890"}},"outputId":"f48e6885-997f-4c34-c6c6-47b3249d6697"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting transformers\n","  Downloading transformers-4.24.0-py3-none-any.whl (5.5 MB)\n","\u001b[K     |████████████████████████████████| 5.5 MB 33.2 MB/s \n","\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n","Collecting huggingface-hub<1.0,>=0.10.0\n","  Downloading huggingface_hub-0.11.0-py3-none-any.whl (182 kB)\n","\u001b[K     |████████████████████████████████| 182 kB 70.9 MB/s \n","\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n","Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n","  Downloading tokenizers-0.13.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n","\u001b[K     |████████████████████████████████| 7.6 MB 55.6 MB/s \n","\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.10.0)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n","Installing collected packages: tokenizers, huggingface-hub, transformers\n","Successfully installed huggingface-hub-0.11.0 tokenizers-0.13.2 transformers-4.24.0\n"]}]},{"cell_type":"code","source":["from transformers import RobertaTokenizer\n","tokenizer = RobertaTokenizer.from_pretrained('roberta-base')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":113,"referenced_widgets":["01e45c4f760f4647b95cc27fc043fb47","0efd5116f60e45759252aed4ef4e0f83","9ef8f0e57ec8413cb9bdf3a90d7b69ac","48926d984b9548488eeb06dbbbeb6cde","223001bf2313466c974004ab25af53bf","6476b44fe7ff4c1d9a453963356d5089","144828376d0e4f1e8812453824a68a41","8d2085169c3d4d7293d4e6b205f8957c","c825f64d011e4df0b39dd239824dfe39","b49a591618594729a2a804f825cc7089","bce7ca4446904450a29898ed9c9c9070","e625f6e41c084cd38c0bed43d534d554","7e0c581a98f6404d83404e47f931cc84","45d8d1af927d4f52af010ba0cef08569","b0c7271d75e94d088fa336b5aa9d0f0b","44958a7b85354a4f861aeedf24cdd882","c86d66aa9fe04c15ab85d7bb9ae84a6e","bb5577485f8b4612a5ea25ab499782d8","32332047d298465aaf867cf044c1a6a1","81f96cbb64194ee091fe4ce227f431dd","6cf08fc65f6e4c98bbcb4cd86094c72a","9c05b07059474fa8bf126fe924ecb643","2bd5f95f6306424798877317c750f8b9","0bc46dd19234490ab7d3bf0ed8c33ad2","3fb3b96101654ef988b509c059430cd5","a5789d274a284497a8feea7b670751ec","bf6895831ee5481cadadea1674582dbb","3fa2e83872594062a3d53aaf61d07165","d116e792247541d28c22e97af0972309","a7156fdc27f042aab5674b9d21d133c9","3f2c4bf2487741f691fe7587b280a02e","1aeffcdafe024821a6092423e5f4d466","0db8a7ce76c543bdb05bb82e3cf8cf4b"]},"id":"tYX0kbWwDkbu","executionInfo":{"status":"ok","timestamp":1669468497291,"user_tz":-480,"elapsed":13604,"user":{"displayName":"Trenton Wong","userId":"06174541562312717890"}},"outputId":"db6de9f0-6737-4dde-ae6a-5a0b1d9c56ce"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/899k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"01e45c4f760f4647b95cc27fc043fb47"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/456k [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e625f6e41c084cd38c0bed43d534d554"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/481 [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2bd5f95f6306424798877317c750f8b9"}},"metadata":{}}]},{"cell_type":"markdown","source":["Apply the pre-trained model to the text by the tokenizer."],"metadata":{"id":"7yeTShTZE_Qo"}},{"cell_type":"code","source":["train_encodings = tokenizer(train_text, truncation=True, padding=True)\n","val_encodings = tokenizer(val_text, truncation=True, padding=True)"],"metadata":{"id":"Lvnft0D1DyhE"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## 3 Defining the Dataset Class\n","\n","Before we input the dataset into Neural Network, we should define a Class then instantiate it to store the encodings and labels of the data."],"metadata":{"id":"P6uBIqmeFFEX"}},{"cell_type":"code","source":["import torch\n","\n","class DatasetClass(torch.utils.data.Dataset):\n","    def __init__(self, encodings, labels):\n","        self.encodings = encodings\n","        self.labels = labels\n","\n","    def __getitem__(self, idx):\n","        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n","        item['labels'] = torch.tensor(self.labels[idx])\n","        return item\n","\n","    def __len__(self):\n","        return len(self.labels)\n","\n","train_dataset = DatasetClass(train_encodings, train_label)\n","val_dataset = DatasetClass(val_encodings, val_label)"],"metadata":{"id":"FT-BXRI9FEuf"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["train_dataset.__getitem__(1)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"y5Q0LR0mFTGl","executionInfo":{"status":"ok","timestamp":1669468609798,"user_tz":-480,"elapsed":1280,"user":{"displayName":"Trenton Wong","userId":"06174541562312717890"}},"outputId":"b42cb62c-a302-4422-a1aa-00ecaab60bfd"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'input_ids': tensor([    0,   475,  7309,  1851,   225,   462,   206,  8694,   356,  1690,\n","         32991,   337, 13069, 35145,  1104,   790,  1437,     2,     1,     1,\n","             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n","             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n","             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n","             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n","             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n","             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n","             1,     1,     1,     1,     1,     1,     1,     1,     1,     1,\n","             1]),\n"," 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,\n","         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n","         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n","         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n"," 'labels': tensor(1)}"]},"metadata":{},"execution_count":15}]},{"cell_type":"markdown","source":["## 4 Defining Evaluation Matrics"],"metadata":{"id":"RUpEerWJFYcm"}},{"cell_type":"code","source":["from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n","\n","def evaluation_metrics(pred):\n","    labels = pred.label_ids\n","    preds = pred.predictions.argmax(-1)\n","    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='binary')\n","    acc = accuracy_score(labels, preds)\n","    return {\n","        'accuracy': acc,\n","        'f1': f1,\n","        'precision': precision,\n","        'recall': recall\n","    }"],"metadata":{"id":"SxCSfNXCFYRO"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## 5 Fine-tuning with Huggingface Trainer\n","\n","Huggingface Trainer is a highly packaged trainer, we need to define the following arguments before we train:\n","\n","- **args** -> TrainingArguments (Contains the definition of hyperparameters, which is also an important feature of the trainer, where most of the training-related parameters are set)\n","- **model** -> Model (is a model that integrates *transformers.PreTrainedMode* or *torch.nn.module*, which is officially mentioned as being optimised by Trainer for transformers.PreTrainedModel)\n","- **compute_metrics** -> Evaluation Metrics (to define how to evaluate the results of the fine-tuned model)\n","- **train_dataset** -> Train Dataset\n","- **eval_dataset** -> Validation Dataset"],"metadata":{"id":"PQ0ABkeWFd8H"}},{"cell_type":"code","source":["from torch import cuda\n","device = 'cuda' if cuda.is_available() else 'cpu'\n","device"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"ttpZNmq3FVUt","executionInfo":{"status":"ok","timestamp":1669468653959,"user_tz":-480,"elapsed":492,"user":{"displayName":"Trenton Wong","userId":"06174541562312717890"}},"outputId":"2634124f-b5ab-4b43-f9ac-519cf7572d14"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'cuda'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":17}]},{"cell_type":"code","source":["from transformers import RobertaForSequenceClassification, Trainer, TrainingArguments\n","import os\n","os.environ[\"WANDB_DISABLED\"] = \"true\"\n","\n","training_args = TrainingArguments(\n","    output_dir='./RoBERTa-results',# output directory\n","    num_train_epochs=3,              # total number of training epochs\n","    per_device_train_batch_size=16,  # batch size per device during training\n","    per_device_eval_batch_size=64,   # batch size for evaluation\n","    warmup_steps=500,                # number of warmup steps for learning rate scheduler\n","    weight_decay=0.01,               # strength of weight decay\n","    logging_dir='./logs',            # directory for storing logs\n","    logging_steps=10,\n",")\n","\n","model = RobertaForSequenceClassification.from_pretrained(\"roberta-base\")\n","\n","trainer = Trainer(\n","    model=model,                         # the instantiated pre-trained model to be trained\n","    args=training_args,                  # training arguments, defined above\n","    compute_metrics=evaluation_metrics,\n","    train_dataset=train_dataset,         # training dataset\n","    eval_dataset=val_dataset,             # validation dataset\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":179,"referenced_widgets":["94a2a7c840854eb79f44975b2dc7dfd3","20346b1b5ecf4550b89e10825695bb59","55c752f0b2034402badab573dd190aa3","ac16daa298f643ca9fe10993a2298218","98a632051cf24b51ab7ba22aa154801f","470d65ac5bce49a38dc38e99600b3102","adc1a54046c745598c7d665e8cf570a1","d014a7425c27417ea1883e1ffba273a4","9a2f3d1f4f03497ab36890ff51ff212e","af65ce962cbc493abd8726c483892218","a048624a3b4445d19bb9e28926184bae"]},"id":"c5YphTjDFgZC","executionInfo":{"status":"ok","timestamp":1669468775757,"user_tz":-480,"elapsed":16610,"user":{"displayName":"Trenton Wong","userId":"06174541562312717890"}},"outputId":"1e3bf0bf-d5c9-40e0-d3b7-26ace835ae6e"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"]},{"output_type":"display_data","data":{"text/plain":["Downloading:   0%|          | 0.00/501M [00:00<?, ?B/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"94a2a7c840854eb79f44975b2dc7dfd3"}},"metadata":{}},{"output_type":"stream","name":"stderr","text":["Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.bias', 'roberta.pooler.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'roberta.pooler.dense.weight', 'lm_head.dense.weight']\n","- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]}]},{"cell_type":"code","source":["trainer.train()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"45zWFXIPF5m4","executionInfo":{"status":"ok","timestamp":1669469113238,"user_tz":-480,"elapsed":333982,"user":{"displayName":"Trenton Wong","userId":"06174541562312717890"}},"outputId":"90c0ea0b-6075-4c31-ba39-9f0f4dee2700"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n","  FutureWarning,\n","***** Running training *****\n","  Num examples = 6090\n","  Num Epochs = 3\n","  Instantaneous batch size per device = 16\n","  Total train batch size (w. parallel, distributed & accumulation) = 16\n","  Gradient Accumulation steps = 1\n","  Total optimization steps = 1143\n","  Number of trainable parameters = 124647170\n"]},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","    <div>\n","      \n","      <progress value='1143' max='1143' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [1143/1143 05:29, Epoch 3/3]\n","    </div>\n","    <table border=\"1\" class=\"dataframe\">\n","  <thead>\n"," <tr style=\"text-align: left;\">\n","      <th>Step</th>\n","      <th>Training Loss</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <td>10</td>\n","      <td>0.736800</td>\n","    </tr>\n","    <tr>\n","      <td>20</td>\n","      <td>0.721000</td>\n","    </tr>\n","    <tr>\n","      <td>30</td>\n","      <td>0.717000</td>\n","    </tr>\n","    <tr>\n","      <td>40</td>\n","      <td>0.705200</td>\n","    </tr>\n","    <tr>\n","      <td>50</td>\n","      <td>0.692200</td>\n","    </tr>\n","    <tr>\n","      <td>60</td>\n","      <td>0.696400</td>\n","    </tr>\n","    <tr>\n","      <td>70</td>\n","      <td>0.677700</td>\n","    </tr>\n","    <tr>\n","      <td>80</td>\n","      <td>0.648600</td>\n","    </tr>\n","    <tr>\n","      <td>90</td>\n","      <td>0.630200</td>\n","    </tr>\n","    <tr>\n","      <td>100</td>\n","      <td>0.491700</td>\n","    </tr>\n","    <tr>\n","      <td>110</td>\n","      <td>0.605200</td>\n","    </tr>\n","    <tr>\n","      <td>120</td>\n","      <td>0.458900</td>\n","    </tr>\n","    <tr>\n","      <td>130</td>\n","      <td>0.570400</td>\n","    </tr>\n","    <tr>\n","      <td>140</td>\n","      <td>0.486600</td>\n","    </tr>\n","    <tr>\n","      <td>150</td>\n","      <td>0.532900</td>\n","    </tr>\n","    <tr>\n","      <td>160</td>\n","      <td>0.488600</td>\n","    </tr>\n","    <tr>\n","      <td>170</td>\n","      <td>0.583100</td>\n","    </tr>\n","    <tr>\n","      <td>180</td>\n","      <td>0.535900</td>\n","    </tr>\n","    <tr>\n","      <td>190</td>\n","      <td>0.555800</td>\n","    </tr>\n","    <tr>\n","      <td>200</td>\n","      <td>0.583200</td>\n","    </tr>\n","    <tr>\n","      <td>210</td>\n","      <td>0.544500</td>\n","    </tr>\n","    <tr>\n","      <td>220</td>\n","      <td>0.476700</td>\n","    </tr>\n","    <tr>\n","      <td>230</td>\n","      <td>0.366700</td>\n","    </tr>\n","    <tr>\n","      <td>240</td>\n","      <td>0.494600</td>\n","    </tr>\n","    <tr>\n","      <td>250</td>\n","      <td>0.535300</td>\n","    </tr>\n","    <tr>\n","      <td>260</td>\n","      <td>0.518000</td>\n","    </tr>\n","    <tr>\n","      <td>270</td>\n","      <td>0.474500</td>\n","    </tr>\n","    <tr>\n","      <td>280</td>\n","      <td>0.420700</td>\n","    </tr>\n","    <tr>\n","      <td>290</td>\n","      <td>0.488700</td>\n","    </tr>\n","    <tr>\n","      <td>300</td>\n","      <td>0.364700</td>\n","    </tr>\n","    <tr>\n","      <td>310</td>\n","      <td>0.630900</td>\n","    </tr>\n","    <tr>\n","      <td>320</td>\n","      <td>0.545300</td>\n","    </tr>\n","    <tr>\n","      <td>330</td>\n","      <td>0.417000</td>\n","    </tr>\n","    <tr>\n","      <td>340</td>\n","      <td>0.526400</td>\n","    </tr>\n","    <tr>\n","      <td>350</td>\n","      <td>0.586000</td>\n","    </tr>\n","    <tr>\n","      <td>360</td>\n","      <td>0.423900</td>\n","    </tr>\n","    <tr>\n","      <td>370</td>\n","      <td>0.460000</td>\n","    </tr>\n","    <tr>\n","      <td>380</td>\n","      <td>0.468800</td>\n","    </tr>\n","    <tr>\n","      <td>390</td>\n","      <td>0.647800</td>\n","    </tr>\n","    <tr>\n","      <td>400</td>\n","      <td>0.458600</td>\n","    </tr>\n","    <tr>\n","      <td>410</td>\n","      <td>0.430600</td>\n","    </tr>\n","    <tr>\n","      <td>420</td>\n","      <td>0.504800</td>\n","    </tr>\n","    <tr>\n","      <td>430</td>\n","      <td>0.448200</td>\n","    </tr>\n","    <tr>\n","      <td>440</td>\n","      <td>0.360400</td>\n","    </tr>\n","    <tr>\n","      <td>450</td>\n","      <td>0.467100</td>\n","    </tr>\n","    <tr>\n","      <td>460</td>\n","      <td>0.486100</td>\n","    </tr>\n","    <tr>\n","      <td>470</td>\n","      <td>0.466400</td>\n","    </tr>\n","    <tr>\n","      <td>480</td>\n","      <td>0.483500</td>\n","    </tr>\n","    <tr>\n","      <td>490</td>\n","      <td>0.534900</td>\n","    </tr>\n","    <tr>\n","      <td>500</td>\n","      <td>0.521000</td>\n","    </tr>\n","    <tr>\n","      <td>510</td>\n","      <td>0.544000</td>\n","    </tr>\n","    <tr>\n","      <td>520</td>\n","      <td>0.548800</td>\n","    </tr>\n","    <tr>\n","      <td>530</td>\n","      <td>0.419600</td>\n","    </tr>\n","    <tr>\n","      <td>540</td>\n","      <td>0.428900</td>\n","    </tr>\n","    <tr>\n","      <td>550</td>\n","      <td>0.475200</td>\n","    </tr>\n","    <tr>\n","      <td>560</td>\n","      <td>0.403600</td>\n","    </tr>\n","    <tr>\n","      <td>570</td>\n","      <td>0.475900</td>\n","    </tr>\n","    <tr>\n","      <td>580</td>\n","      <td>0.516500</td>\n","    </tr>\n","    <tr>\n","      <td>590</td>\n","      <td>0.623000</td>\n","    </tr>\n","    <tr>\n","      <td>600</td>\n","      <td>0.631100</td>\n","    </tr>\n","    <tr>\n","      <td>610</td>\n","      <td>0.389800</td>\n","    </tr>\n","    <tr>\n","      <td>620</td>\n","      <td>0.565600</td>\n","    </tr>\n","    <tr>\n","      <td>630</td>\n","      <td>0.476900</td>\n","    </tr>\n","    <tr>\n","      <td>640</td>\n","      <td>0.451100</td>\n","    </tr>\n","    <tr>\n","      <td>650</td>\n","      <td>0.478700</td>\n","    </tr>\n","    <tr>\n","      <td>660</td>\n","      <td>0.509700</td>\n","    </tr>\n","    <tr>\n","      <td>670</td>\n","      <td>0.467900</td>\n","    </tr>\n","    <tr>\n","      <td>680</td>\n","      <td>0.415900</td>\n","    </tr>\n","    <tr>\n","      <td>690</td>\n","      <td>0.391800</td>\n","    </tr>\n","    <tr>\n","      <td>700</td>\n","      <td>0.419300</td>\n","    </tr>\n","    <tr>\n","      <td>710</td>\n","      <td>0.502500</td>\n","    </tr>\n","    <tr>\n","      <td>720</td>\n","      <td>0.493600</td>\n","    </tr>\n","    <tr>\n","      <td>730</td>\n","      <td>0.530900</td>\n","    </tr>\n","    <tr>\n","      <td>740</td>\n","      <td>0.426200</td>\n","    </tr>\n","    <tr>\n","      <td>750</td>\n","      <td>0.368500</td>\n","    </tr>\n","    <tr>\n","      <td>760</td>\n","      <td>0.448200</td>\n","    </tr>\n","    <tr>\n","      <td>770</td>\n","      <td>0.442000</td>\n","    </tr>\n","    <tr>\n","      <td>780</td>\n","      <td>0.421700</td>\n","    </tr>\n","    <tr>\n","      <td>790</td>\n","      <td>0.394500</td>\n","    </tr>\n","    <tr>\n","      <td>800</td>\n","      <td>0.307500</td>\n","    </tr>\n","    <tr>\n","      <td>810</td>\n","      <td>0.477600</td>\n","    </tr>\n","    <tr>\n","      <td>820</td>\n","      <td>0.422200</td>\n","    </tr>\n","    <tr>\n","      <td>830</td>\n","      <td>0.376500</td>\n","    </tr>\n","    <tr>\n","      <td>840</td>\n","      <td>0.386900</td>\n","    </tr>\n","    <tr>\n","      <td>850</td>\n","      <td>0.319900</td>\n","    </tr>\n","    <tr>\n","      <td>860</td>\n","      <td>0.355200</td>\n","    </tr>\n","    <tr>\n","      <td>870</td>\n","      <td>0.388300</td>\n","    </tr>\n","    <tr>\n","      <td>880</td>\n","      <td>0.463600</td>\n","    </tr>\n","    <tr>\n","      <td>890</td>\n","      <td>0.378700</td>\n","    </tr>\n","    <tr>\n","      <td>900</td>\n","      <td>0.553300</td>\n","    </tr>\n","    <tr>\n","      <td>910</td>\n","      <td>0.414000</td>\n","    </tr>\n","    <tr>\n","      <td>920</td>\n","      <td>0.386900</td>\n","    </tr>\n","    <tr>\n","      <td>930</td>\n","      <td>0.353600</td>\n","    </tr>\n","    <tr>\n","      <td>940</td>\n","      <td>0.364600</td>\n","    </tr>\n","    <tr>\n","      <td>950</td>\n","      <td>0.323100</td>\n","    </tr>\n","    <tr>\n","      <td>960</td>\n","      <td>0.385300</td>\n","    </tr>\n","    <tr>\n","      <td>970</td>\n","      <td>0.371200</td>\n","    </tr>\n","    <tr>\n","      <td>980</td>\n","      <td>0.401200</td>\n","    </tr>\n","    <tr>\n","      <td>990</td>\n","      <td>0.390700</td>\n","    </tr>\n","    <tr>\n","      <td>1000</td>\n","      <td>0.473300</td>\n","    </tr>\n","    <tr>\n","      <td>1010</td>\n","      <td>0.405200</td>\n","    </tr>\n","    <tr>\n","      <td>1020</td>\n","      <td>0.374700</td>\n","    </tr>\n","    <tr>\n","      <td>1030</td>\n","      <td>0.357500</td>\n","    </tr>\n","    <tr>\n","      <td>1040</td>\n","      <td>0.354500</td>\n","    </tr>\n","    <tr>\n","      <td>1050</td>\n","      <td>0.340300</td>\n","    </tr>\n","    <tr>\n","      <td>1060</td>\n","      <td>0.353900</td>\n","    </tr>\n","    <tr>\n","      <td>1070</td>\n","      <td>0.331200</td>\n","    </tr>\n","    <tr>\n","      <td>1080</td>\n","      <td>0.354900</td>\n","    </tr>\n","    <tr>\n","      <td>1090</td>\n","      <td>0.309500</td>\n","    </tr>\n","    <tr>\n","      <td>1100</td>\n","      <td>0.378600</td>\n","    </tr>\n","    <tr>\n","      <td>1110</td>\n","      <td>0.276400</td>\n","    </tr>\n","    <tr>\n","      <td>1120</td>\n","      <td>0.265300</td>\n","    </tr>\n","    <tr>\n","      <td>1130</td>\n","      <td>0.428200</td>\n","    </tr>\n","    <tr>\n","      <td>1140</td>\n","      <td>0.237100</td>\n","    </tr>\n","  </tbody>\n","</table><p>"]},"metadata":{}},{"output_type":"stream","name":"stderr","text":["Saving model checkpoint to ./RoBERTa-results/checkpoint-500\n","Configuration saved in ./RoBERTa-results/checkpoint-500/config.json\n","Model weights saved in ./RoBERTa-results/checkpoint-500/pytorch_model.bin\n","Saving model checkpoint to ./RoBERTa-results/checkpoint-1000\n","Configuration saved in ./RoBERTa-results/checkpoint-1000/config.json\n","Model weights saved in ./RoBERTa-results/checkpoint-1000/pytorch_model.bin\n","\n","\n","Training completed. Do not forget to share your model on huggingface.co/models =)\n","\n","\n"]},{"output_type":"execute_result","data":{"text/plain":["TrainOutput(global_step=1143, training_loss=0.4681619150223486, metrics={'train_runtime': 332.8016, 'train_samples_per_second': 54.898, 'train_steps_per_second': 3.434, 'total_flos': 854376068964600.0, 'train_loss': 0.4681619150223486, 'epoch': 3.0})"]},"metadata":{},"execution_count":19}]},{"cell_type":"markdown","source":["## 6 Results of Evaluating the Validation Dataset"],"metadata":{"id":"HUeAOU5dGLUm"}},{"cell_type":"code","source":["trainer.evaluate()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":258},"id":"MpWOToqLF-y3","executionInfo":{"status":"ok","timestamp":1669469117720,"user_tz":-480,"elapsed":4503,"user":{"displayName":"Trenton Wong","userId":"06174541562312717890"}},"outputId":"4402ddb1-a831-4914-d214-37c72e9cb082"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["***** Running Evaluation *****\n","  Num examples = 1523\n","  Batch size = 64\n"]},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["\n","    <div>\n","      \n","      <progress value='24' max='24' style='width:300px; height:20px; vertical-align: middle;'></progress>\n","      [24/24 00:04]\n","    </div>\n","    "]},"metadata":{}},{"output_type":"execute_result","data":{"text/plain":["{'eval_loss': 0.48548606038093567,\n"," 'eval_accuracy': 0.8154957321076822,\n"," 'eval_f1': 0.7709861450692747,\n"," 'eval_precision': 0.8044217687074829,\n"," 'eval_recall': 0.7402190923317684,\n"," 'eval_runtime': 4.9712,\n"," 'eval_samples_per_second': 306.363,\n"," 'eval_steps_per_second': 4.828,\n"," 'epoch': 3.0}"]},"metadata":{},"execution_count":20}]},{"cell_type":"markdown","source":["Some code references from:\n","1. https://huggingface.co/transformers/v3.2.0/custom_datasets.html\n","2. https://huggingface.co/docs/transformers/v4.24.0/en/model_doc/roberta"],"metadata":{"id":"chRI7VN-GQDG"}},{"cell_type":"code","source":[],"metadata":{"id":"4cTmRtzrGRBu"},"execution_count":null,"outputs":[]}]}