project04.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "project04.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Re79v0pTS7a"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RPkK_IL6QKuW",
        "outputId": "83431cb0-707e-4d5a-a41d-9a8f2e3ffb28"
      },
      "source": [
        "!pip install python-crfsuite\n",
        "from itertools import chain\n",
        "import nltk\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "import sklearn\n",
        "import pycrfsuite\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "plt.style.use('ggplot')\n",
        "from itertools import chain\n",
        "\n",
        "!pip install sklearn_crfsuite\n",
        "\n",
        "import nltk\n",
        "import sklearn\n",
        "import scipy.stats\n",
        "from sklearn.metrics import make_scorer\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "\n",
        "import sklearn_crfsuite\n",
        "from sklearn_crfsuite import scorers\n",
        "from sklearn_crfsuite import metrics\n",
        "from sklearn.model_selection import train_test_split\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import copy\n",
        "\n",
        "\n",
        "ne_path =\"/content/drive/MyDrive/Colab Notebooks/project4_data/NE.txt\"\n",
        "ma_path = \"/content/drive/MyDrive/Colab Notebooks/project4_data/NE.ma.txt\"\n",
        "path_loc =  \"/content/drive/MyDrive/Colab Notebooks/project4_data/lexicon_location.txt\"\n",
        "path_per =  \"/content/drive/MyDrive/Colab Notebooks/project4_data/lexicon_person.txt\"\n",
        "path_org =  \"/content/drive/MyDrive/Colab Notebooks/project4_data/lexicon_organization.txt\"\n",
        "\n",
        "ne_deneme_path = \"/content/drive/MyDrive/Colab Notebooks/project4_data/NE.prep.txt\" "
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting python-crfsuite\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/95/99/869dde6dbf3e0d07a013c8eebfb0a3d30776334e0097f8432b631a9a3a19/python_crfsuite-0.9.7-cp36-cp36m-manylinux1_x86_64.whl (743kB)\n",
            "\r\u001b[K     |▍                               | 10kB 13.9MB/s eta 0:00:01\r\u001b[K     |▉                               | 20kB 9.9MB/s eta 0:00:01\r\u001b[K     |█▎                              | 30kB 7.7MB/s eta 0:00:01\r\u001b[K     |█▊                              | 40kB 6.7MB/s eta 0:00:01\r\u001b[K     |██▏                             | 51kB 4.3MB/s eta 0:00:01\r\u001b[K     |██▋                             | 61kB 4.7MB/s eta 0:00:01\r\u001b[K     |███                             | 71kB 4.9MB/s eta 0:00:01\r\u001b[K     |███▌                            | 81kB 5.0MB/s eta 0:00:01\r\u001b[K     |████                            | 92kB 5.2MB/s eta 0:00:01\r\u001b[K     |████▍                           | 102kB 4.2MB/s eta 0:00:01\r\u001b[K     |████▉                           | 112kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 122kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 133kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████▏                         | 143kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 153kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████                         | 163kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████▌                        | 174kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████                        | 184kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 194kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 204kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 215kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 225kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 235kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 245kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████                     | 256kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 266kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████                    | 276kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 286kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 296kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 307kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 317kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 327kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 337kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 348kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 358kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 368kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 378kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 389kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 399kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 409kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 419kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 430kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 440kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 450kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 460kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 471kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 481kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 491kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 501kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 512kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 522kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 532kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 542kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 552kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 563kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 573kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 583kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 593kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 604kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 614kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 624kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 634kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 645kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 655kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 665kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 675kB 4.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 686kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 696kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 706kB 4.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 716kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 727kB 4.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 737kB 4.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 747kB 4.2MB/s \n",
            "\u001b[?25hInstalling collected packages: python-crfsuite\n",
            "Successfully installed python-crfsuite-0.9.7\n",
            "Collecting sklearn_crfsuite\n",
            "  Downloading https://files.pythonhosted.org/packages/25/74/5b7befa513482e6dee1f3dd68171a6c9dfc14c0eaa00f885ffeba54fe9b0/sklearn_crfsuite-0.3.6-py2.py3-none-any.whl\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from sklearn_crfsuite) (0.8.7)\n",
            "Requirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.6/dist-packages (from sklearn_crfsuite) (4.41.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sklearn_crfsuite) (1.15.0)\n",
            "Requirement already satisfied: python-crfsuite>=0.8.3 in /usr/local/lib/python3.6/dist-packages (from sklearn_crfsuite) (0.9.7)\n",
            "Installing collected packages: sklearn-crfsuite\n",
            "Successfully installed sklearn-crfsuite-0.3.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nvk_TLweTVMy"
      },
      "source": [
        "# Preparing Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ecM6byKQPLI"
      },
      "source": [
        "my_data = []\n",
        "with open(ne_path) as f:\n",
        "    for line in f:\n",
        "      my_data.append(line[:-1])\n",
        "\n",
        "\n",
        "my_data = []\n",
        "with open(ne_path) as f:\n",
        "    for line in f:\n",
        "      my_data.append(line[:-1])\n",
        "\n",
        "tagged_data = []\n",
        "mode = 0 # 0->no tag, 1->beg_tag 2-> mid_tag\n",
        "for i in range(len(my_data)):\n",
        "  for each in my_data[i].split():\n",
        "    if mode == 0: # no tag\n",
        "      if each == \"<b_enamex\":\n",
        "        mode = 1\n",
        "      else:\n",
        "        tagged_data.append(str(i+1) + \" \" + each + \" O\")\n",
        "    elif mode == 1: # beg_tag\n",
        "      tag_type = each.split(\"\\\"\")[1]\n",
        "      if \"<e_enamex>\" in each:\n",
        "        tagged_data.append(str(i+1) + \" \" + each.split(\"\\\"\")[2].split(\"<\")[0][1:] + \" B-\" + tag_type[:3] )\n",
        "        mode = 0\n",
        "      else:\n",
        "        tagged_data.append(str(i+1) + \" \" + each.split(\"\\\"\")[2].split(\">\")[1] + \" B-\" + tag_type[:3] )\n",
        "        mode = 2\n",
        "    elif mode == 2: # mid_tag\n",
        "      if \"<e_enamex>\" in each:\n",
        "        tagged_data.append(str(i+1) + \" \" + each.split(\"<\")[0] + \" I-\" + tag_type[:3] )\n",
        "        mode = 0\n",
        "      else:\n",
        "        tagged_data.append(str(i+1) + \" \" + each + \" I-\" + tag_type[:3] )\n",
        "\n",
        "\n",
        "ner_tagged_data = tagged_data\n",
        "ner_tagged_list_of_sentences = []\n",
        "\n",
        "line_cnt = 1\n",
        "temp_line = []\n",
        "for each in ner_tagged_data:\n",
        "  current_line = int(each.split()[0])\n",
        "  if current_line == line_cnt:\n",
        "    temp_line.append(each)\n",
        "  else:\n",
        "    ner_tagged_list_of_sentences.append(temp_line)\n",
        "    temp_line = [each]\n",
        "    line_cnt += 1\n",
        "ner_tagged_list_of_sentences.append(temp_line)\n",
        "\n",
        "\n",
        "ma_data = []\n",
        "with open(ma_path) as f:\n",
        "    for line in f:\n",
        "        ma_data.append(line[:-1])\n",
        "ma_data_list_of_sentences = []\n",
        "\n",
        "line_cnt = 1\n",
        "temp_line = []\n",
        "for each in ma_data:\n",
        "  current_line = int(each.split()[0])\n",
        "  if current_line == line_cnt:\n",
        "    temp_line.append(each)\n",
        "  else:\n",
        "    ma_data_list_of_sentences.append(temp_line)\n",
        "    temp_line = [each]\n",
        "    line_cnt += 1\n",
        "ma_data_list_of_sentences.append(temp_line)\n",
        "\n",
        "df = []\n",
        "\n",
        "for i in range(len(ner_tagged_list_of_sentences)):\n",
        "    sentence = ner_tagged_list_of_sentences[i]\n",
        "    ma_cnt = 0\n",
        "    temp_sent = []\n",
        "    for j in range(len(sentence)):\n",
        "      word = sentence[j]\n",
        "      if ma_cnt < len(ma_data_list_of_sentences[i])  and  word.split()[1] in ma_data_list_of_sentences[i][ma_cnt].split()[1]:\n",
        "        ma_tag = ma_data_list_of_sentences[i][ma_cnt].split()[2]\n",
        "        ma_cnt += 1\n",
        "      else:\n",
        "        ma_tag = \"*UNKNOWN*\"\n",
        "      line_no = word.split()[0]\n",
        "      token = word.split()[1]\n",
        "      if(len(word.split()) < 3):\n",
        "        ner_tag = word.split()[1]\n",
        "      else:\n",
        "        ner_tag = word.split()[2]\n",
        "\n",
        "      temp_sent.append(line_no + \" \" + token + \" \" + ma_tag + \" \" + ner_tag)\n",
        "    df.append(temp_sent)\n",
        "  \n",
        "ner_data_sentenced = df\n",
        "\n",
        "to_file = []\n",
        "for each in ner_data_sentenced:\n",
        "  for a in each:\n",
        "    to_file.append(a+\"\\n\")\n",
        "with open(ne_deneme_path, \"w\") as file1:\n",
        "    file1.writelines(to_file)\n"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SWlIa_t5TX9V"
      },
      "source": [
        "# Reading Lexicon and Prepared Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jt9ib2MoQpPN"
      },
      "source": [
        "lexicon_person = []\n",
        "lexicon_location = []\n",
        "lexicon_organization = []\n",
        "\n",
        "\n",
        "with open(path_per) as f:\n",
        "  for line in f:\n",
        "    lexicon_person.append(line[:-1])\n",
        "\n",
        "with open(path_loc) as f:\n",
        "  for line in f:\n",
        "    lexicon_location.append(line[:-1])  \n",
        "\n",
        "with open(path_org) as f:\n",
        "  for line in f:\n",
        "    lexicon_organization.append(line[:-1])\n",
        "\n",
        "\n",
        "ner_data = []\n",
        "with open(ne_deneme_path) as f:\n",
        "    for line in f:\n",
        "      ner_data.append(line[:-1])\n",
        "\n",
        "ner_data_sentenced = []\n",
        "\n",
        "line_cnt = 1\n",
        "temp_line = []\n",
        "for each in ner_data:\n",
        "  current_line = int(each.split()[0])\n",
        "  if current_line == line_cnt:\n",
        "    temp_line.append(each)\n",
        "  else:\n",
        "    ner_data_sentenced.append(temp_line)\n",
        "    temp_line = [each]\n",
        "    line_cnt += 1\n",
        "ner_data_sentenced.append(temp_line)\n",
        "\n",
        "new_data = []\n",
        "\n",
        "for i in range(len(ner_data_sentenced)):\n",
        "  if i % 5 == 0:\n",
        "    new_data.append(ner_data_sentenced[i])\n",
        "\n",
        "for i in range(len(ner_data_sentenced)):\n",
        "  if i % 5 == 1:\n",
        "    new_data.append(ner_data_sentenced[i])\n",
        "\n",
        "for i in range(len(ner_data_sentenced)):\n",
        "  if i % 5 == 2:\n",
        "    new_data.append(ner_data_sentenced[i])\n",
        "\n",
        "for i in range(len(ner_data_sentenced)):\n",
        "  if i % 5 == 3:\n",
        "    new_data.append(ner_data_sentenced[i])\n",
        "\n",
        "for i in range(len(ner_data_sentenced)):\n",
        "  if i % 5 == 4:\n",
        "    new_data.append(ner_data_sentenced[i])\n"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ikOerDrjTb7P"
      },
      "source": [
        "# CRF function definations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aBBBeQSdQrsb"
      },
      "source": [
        "def get_pos(line):\n",
        "    tags = line.split()[2].split(\"^\")[-1]\n",
        "    tag = \"0\"\n",
        "    if(len(tags.split(\"+\")) > 1):\n",
        "        tag = tags.split(\"+\")[1]\n",
        "    return tag\n",
        "\n",
        "def get_root(line):\n",
        "    tags = line.split()[2]\n",
        "    tag = tags.split(\"+\")[0]\n",
        "    return tag\n",
        "\n",
        "def get_prop(line):\n",
        "    tag = False\n",
        "    if \"+Prop\" in line:\n",
        "        tag = True\n",
        "    return tag\n",
        "\n",
        "def get_noun_case(line):\n",
        "    tags = line.split()[2]\n",
        "    tag = tags.split(\"+\")[-1]\n",
        "    if tag not in [\"Acc\", \"Dat\", \"Equ\", \"Gen\", \"Ins\", \"Nom\", \"Loc\", \"Abl\"]:\n",
        "        tag = \"0\"\n",
        "    return tag\n",
        "\n",
        "def get_orthographic_case(line):\n",
        "    token = line.split()[1]\n",
        "    tag = \"LC\"\n",
        "    if token[0].isupper():\n",
        "        tag = \"UC\"\n",
        "    return tag\n",
        "\n",
        "def get_all_inflectional(line):\n",
        "    tags = line.split()[2].split(\"^\")[-1]\n",
        "    all_inf = \"\"\n",
        "    if(len(tags.split(\"+\")) > 1):\n",
        "        tag = tags.split(\"+\")[2:]\n",
        "        all_inf = ' '.join(tag)\n",
        "    else:\n",
        "        all_inf = \"0\"\n",
        "    return all_inf\n",
        "\n",
        "\n",
        "def check_lexicon(line):\n",
        "    token = line.split()[1]\n",
        "    in_lexicon = 0\n",
        "    if token[0].isupper() and (token in lexicon_person):\n",
        "      in_lexicon = \"PER\"\n",
        "    elif token[0].isupper() and (token in lexicon_location):\n",
        "      in_lexicon = \"LOC\"\n",
        "    elif token[0].isupper() and (token in lexicon_organization):\n",
        "      in_lexicon = \"ORG\"\n",
        "    return in_lexicon\n",
        "\n",
        "# def check_person(line):\n",
        "#   token = line.split()[1]\n",
        "#   return (token[0].isupper() and (token in lexicon_person))\n",
        "\n",
        "\n",
        "# def check_organization(line):\n",
        "#   token = line.split()[1]\n",
        "#   return (token[0].isupper() and (token in lexicon_organization))\n",
        "\n",
        "\n",
        "# def check_location(line):\n",
        "#   token = line.split()[1]\n",
        "#   return (token[0].isupper() and (token in lexicon_location))\n",
        "\n",
        "\n",
        "\n",
        "def word2features(sent, i):\n",
        "\n",
        "    word = sent[i].split()[1]\n",
        "    root = get_root(sent[i])\n",
        "    postag = get_pos(sent[i])\n",
        "    prop = get_prop(sent[i])\n",
        "    noun_case = get_noun_case(sent[i])\n",
        "    orthographic_case = get_orthographic_case(sent[i])\n",
        "    all_inflectional = get_all_inflectional(sent[i])\n",
        "\n",
        "    in_lexicon = check_lexicon(sent[i])\n",
        "\n",
        "    # in_loc_lexicon = False\n",
        "    # in_org_lexicon = False\n",
        "    # in_per_lexicon = False\n",
        "\n",
        "    # in_loc_lexicon = check_location(sent[i])\n",
        "    # if not in_loc_lexicon:\n",
        "    #   in_org_lexicon = check_organization(sent[i])\n",
        "    # if not in_loc_lexicon and not in_org_lexicon:\n",
        "    #   in_per_lexicon = check_person(sent[i])\n",
        "\n",
        "\n",
        "    bos = False\n",
        "    if i == 0:\n",
        "      bos = True\n",
        "\n",
        "    features = {\n",
        "        'word.bias': 1.0,\n",
        "        'word.root' : root,\n",
        "        'word.postag': postag,\n",
        "        'word.prop' : prop,\n",
        "        'word.noun_case' : noun_case,\n",
        "        'word.orthographic_case' : orthographic_case,\n",
        "        'word.all_inflectional' : all_inflectional,\n",
        "        'word.BOS' : bos,\n",
        "        'word.lower()': word.lower(),\n",
        "        'word[-3:]': word[-3:],\n",
        "        'word[-2:]': word[-2:],\n",
        "        'word.isdigit()': word.isdigit(),\n",
        "        'word.isnotpunc()': word.isalpha(),\n",
        "        'word.in_lexicon': in_lexicon,\n",
        "\n",
        "        # 'word.in_loc_lexicon' : in_loc_lexicon,\n",
        "        # 'word.in_org_lexicon' : in_org_lexicon,\n",
        "        # 'word.in_per_lexicon' : in_per_lexicon,\n",
        "    }\n",
        "    if i > 0:\n",
        "      root1 = get_root(sent[i-1])\n",
        "      postag1 = get_pos(sent[i-1])\n",
        "      prop1 = get_prop(sent[i-1])\n",
        "      noun_case1 = get_noun_case(sent[i-1])\n",
        "      orthographic_case1 = get_orthographic_case(sent[i-1])\n",
        "      all_inflectional1 = get_all_inflectional(sent[i-1])\n",
        "      word1 = sent[i-1].split()[1]\n",
        "      in_lexicon1 = check_lexicon(sent[i-1])\n",
        "\n",
        "      # in_loc_lexicon1 = False\n",
        "      # in_org_lexicon1 = False\n",
        "      # in_per_lexicon1 = False\n",
        "\n",
        "      # in_loc_lexicon1 = check_location(sent[i-1])\n",
        "      # if not in_loc_lexicon1:\n",
        "      #   in_org_lexicon1 = check_organization(sent[i-1])\n",
        "      # if not in_loc_lexicon1 and not in_org_lexicon1:\n",
        "      #   in_per_lexicon1 = check_person(sent[i-1])\n",
        "\n",
        "      features.update({\n",
        "        '-1:word.root' : root1,\n",
        "        '-1:word.postag': postag1,\n",
        "        '-1:word.prop' : prop1,\n",
        "        '-1:word.noun_case' : noun_case1,\n",
        "        '-1:word.orthographic_case' : orthographic_case1,\n",
        "        '-1:word.all_inflectional' : all_inflectional1,\n",
        "        '-1:word.lower()': word1.lower(),\n",
        "        '-1:word[-3:]': word1[-3:],\n",
        "        '-1:word[-2:]': word1[-2:],\n",
        "        '-1:word.isdigit()': word1.isdigit(),\n",
        "        '-1:word.in_lexicon': in_lexicon1,\n",
        "        # '-1:word.in_loc_lexicon' : in_loc_lexicon1,\n",
        "        # '-1:word.in_org_lexicon' : in_org_lexicon1,\n",
        "        # '-1:word.in_per_lexicon' : in_per_lexicon1,\n",
        "      })\n",
        "        \n",
        "    if i < len(sent)-1:\n",
        "      root1 = get_root(sent[i+1])\n",
        "      postag1 = get_pos(sent[i+1])\n",
        "      prop1 = get_prop(sent[i+1])\n",
        "      noun_case1 = get_noun_case(sent[i+1])\n",
        "      orthographic_case1 = get_orthographic_case(sent[i+1])\n",
        "      all_inflectional1 = get_all_inflectional(sent[i+1])\n",
        "      word1 = sent[i+1].split()[1]\n",
        "      in_lexicon1 = check_lexicon(sent[i+1])\n",
        "\n",
        "      # in_loc_lexicon1 = False\n",
        "      # in_org_lexicon1 = False\n",
        "      # in_per_lexicon1 = False\n",
        "\n",
        "      # in_loc_lexicon1 = check_location(sent[i+1])\n",
        "      # if not in_loc_lexicon1:\n",
        "      #   in_org_lexicon1 = check_organization(sent[i+1])\n",
        "      # if not in_loc_lexicon1 and not  in_org_lexicon1:\n",
        "      #   in_per_lexicon1 = check_person(sent[i+1])\n",
        "\n",
        "      features.update({\n",
        "        '+1:word.root' : root1,\n",
        "        '+1:word.postag': postag1,\n",
        "        '+1:word.prop' : prop1,\n",
        "        '+1:word.noun_case' : noun_case1,\n",
        "        '+1:word.orthographic_case' : orthographic_case1,\n",
        "        '+1:word.all_inflectional' : all_inflectional1,\n",
        "        '+1:word.lower()': word1.lower(),\n",
        "        '+1:word[-3:]': word1[-3:],\n",
        "        '+1:word[-2:]': word1[-2:],\n",
        "        '+1:word.isdigit()': word1.isdigit(),\n",
        "        '+1:word.in_lexicon': in_lexicon1,\n",
        "        # '+1:word.in_loc_lexicon' : in_loc_lexicon1,\n",
        "        # '+1:word.in_org_lexicon' : in_org_lexicon1,\n",
        "        # '+1:word.in_per_lexicon' : in_per_lexicon1,\n",
        "      })\n",
        "    else:\n",
        "        features['word.EOS'] = True\n",
        "    return features\n",
        "\n",
        "\n",
        "def sent2features(sent):\n",
        "    return [word2features(sent, i) for i in range(len(sent))]\n",
        "\n",
        "def sent2labels(sent):\n",
        "  label_list = []\n",
        "  for each in sent:\n",
        "    label_list.append(each.split()[-1])\n",
        "  return label_list\n",
        "\n",
        "def sent2tokens(sent):\n",
        "  token_list = []\n",
        "  for each in sent:\n",
        "    token_list.append(each.split()[1])\n",
        "  return token_list\n",
        "\n",
        "# sent2features(test_data[10])[0]\n",
        "# test_data[10]\n",
        "# sent2tokens(train_data[0])[0]\n",
        "# sent2labels(train_data[0])[0]\n",
        "\n",
        "\n",
        "def run_crf(fold):\n",
        "  # data = copy.copy(new_data)\n",
        "\n",
        "  temp_data = copy.copy(featurezed_data)\n",
        "  temp_labels = copy.copy(featurezed_labels)\n",
        "\n",
        "\n",
        "  X_test = temp_data[fold[0]:fold[1]]\n",
        "  y_test = temp_labels[fold[0]:fold[1]]\n",
        "  del temp_data[fold[0]:fold[1]]\n",
        "  del temp_labels[fold[0]:fold[1]]\n",
        "\n",
        "  X_train = temp_data\n",
        "  y_train = temp_labels\n",
        "\n",
        "\n",
        "  # test_data = data[fold[0]:fold[1]]\n",
        "  # del data[fold[0]:fold[1]]\n",
        "  # train_data = data\n",
        "\n",
        "  # X_train = [sent2features(s) for s in train_data]\n",
        "  # y_train = [sent2labels(s) for s in train_data]\n",
        "\n",
        "  # X_test = [sent2features(s) for s in test_data]\n",
        "  # y_test = [sent2labels(s) for s in test_data]\n",
        "\n",
        "  crf = sklearn_crfsuite.CRF(\n",
        "      algorithm='lbfgs', \n",
        "      c1=0.1, \n",
        "      c2=0.1, \n",
        "      max_iterations=100, \n",
        "      all_possible_transitions=True,\n",
        "      # num_memories = 30\n",
        "  )\n",
        "\n",
        "  crf.fit(X_train, y_train)\n",
        "  labels = list(crf.classes_)\n",
        "  labels.remove('O')\n",
        "  y_pred = crf.predict(X_test)\n",
        "\n",
        "  sorted_labels = sorted(\n",
        "      labels, \n",
        "      key=lambda name: (name[1:], name[0])\n",
        "      )\n",
        "  print(\"F1 Score for:\" , fold , \"is: \" ,metrics.flat_f1_score(y_test, y_pred, average='weighted', labels=labels))\n",
        "  # print(\"Classification Report for:\" , fold , \"is: \\n\" ,metrics.flat_classification_report(y_test, y_pred, labels=sorted_labels, digits=3))\n",
        "  return metrics.flat_classification_report(y_test, y_pred, labels=sorted_labels, digits=3)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z9Lx9BKvTgAT"
      },
      "source": [
        "# Folds and CRF Runs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fXpIJlCVQwMg",
        "outputId": "9816a329-5dd6-4627-c6e4-726b91d0d7ed"
      },
      "source": [
        "folds = [(0,2000), (2000,4000), (4000,6000), (6000,8000), (8000,10000)]\n",
        "featurezed_data = [sent2features(s) for s in new_data]\n",
        "featurezed_labels = [sent2labels(s) for s in new_data]\n",
        "print(\"featurezation done\")\n",
        "reps =[]\n",
        "for fold in folds:\n",
        "  reps.append(run_crf(fold))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "featurezation done\n",
            "F1 Score for: (0, 2000) is:  0.910418113904667\n",
            "F1 Score for: (2000, 4000) is:  0.9248508042918707\n",
            "F1 Score for: (4000, 6000) is:  0.92677417853977\n",
            "F1 Score for: (6000, 8000) is:  0.9133060665223309\n",
            "F1 Score for: (8000, 10000) is:  0.9156499918978198\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rHgkELDOTk32"
      },
      "source": [
        "# Calculating Average and Outputs of the results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DxBjfec6Qxzp",
        "outputId": "584c0fbc-112e-4b9b-9115-56c62a60a242"
      },
      "source": [
        "index_B_LOC_precision = 5\n",
        "index_B_LOC_recall = 6\n",
        "index_B_LOC_f1_score = 7\n",
        "\n",
        "index_I_LOC_precision = 10\n",
        "index_I_LOC_recall = 11\n",
        "index_I_LOC_f1_score = 12\n",
        "\n",
        "\n",
        "index_B_ORG_precision = 15\n",
        "index_B_ORG_recall = 16\n",
        "index_B_ORG_f1_score = 17\n",
        "\n",
        "index_I_ORG_precision = 20\n",
        "index_I_ORG_recall = 21\n",
        "index_I_ORG_f1_score = 22\n",
        "\n",
        "\n",
        "index_B_PER_precision = 25\n",
        "index_B_PER_recall = 26\n",
        "index_B_PER_f1_score = 27\n",
        "\n",
        "index_I_PER_precision = 30\n",
        "index_I_PER_recall = 31\n",
        "index_I_PER_f1_score = 32\n",
        "\n",
        "\n",
        "index_micro_avg_precision = 36\n",
        "index_micro_avg_recall = 37\n",
        "index_micro_avg_f1_score = 38\n",
        "\n",
        "\n",
        "index_macro_avg_precision = 42\n",
        "index_macro_avg_recall = 43\n",
        "index_macro_avg_f1_score = 44\n",
        "\n",
        "\n",
        "index_weighted_avg_precision = 48\n",
        "index_weighted_avg_recall = 49\n",
        "index_weighted_avg_f1_score = 50\n",
        "\n",
        "\n",
        "\n",
        "total_B_LOC_precision = 0\n",
        "total_B_LOC_recall = 0\n",
        "total_B_LOC_f1_score = 0\n",
        "\n",
        "total_I_LOC_precision = 0\n",
        "total_I_LOC_recall = 0\n",
        "total_I_LOC_f1_score = 0\n",
        "\n",
        "\n",
        "total_B_ORG_precision = 0\n",
        "total_B_ORG_recall = 0\n",
        "total_B_ORG_f1_score = 0\n",
        "\n",
        "total_I_ORG_precision = 0\n",
        "total_I_ORG_recall = 0\n",
        "total_I_ORG_f1_score = 0\n",
        "\n",
        "\n",
        "total_B_PER_precision = 0\n",
        "total_B_PER_recall = 0\n",
        "total_B_PER_f1_score = 0\n",
        "\n",
        "total_I_PER_precision = 0\n",
        "total_I_PER_recall = 0\n",
        "total_I_PER_f1_score = 0\n",
        "\n",
        "\n",
        "total_micro_avg_precision = 0\n",
        "total_micro_avg_recall = 0\n",
        "total_micro_avg_f1_score = 0\n",
        "\n",
        "\n",
        "total_macro_avg_precision = 0\n",
        "total_macro_avg_recall = 0\n",
        "total_macro_avg_f1_score = 0\n",
        "\n",
        "\n",
        "total_weighted_avg_precision = 0\n",
        "total_weighted_avg_recall = 0\n",
        "total_weighted_avg_f1_score = 0\n",
        "\n",
        "for each_fold in reps:\n",
        "\n",
        "  total_B_LOC_precision += float(each_fold.split()[index_B_LOC_precision])\n",
        "  total_B_LOC_recall += float(each_fold.split()[index_B_LOC_recall])\n",
        "  total_B_LOC_f1_score += float(each_fold.split()[index_B_LOC_f1_score])\n",
        "\n",
        "  total_I_LOC_precision += float(each_fold.split()[index_I_LOC_precision])\n",
        "  total_I_LOC_recall += float(each_fold.split()[index_I_LOC_recall])\n",
        "  total_I_LOC_f1_score += float(each_fold.split()[index_I_LOC_f1_score])\n",
        "\n",
        "\n",
        "  total_B_ORG_precision += float(each_fold.split()[index_B_ORG_precision])\n",
        "  total_B_ORG_recall += float(each_fold.split()[index_B_ORG_recall])\n",
        "  total_B_ORG_f1_score += float(each_fold.split()[index_B_ORG_f1_score])\n",
        "\n",
        "  total_I_ORG_precision += float(each_fold.split()[index_I_ORG_precision])\n",
        "  total_I_ORG_recall += float(each_fold.split()[index_I_ORG_recall])\n",
        "  total_I_ORG_f1_score += float(each_fold.split()[index_I_ORG_f1_score])\n",
        "\n",
        "\n",
        "  total_B_PER_precision += float(each_fold.split()[index_B_PER_precision])\n",
        "  total_B_PER_recall += float(each_fold.split()[index_B_PER_recall])\n",
        "  total_B_PER_f1_score += float(each_fold.split()[index_B_PER_f1_score])\n",
        "\n",
        "  total_I_PER_precision += float(each_fold.split()[index_I_PER_precision])\n",
        "  total_I_PER_recall += float(each_fold.split()[index_I_PER_recall])\n",
        "  total_I_PER_f1_score += float(each_fold.split()[index_I_PER_f1_score])\n",
        "\n",
        "\n",
        "  total_micro_avg_precision += float(each_fold.split()[index_micro_avg_precision])\n",
        "  total_micro_avg_recall += float(each_fold.split()[index_micro_avg_recall])\n",
        "  total_micro_avg_f1_score += float(each_fold.split()[index_micro_avg_f1_score])\n",
        "\n",
        "\n",
        "  total_macro_avg_precision += float(each_fold.split()[index_macro_avg_precision])\n",
        "  total_macro_avg_recall += float(each_fold.split()[index_macro_avg_recall])\n",
        "  total_macro_avg_f1_score += float(each_fold.split()[index_macro_avg_f1_score])\n",
        "\n",
        "\n",
        "  total_weighted_avg_precision += float(each_fold.split()[index_weighted_avg_precision])\n",
        "  total_weighted_avg_recall += float(each_fold.split()[index_weighted_avg_recall])\n",
        "  total_weighted_avg_f1_score += float(each_fold.split()[index_weighted_avg_f1_score])\n",
        "\n",
        "\n",
        "total_B_LOC_precision /= 5\n",
        "total_B_LOC_recall /= 5\n",
        "total_B_LOC_f1_score /= 5\n",
        "\n",
        "total_I_LOC_precision /= 5\n",
        "total_I_LOC_recall /= 5\n",
        "total_I_LOC_f1_score /= 5\n",
        "\n",
        "\n",
        "total_B_ORG_precision /= 5\n",
        "total_B_ORG_recall /= 5\n",
        "total_B_ORG_f1_score /= 5\n",
        "\n",
        "total_I_ORG_precision /= 5\n",
        "total_I_ORG_recall /= 5\n",
        "total_I_ORG_f1_score /= 5\n",
        "\n",
        "\n",
        "total_B_PER_precision /= 5\n",
        "total_B_PER_recall /= 5\n",
        "total_B_PER_f1_score /= 5\n",
        "\n",
        "total_I_PER_precision /= 5\n",
        "total_I_PER_recall /= 5\n",
        "total_I_PER_f1_score /= 5\n",
        "\n",
        "\n",
        "total_micro_avg_precision /= 5\n",
        "total_micro_avg_recall /= 5\n",
        "total_micro_avg_f1_score /= 5\n",
        "\n",
        "\n",
        "total_macro_avg_precision /= 5\n",
        "total_macro_avg_recall /= 5\n",
        "total_macro_avg_f1_score /= 5\n",
        "\n",
        "\n",
        "total_weighted_avg_precision /= 5\n",
        "total_weighted_avg_recall /= 5\n",
        "total_weighted_avg_f1_score /= 5\n",
        "\n",
        "\n",
        "average_classification_report = \"\\t\\tprecision\\trecall\\t\\tf1-score\\n\\n\" \n",
        "average_classification_report += \"\\tB-LOC\\t\" + str(total_B_LOC_precision)[:5] +\"\\t\\t\" + str(total_B_LOC_recall)[:5] + \"\\t\\t\" + str(total_B_LOC_f1_score)[:5] + \"\\n\"\n",
        "average_classification_report += \"\\tI_LOC\\t\" + str(total_I_LOC_precision)[:5] +\"\\t\\t\" + str(total_I_LOC_recall)[:5] + \"\\t\\t\" + str(total_I_LOC_f1_score)[:5] + \"\\n\"\n",
        "\n",
        "average_classification_report += \"\\tB-ORG\\t\" + str(total_B_ORG_precision)[:5] +\"\\t\\t\" + str(total_B_ORG_recall)[:5] + \"\\t\\t\" + str(total_B_ORG_f1_score)[:5] + \"\\n\"\n",
        "average_classification_report += \"\\tI_ORG\\t\" + str(total_I_ORG_precision)[:5] +\"\\t\\t\" + str(total_I_ORG_recall)[:5] + \"\\t\\t\" + str(total_I_ORG_f1_score)[:5] + \"\\n\"\n",
        "\n",
        "average_classification_report += \"\\tB-PER\\t\" + str(total_B_PER_precision)[:5] +\"\\t\\t\" + str(total_B_PER_recall)[:5] + \"\\t\\t\" + str(total_B_PER_f1_score)[:5] + \"\\n\"\n",
        "average_classification_report += \"\\tI_PER\\t\" + str(total_I_PER_precision)[:5] +\"\\t\\t\" + str(total_I_PER_recall)[:5] + \"\\t\\t\" + str(total_I_PER_f1_score)[:5] + \"\\n\\n\"\n",
        "\n",
        "average_classification_report += \"    micro avg\\t\" + str(total_micro_avg_precision)[:5] +\"\\t\\t\" + str(total_micro_avg_recall)[:5] + \"\\t\\t\" + str(total_micro_avg_f1_score)[:5] + \"\\n\"\n",
        "average_classification_report += \"    macro avg\\t\" + str(total_macro_avg_precision)[:5] +\"\\t\\t\" + str(total_macro_avg_recall)[:5] + \"\\t\\t\" + str(total_macro_avg_f1_score)[:5] + \"\\n\"\n",
        "average_classification_report += \" weighted avg\\t\" + str(total_weighted_avg_precision)[:5] +\"\\t\\t\" + str(total_weighted_avg_recall)[:5] + \"\\t\\t\" + str(total_weighted_avg_f1_score)[:5] + \"\\n\"\n",
        "\n",
        "print(\"Average --> (Fold 1 + Fold 2 + Fold 3 + Fold 4 + Fold 5) / 5\")\n",
        "print(average_classification_report, \"\\n<|--------------------------------------------------------|>\\n\\n\")\n",
        "\n",
        "for i in range(len(reps)):\n",
        "  print(\"Fold\" ,i, \"-->\" , folds[i])\n",
        "  print(reps[i] , \"\\n<|--------------------------------------------------------|>\\n\\n\")"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Average --> (Fold 1 + Fold 2 + Fold 3 + Fold 4 + Fold 5) / 5\n",
            "\t\tprecision\trecall\t\tf1-score\n",
            "\n",
            "\tB-LOC\t0.949\t\t0.937\t\t0.943\n",
            "\tI_LOC\t0.824\t\t0.722\t\t0.768\n",
            "\tB-ORG\t0.923\t\t0.912\t\t0.917\n",
            "\tI_ORG\t0.868\t\t0.860\t\t0.863\n",
            "\tB-PER\t0.933\t\t0.937\t\t0.935\n",
            "\tI_PER\t0.909\t\t0.921\t\t0.915\n",
            "\n",
            "    micro avg\t0.921\t\t0.915\t\t0.918\n",
            "    macro avg\t0.901\t\t0.882\t\t0.890\n",
            " weighted avg\t0.921\t\t0.915\t\t0.918\n",
            " \n",
            "<|--------------------------------------------------------|>\n",
            "\n",
            "\n",
            "Fold 0 --> (0, 2000)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       B-LOC      0.945     0.938     0.942       850\n",
            "       I-LOC      0.849     0.646     0.734       113\n",
            "       B-ORG      0.909     0.902     0.905       650\n",
            "       I-ORG      0.852     0.828     0.840       424\n",
            "       B-PER      0.932     0.942     0.937      1055\n",
            "       I-PER      0.905     0.910     0.908       458\n",
            "\n",
            "   micro avg      0.916     0.906     0.911      3550\n",
            "   macro avg      0.899     0.861     0.877      3550\n",
            "weighted avg      0.915     0.906     0.910      3550\n",
            " \n",
            "<|--------------------------------------------------------|>\n",
            "\n",
            "\n",
            "Fold 1 --> (2000, 4000)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       B-LOC      0.943     0.953     0.948       762\n",
            "       I-LOC      0.793     0.793     0.793        92\n",
            "       B-ORG      0.928     0.903     0.915       568\n",
            "       I-ORG      0.879     0.830     0.854       395\n",
            "       B-PER      0.946     0.943     0.944      1064\n",
            "       I-PER      0.933     0.946     0.939       498\n",
            "\n",
            "   micro avg      0.929     0.922     0.925      3379\n",
            "   macro avg      0.904     0.895     0.899      3379\n",
            "weighted avg      0.928     0.922     0.925      3379\n",
            " \n",
            "<|--------------------------------------------------------|>\n",
            "\n",
            "\n",
            "Fold 2 --> (4000, 6000)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       B-LOC      0.958     0.932     0.945       811\n",
            "       I-LOC      0.884     0.731     0.800       104\n",
            "       B-ORG      0.931     0.936     0.934       594\n",
            "       I-ORG      0.893     0.909     0.901       396\n",
            "       B-PER      0.927     0.944     0.936      1109\n",
            "       I-PER      0.907     0.924     0.916       488\n",
            "\n",
            "   micro avg      0.927     0.927     0.927      3502\n",
            "   macro avg      0.917     0.896     0.905      3502\n",
            "weighted avg      0.927     0.927     0.927      3502\n",
            " \n",
            "<|--------------------------------------------------------|>\n",
            "\n",
            "\n",
            "Fold 3 --> (6000, 8000)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       B-LOC      0.953     0.932     0.943       899\n",
            "       I-LOC      0.817     0.742     0.777       120\n",
            "       B-ORG      0.937     0.901     0.919       646\n",
            "       I-ORG      0.851     0.845     0.848       438\n",
            "       B-PER      0.931     0.929     0.930      1141\n",
            "       I-PER      0.893     0.920     0.906       498\n",
            "\n",
            "   micro avg      0.919     0.908     0.913      3742\n",
            "   macro avg      0.897     0.878     0.887      3742\n",
            "weighted avg      0.919     0.908     0.913      3742\n",
            " \n",
            "<|--------------------------------------------------------|>\n",
            "\n",
            "\n",
            "Fold 4 --> (8000, 10000)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       B-LOC      0.948     0.932     0.940       859\n",
            "       I-LOC      0.780     0.702     0.739       121\n",
            "       B-ORG      0.914     0.918     0.916       588\n",
            "       I-ORG      0.865     0.889     0.877       388\n",
            "       B-PER      0.933     0.930     0.932      1175\n",
            "       I-PER      0.910     0.907     0.909       549\n",
            "\n",
            "   micro avg      0.918     0.914     0.916      3680\n",
            "   macro avg      0.892     0.880     0.885      3680\n",
            "weighted avg      0.918     0.914     0.916      3680\n",
            " \n",
            "<|--------------------------------------------------------|>\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}