project4_prep.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1plvLtlnr4n4",
        "outputId": "22be8be8-4cdc-4051-c840-fb29200b29ea"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting python-crfsuite\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/95/99/869dde6dbf3e0d07a013c8eebfb0a3d30776334e0097f8432b631a9a3a19/python_crfsuite-0.9.7-cp36-cp36m-manylinux1_x86_64.whl (743kB)\n",
            "\r\u001b[K     |▍                               | 10kB 16.7MB/s eta 0:00:01\r\u001b[K     |▉                               | 20kB 21.6MB/s eta 0:00:01\r\u001b[K     |█▎                              | 30kB 13.8MB/s eta 0:00:01\r\u001b[K     |█▊                              | 40kB 11.0MB/s eta 0:00:01\r\u001b[K     |██▏                             | 51kB 9.3MB/s eta 0:00:01\r\u001b[K     |██▋                             | 61kB 8.6MB/s eta 0:00:01\r\u001b[K     |███                             | 71kB 9.0MB/s eta 0:00:01\r\u001b[K     |███▌                            | 81kB 10.0MB/s eta 0:00:01\r\u001b[K     |████                            | 92kB 9.2MB/s eta 0:00:01\r\u001b[K     |████▍                           | 102kB 9.4MB/s eta 0:00:01\r\u001b[K     |████▉                           | 112kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 122kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 133kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████▏                         | 143kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 153kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████                         | 163kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████▌                        | 174kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████                        | 184kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 194kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 204kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 215kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 225kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 235kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 245kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████                     | 256kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 266kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████                    | 276kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 286kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 296kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 307kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 317kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 327kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 337kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 348kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 358kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 368kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 378kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 389kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 399kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 409kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 419kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 430kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 440kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 450kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 460kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 471kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 481kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 491kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 501kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 512kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 522kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 532kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 542kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 552kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 563kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 573kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 583kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 593kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 604kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 614kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 624kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 634kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 645kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 655kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 665kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 675kB 9.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 686kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 696kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 706kB 9.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 716kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 727kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 737kB 9.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 747kB 9.4MB/s \n",
            "\u001b[?25hInstalling collected packages: python-crfsuite\n",
            "Successfully installed python-crfsuite-0.9.7\n"
          ]
        }
      ],
      "source": [
        "!pip install python-crfsuite\n",
        "from itertools import chain\n",
        "import nltk\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "import sklearn\n",
        "import pycrfsuite\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "ne_path =\"/content/drive/MyDrive/Colab Notebooks/project4_data/NE.txt\"\n",
        "ma_path = \"/content/drive/MyDrive/Colab Notebooks/project4_data/NE.ma.txt\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jlasDsT646ks"
      },
      "outputs": [],
      "source": [
        "my_data = []\n",
        "with open(ne_path) as f:\n",
        "    for line in f:\n",
        "      my_data.append(line[:-1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sc-2RIaU53dz"
      },
      "outputs": [],
      "source": [
        "# ma_path = \"NE.ma.txt\"\n",
        "# ne_path = \"NE.txt\"\n",
        "\n",
        "my_data = []\n",
        "with open(ne_path) as f:\n",
        "    for line in f:\n",
        "      my_data.append(line[:-1])\n",
        "\n",
        "tagged_data = []\n",
        "mode = 0 # 0->no tag, 1->beg_tag 2-> mid_tag\n",
        "for i in range(len(my_data)):\n",
        "  for each in my_data[i].split():\n",
        "    if mode == 0: # no tag\n",
        "      if each == \"<b_enamex\":\n",
        "        mode = 1\n",
        "      else:\n",
        "        tagged_data.append(str(i+1) + \" \" + each + \" O\")\n",
        "    elif mode == 1: # beg_tag\n",
        "      tag_type = each.split(\"\\\"\")[1]\n",
        "      if \"<e_enamex>\" in each:\n",
        "        tagged_data.append(str(i+1) + \" \" + each.split(\"\\\"\")[2].split(\"<\")[0][1:] + \" B-\" + tag_type[:3] )\n",
        "        mode = 0\n",
        "      else:\n",
        "        tagged_data.append(str(i+1) + \" \" + each.split(\"\\\"\")[2].split(\">\")[1] + \" B-\" + tag_type[:3] )\n",
        "        mode = 2\n",
        "    elif mode == 2: # mid_tag\n",
        "      if \"<e_enamex>\" in each:\n",
        "        tagged_data.append(str(i+1) + \" \" + each.split(\"<\")[0] + \" I-\" + tag_type[:3] )\n",
        "        mode = 0\n",
        "      else:\n",
        "        tagged_data.append(str(i+1) + \" \" + each + \" I-\" + tag_type[:3] )\n",
        "\n",
        "\n",
        "ner_tagged_data = tagged_data\n",
        "ner_tagged_list_of_sentences = []\n",
        "\n",
        "line_cnt = 1\n",
        "temp_line = []\n",
        "for each in ner_tagged_data:\n",
        "  current_line = int(each.split()[0])\n",
        "  if current_line == line_cnt:\n",
        "    temp_line.append(each)\n",
        "  else:\n",
        "    ner_tagged_list_of_sentences.append(temp_line)\n",
        "    temp_line = [each]\n",
        "    line_cnt += 1\n",
        "ner_tagged_list_of_sentences.append(temp_line)\n",
        "\n",
        "\n",
        "ma_data = []\n",
        "with open(ma_path) as f:\n",
        "    for line in f:\n",
        "        ma_data.append(line[:-1])\n",
        "ma_data_list_of_sentences = []\n",
        "\n",
        "line_cnt = 1\n",
        "temp_line = []\n",
        "for each in ma_data:\n",
        "  current_line = int(each.split()[0])\n",
        "  if current_line == line_cnt:\n",
        "    temp_line.append(each)\n",
        "  else:\n",
        "    ma_data_list_of_sentences.append(temp_line)\n",
        "    temp_line = [each]\n",
        "    line_cnt += 1\n",
        "ma_data_list_of_sentences.append(temp_line)\n",
        "\n",
        "\n",
        "# for i in range(len(ma_data_list_of_sentences)):\n",
        "#     sentence = ma_data_list_of_sentences[i]\n",
        "#     for j in range(len(sentence)):\n",
        "#         word = sentence[j]\n",
        "#         ner_cnt = j\n",
        "#         ner_tag = \"O\"\n",
        "#         while ner_cnt < len(ner_tagged_list_of_sentences[i]):\n",
        "#             if ner_tagged_list_of_sentences[i][ner_cnt].split()[1] in word:\n",
        "#                 ner_tag = ner_tagged_list_of_sentences[i][ner_cnt].split()[2]\n",
        "#                 break\n",
        "#             else:\n",
        "#                 ner_cnt += 1\n",
        "#         word += \" \" + ner_tag\n",
        "#         ma_data_list_of_sentences[i][j] = word\n",
        "\n",
        "df = []\n",
        "\n",
        "for i in range(len(ner_tagged_list_of_sentences)):\n",
        "    sentence = ner_tagged_list_of_sentences[i]\n",
        "    ma_cnt = 0\n",
        "    temp_sent = []\n",
        "    for j in range(len(sentence)):\n",
        "      word = sentence[j]\n",
        "      if ma_cnt < len(ma_data_list_of_sentences[i])  and  word.split()[1] in ma_data_list_of_sentences[i][ma_cnt].split()[1]:\n",
        "        ma_tag = ma_data_list_of_sentences[i][ma_cnt].split()[2]\n",
        "        ma_cnt += 1\n",
        "      else:\n",
        "        ma_tag = \"*UNKNOWN*\"\n",
        "      line_no = word.split()[0]\n",
        "      token = word.split()[1]\n",
        "      if(len(word.split()) < 3):\n",
        "        ner_tag = word.split()[1]\n",
        "      else:\n",
        "        ner_tag = word.split()[2]\n",
        "\n",
        "      temp_sent.append(line_no + \" \" + token + \" \" + ma_tag + \" \" + ner_tag)\n",
        "    df.append(temp_sent)\n",
        "  \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "ner_data_sentenced = df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "klRFiFrZi1Qy",
        "outputId": "9dfb2661-ea91-4fba-8347-4a5f52fc766a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['1 Müzik müzik+Noun+A3sg+Pnon+Nom O',\n",
              " '1 Şenliği Şenlik+Noun+Prop+A3sg+P3sg+Dat O',\n",
              " \"1 'ne *UNKNOWN* O\",\n",
              " '1 hazırlanın hazırla+Verb^DB+Verb+Reflex+Pos+Imp+A2pl O',\n",
              " '1 POZİTİF pozitif+Adj B-ORG',\n",
              " '1 ve ve+Conj I-ORG',\n",
              " '1 Açık açık+Adj I-ORG',\n",
              " '1 Radyo radyo+Noun+A3sg+Pnon+Nom I-ORG',\n",
              " '1 işbirliğiyle işbirliği+Noun+A3sg+Pnon+Ins O',\n",
              " '1 düzenlenecek düzenle+Verb^DB+Verb+Pass+Pos^DB+Adj+FutPart+Pnon O',\n",
              " '1 olan ol+Verb+Pos^DB+Adj+PresPart O',\n",
              " '1 İstanbul İstanbul+Noun+Prop+A3sg+Pnon+Nom B-LOC',\n",
              " '1 Müzik müzik+Noun+A3sg+Pnon+Nom O',\n",
              " '1 Şenliği şenlik+Noun+A3sg+P3sg+Nom O',\n",
              " '1 2 2+Num+Card O',\n",
              " '1 , ,+Punct O',\n",
              " '1 müzikseverlere müziksever+Noun+A3pl+Pnon+Dat O',\n",
              " '1 Aralık aralık+Noun+A3sg+Pnon+Nom O',\n",
              " '1 ayında ay+Noun+A3sg+P3sg+Loc O',\n",
              " '1 merhaba merhaba+Noun+A3sg+Pnon+Nom O',\n",
              " '1 demeye de+Verb+Pos^DB+Noun+Inf2+A3sg+Pnon+Dat O',\n",
              " '1 hazırlanıyor hazırla+Verb^DB+Verb+Pass+Pos+Prog1+A3sg O']"
            ]
          },
          "execution_count": 26,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fzb5R5AO9sSR"
      },
      "outputs": [],
      "source": [
        "to_file = []\n",
        "for each in ner_data_sentenced:\n",
        "  for a in each:\n",
        "    to_file.append(a+\"\\n\")\n",
        "with open(\"/content/drive/MyDrive/Colab Notebooks/project4_data/NE.prep.txt\", \"w\") as file1:\n",
        "    file1.writelines(to_file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "esaEkhnH5867",
        "outputId": "56a31334-b1ed-4433-93ae-8d7c9ea1f25e"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'1 Müzik O'"
            ]
          },
          "execution_count": 13,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tagged_data[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l39exo_bAYsM"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "project4_prep_24_01.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}