task1_20newsgroups.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Тематическая классификация длинных текстов - TFIDF и LogReg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Если Вы запускаете ноутбук на colab или kaggle,\n",
    "# выполните следующие строчки, чтобы подгрузить библиотеку dlnlputils:\n",
    "\n",
    "# !git clone https://github.com/Samsung-IT-Academy/stepik-dl-nlp.git && pip install -r stepik-dl-nlp/requirements.txt\n",
    "# import sys; sys.path.append('./stepik-dl-nlp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:42:57.265628Z",
     "start_time": "2019-09-12T12:42:55.188211Z"
    }
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import collections\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "\n",
    "import dlnlputils\n",
    "from dlnlputils.data import tokenize_text_simple_regex, tokenize_corpus, build_vocabulary, \\\n",
    "    vectorize_texts, SparseFeaturesDataset\n",
    "from dlnlputils.pipeline import train_eval_loop, predict_with_model, init_random_seed\n",
    "\n",
    "init_random_seed()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Предобработка текстов и подготовка признаков"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:42:57.847399Z",
     "start_time": "2019-09-12T12:42:57.268037Z"
    }
   },
   "outputs": [],
   "source": [
    "train_source = fetch_20newsgroups(subset='train')\n",
    "test_source = fetch_20newsgroups(subset='test')\n",
    "\n",
    "print('Количество обучающих текстов', len(train_source['data']))\n",
    "print('Количество тестовых текстов', len(test_source['data']))\n",
    "print()\n",
    "print(train_source['data'][0].strip())\n",
    "\n",
    "print()\n",
    "print('Метка', train_source['target'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Подготовка признаков"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:43:00.294422Z",
     "start_time": "2019-09-12T12:42:57.849386Z"
    }
   },
   "outputs": [],
   "source": [
    "train_tokenized = tokenize_corpus(train_source['data'])\n",
    "test_tokenized = tokenize_corpus(test_source['data'])\n",
    "\n",
    "print(' '.join(train_tokenized[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:43:00.825372Z",
     "start_time": "2019-09-12T12:43:00.297392Z"
    }
   },
   "outputs": [],
   "source": [
    "MAX_DF = 0.8\n",
    "MIN_COUNT = 5\n",
    "vocabulary, word_doc_freq = build_vocabulary(train_tokenized, max_doc_freq=MAX_DF, min_count=MIN_COUNT)\n",
    "UNIQUE_WORDS_N = len(vocabulary)\n",
    "print('Количество уникальных токенов', UNIQUE_WORDS_N)\n",
    "print(list(vocabulary.items())[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:43:01.524600Z",
     "start_time": "2019-09-12T12:43:00.829107Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.hist(word_doc_freq, bins=20)\n",
    "plt.title('Распределение относительных частот слов')\n",
    "plt.yscale('log');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:44:16.094816Z",
     "start_time": "2019-09-12T12:43:01.526554Z"
    }
   },
   "outputs": [],
   "source": [
    "VECTORIZATION_MODE = 'tfidf'\n",
    "train_vectors = vectorize_texts(train_tokenized, vocabulary, word_doc_freq, mode=VECTORIZATION_MODE)\n",
    "test_vectors = vectorize_texts(test_tokenized, vocabulary, word_doc_freq, mode=VECTORIZATION_MODE)\n",
    "\n",
    "print('Размерность матрицы признаков обучающей выборки', train_vectors.shape)\n",
    "print('Размерность матрицы признаков тестовой выборки', test_vectors.shape)\n",
    "print()\n",
    "print('Количество ненулевых элементов в обучающей выборке', train_vectors.nnz)\n",
    "print('Процент заполненности матрицы признаков {:.2f}%'.format(train_vectors.nnz * 100 / (train_vectors.shape[0] * train_vectors.shape[1])))\n",
    "print()\n",
    "print('Количество ненулевых элементов в тестовой выборке', test_vectors.nnz)\n",
    "print('Процент заполненности матрицы признаков {:.2f}%'.format(test_vectors.nnz * 100 / (test_vectors.shape[0] * test_vectors.shape[1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:44:16.857114Z",
     "start_time": "2019-09-12T12:44:16.098773Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.hist(train_vectors.data, bins=20)\n",
    "plt.title('Распределение весов признаков')\n",
    "plt.yscale('log');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Распределение классов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:44:16.864960Z",
     "start_time": "2019-09-12T12:44:16.859476Z"
    }
   },
   "outputs": [],
   "source": [
    "UNIQUE_LABELS_N = len(set(train_source['target']))\n",
    "print('Количество уникальных меток', UNIQUE_LABELS_N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:44:17.106036Z",
     "start_time": "2019-09-12T12:44:16.867310Z"
    }
   },
   "outputs": [],
   "source": [
    "plt.hist(train_source['target'], bins=np.arange(0, 21))\n",
    "plt.title('Распределение меток в обучающей выборке');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:44:17.312198Z",
     "start_time": "2019-09-12T12:44:17.109884Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.hist(test_source['target'], bins=np.arange(0, 21))\n",
    "plt.title('Распределение меток в тестовой выборке');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PyTorch Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:44:17.319292Z",
     "start_time": "2019-09-12T12:44:17.315074Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset = SparseFeaturesDataset(train_vectors, train_source['target'])\n",
    "test_dataset = SparseFeaturesDataset(test_vectors, test_source['target'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Обучение модели на PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:46:22.371272Z",
     "start_time": "2019-09-12T12:44:17.322178Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = nn.Linear(UNIQUE_WORDS_N, UNIQUE_LABELS_N)\n",
    "\n",
    "scheduler = lambda optim: \\\n",
    "    torch.optim.lr_scheduler.ReduceLROnPlateau(optim, patience=5, factor=0.5, verbose=True)\n",
    "\n",
    "best_val_loss, best_model = train_eval_loop(model=model,\n",
    "                                            train_dataset=train_dataset,\n",
    "                                            val_dataset=test_dataset,\n",
    "                                            criterion=F.cross_entropy,\n",
    "                                            lr=1e-1,\n",
    "                                            epoch_n=200,\n",
    "                                            batch_size=32,\n",
    "                                            l2_reg_alpha=0,\n",
    "                                            lr_scheduler_ctor=scheduler)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Оценка качества"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:46:25.105663Z",
     "start_time": "2019-09-12T12:46:22.373012Z"
    }
   },
   "outputs": [],
   "source": [
    "train_pred = predict_with_model(best_model, train_dataset)\n",
    "\n",
    "train_loss = F.cross_entropy(torch.from_numpy(train_pred),\n",
    "                             torch.from_numpy(train_source['target']).long())\n",
    "\n",
    "print('Среднее значение функции потерь на обучении', float(train_loss))\n",
    "print('Доля верных ответов', accuracy_score(train_source['target'], train_pred.argmax(-1)))\n",
    "print()\n",
    "\n",
    "\n",
    "\n",
    "test_pred = predict_with_model(best_model, test_dataset)\n",
    "\n",
    "test_loss = F.cross_entropy(torch.from_numpy(test_pred),\n",
    "                            torch.from_numpy(test_source['target']).long())\n",
    "\n",
    "print('Среднее значение функции потерь на валидации', float(test_loss))\n",
    "print('Доля верных ответов', accuracy_score(test_source['target'], test_pred.argmax(-1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Альтернативная реализация на scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:46:31.791405Z",
     "start_time": "2019-09-12T12:46:25.107897Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "sklearn_pipeline = Pipeline((('vect', TfidfVectorizer(tokenizer=tokenize_text_simple_regex,\n",
    "                                                      max_df=MAX_DF,\n",
    "                                                      min_df=MIN_COUNT)),\n",
    "                             ('cls', LogisticRegression())))\n",
    "sklearn_pipeline.fit(train_source['data'], train_source['target']);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Оценка качества"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T12:46:35.454567Z",
     "start_time": "2019-09-12T12:46:31.792832Z"
    }
   },
   "outputs": [],
   "source": [
    "sklearn_train_pred = sklearn_pipeline.predict_proba(train_source['data'])\n",
    "sklearn_train_loss = F.cross_entropy(torch.from_numpy(sklearn_train_pred),\n",
    "                                                 torch.from_numpy(train_source['target']))\n",
    "print('Среднее значение функции потерь на обучении', float(sklearn_train_loss))\n",
    "print('Доля верных ответов', accuracy_score(train_source['target'], sklearn_train_pred.argmax(-1)))\n",
    "print()\n",
    "\n",
    "sklearn_test_pred = sklearn_pipeline.predict_proba(test_source['data'])\n",
    "sklearn_test_loss = F.cross_entropy(torch.from_numpy(sklearn_test_pred),\n",
    "                                                torch.from_numpy(test_source['target']))\n",
    "print('Среднее значение функции потерь на валидации', float(sklearn_test_loss))\n",
    "print('Доля верных ответов', accuracy_score(test_source['target'], sklearn_test_pred.argmax(-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}