scripts/01_parse_toi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "81246535",
   "metadata": {},
   "source": [
    "### Parse TOI XML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea6a6fc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "from zipfile import ZipFile\n",
    "import xmltodict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "af706ae5",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_file = 'toi_parsed_all.csv.gz'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63212d43",
   "metadata": {},
   "source": [
    "### Step 1: Parse all XML files in each zip file to a csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9c4ddfbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 ./csv/TimesOfIndia_20170928213619_00025.csv.gz 29\n",
      "1 ./csv/TimesOfIndia_20170928222359_00036.csv.gz 29\n",
      "2 ./csv/TimesOfIndia_20170928222610_00045.csv.gz 28\n",
      "3 ./csv/TimesOfIndia_20170928214039_00039.csv.gz 28\n",
      "4 ./csv/TimesOfIndia_20170928222609_00044.csv.gz 28\n",
      "5 ./csv/TimesOfIndia_20170928222919_00005.csv.gz 28\n",
      "6 ./csv/TimesOfIndia_20170928215024_00023.csv.gz 29\n",
      "7 ./csv/TimesOfIndia_20170928221723_00014.csv.gz 28\n",
      "8 ./csv/TimesOfIndia_20170928222141_00026.csv.gz 29\n",
      "9 ./csv/TimesOfIndia_20170928214041_00040.csv.gz 29\n",
      "10 ./csv/TimesOfIndia_20170928214811_00014.csv.gz 29\n",
      "11 ./csv/TimesOfIndia_20170928223340_00019.csv.gz 29\n",
      "12 ./csv/TimesOfIndia_20170928221407_00002.csv.gz 28\n",
      "13 ./csv/TimesOfIndia_20170928222920_00006.csv.gz 28\n",
      "14 ./csv/TimesOfIndia_20170928213413_00021.csv.gz 28\n",
      "15 ./csv/TimesOfIndia_20170928223128_00011.csv.gz 29\n",
      "16 ./csv/TimesOfIndia_20170928210518_00026.csv.gz 28\n",
      "17 ./csv/TimesOfIndia_20170928213202_00014.csv.gz 29\n",
      "18 ./csv/TimesOfIndia_20170928222504_00040.csv.gz 29\n",
      "19 ./csv/TimesOfIndia_20170928214250_00046.csv.gz 28\n",
      "20 ./csv/TimesOfIndia_20170928215654_00043.csv.gz 29\n",
      "21 ./csv/TimesOfIndia_20170928213935_00036.csv.gz 28\n",
      "22 ./csv/TimesOfIndia_20170928210412_00022.csv.gz 28\n",
      "23 ./csv/TimesOfIndia_20170928212841_00047.csv.gz 28\n",
      "24 ./csv/TimesOfIndia_20170928223654_00030.csv.gz 29\n",
      "25 ./csv/TimesOfIndia_20170928221156_00041.csv.gz 29\n",
      "26 ./csv/TimesOfIndia_20170928213304_00015.csv.gz 28\n",
      "27 ./csv/TimesOfIndia_20170928211851_00013.csv.gz 28\n",
      "28 ./csv/TimesOfIndia_20170928212109_00024.csv.gz 29\n",
      "29 ./csv/TimesOfIndia_20170928210516_00025.csv.gz 28\n",
      "30 ./csv/TimesOfIndia_20170928223232_00015.csv.gz 29\n",
      "31 ./csv/TimesOfIndia_20170928213830_00032.csv.gz 29\n",
      "32 ./csv/TimesOfIndia_20170928215446_00036.csv.gz 29\n",
      "33 ./csv/TimesOfIndia_20170928220634_00026.csv.gz 29\n",
      "34 ./csv/TimesOfIndia_20170928215652_00041.csv.gz 29\n",
      "35 ./csv/TimesOfIndia_20170928210153_00010.csv.gz 28\n",
      "36 ./csv/TimesOfIndia_20170928222040_00025.csv.gz 28\n",
      "37 ./csv/TimesOfIndia_20170928215021_00021.csv.gz 28\n",
      "38 ./csv/TimesOfIndia_20170928220323_00017.csv.gz 29\n",
      "39 ./csv/TimesOfIndia_20170928212319_00030.csv.gz 28\n",
      "40 ./csv/TimesOfIndia_20170928221055_00040.csv.gz 28\n",
      "41 ./csv/TimesOfIndia_20170928212840_00001.csv.gz 29\n",
      "42 ./csv/TimesOfIndia_20170928221515_00008.csv.gz 29\n",
      "43 ./csv/TimesOfIndia_20170928221202_00044.csv.gz 29\n",
      "44 ./csv/TimesOfIndia_20170928223443_00022.csv.gz 29\n",
      "45 ./csv/TimesOfIndia_20170928222147_00029.csv.gz 28\n",
      "46 ./csv/TimesOfIndia_20170928214500_00005.csv.gz 28\n",
      "47 ./csv/TimesOfIndia_20170928220007_00007.csv.gz 29\n",
      "48 ./csv/TimesOfIndia_20170928220631_00024.csv.gz 28\n",
      "49 ./csv/TimesOfIndia_20170928213620_00026.csv.gz 28\n",
      "50 ./csv/TimesOfIndia_20170928215340_00032.csv.gz 29\n",
      "51 ./csv/TimesOfIndia_20170928212737_00045.csv.gz 29\n",
      "52 ./csv/TimesOfIndia_20170928222037_00024.csv.gz 29\n",
      "53 ./csv/TimesOfIndia_20170928223024_00008.csv.gz 29\n",
      "54 ./csv/TimesOfIndia_20170928214812_00015.csv.gz 28\n",
      "55 ./csv/TimesOfIndia_20170928215858_00002.csv.gz 29\n",
      "56 ./csv/TimesOfIndia_20170928220424_00018.csv.gz 29\n",
      "57 ./csv/TimesOfIndia_20170928220633_00025.csv.gz 29\n",
      "58 ./csv/TimesOfIndia_20170928211436_00004.csv.gz 28\n",
      "59 ./csv/TimesOfIndia_20170928223651_00028.csv.gz 29\n",
      "60 ./csv/TimesOfIndia_20170928223756_00031.csv.gz 28\n",
      "61 ./csv/TimesOfIndia_20170928221827_00017.csv.gz 28\n",
      "62 ./csv/TimesOfIndia_20170928213412_00020.csv.gz 28\n",
      "63 ./csv/TimesOfIndia_20170928221054_00039.csv.gz 29\n",
      "64 ./csv/TimesOfIndia_20170928212421_00032.csv.gz 28\n",
      "65 ./csv/TimesOfIndia_20170928213156_00009.csv.gz 29\n",
      "66 ./csv/TimesOfIndia_20170928215342_00033.csv.gz 29\n",
      "67 ./csv/TimesOfIndia_20170928220846_00033.csv.gz 29\n",
      "68 ./csv/TimesOfIndia_20170928213154_00008.csv.gz 29\n",
      "69 ./csv/TimesOfIndia_20170928221932_00019.csv.gz 29\n",
      "70 ./csv/TimesOfIndia_20170928213831_00033.csv.gz 29\n",
      "71 ./csv/TimesOfIndia_20170928205944_00004.csv.gz 28\n",
      "72 ./csv/TimesOfIndia_20170928211900_00019.csv.gz 28\n",
      "73 ./csv/TimesOfIndia_20170928210305_00018.csv.gz 29\n",
      "74 ./csv/TimesOfIndia_20170928220111_00010.csv.gz 28\n",
      "75 ./csv/TimesOfIndia_20170928215232_00027.csv.gz 28\n",
      "76 ./csv/TimesOfIndia_20170928210304_00017.csv.gz 28\n",
      "77 ./csv/TimesOfIndia_20170928210200_00015.csv.gz 29\n",
      "78 ./csv/TimesOfIndia_20170928211852_00014.csv.gz 28\n",
      "79 ./csv/TimesOfIndia_20170928211540_00006.csv.gz 28\n",
      "80 ./csv/TimesOfIndia_20170928214146_00044.csv.gz 29\n",
      "81 ./csv/TimesOfIndia_20170928221720_00012.csv.gz 28\n",
      "82 ./csv/TimesOfIndia_20170928222250_00031.csv.gz 29\n",
      "83 ./csv/TimesOfIndia_20170928220108_00008.csv.gz 28\n",
      "84 ./csv/TimesOfIndia_20170928211644_00008.csv.gz 28\n",
      "85 ./csv/TimesOfIndia_20170928214558_00004.csv.gz 29\n",
      "86 ./csv/TimesOfIndia_20170928212422_00033.csv.gz 28\n",
      "87 ./csv/TimesOfIndia_20170928212842_00048.csv.gz 28\n",
      "88 ./csv/TimesOfIndia_20170928222814_00001.csv.gz 29\n",
      "89 ./csv/TimesOfIndia_20170928220739_00029.csv.gz 29\n",
      "90 ./csv/TimesOfIndia_20170928205945_00005.csv.gz 29\n",
      "91 ./csv/TimesOfIndia_20170928222400_00037.csv.gz 29\n",
      "92 ./csv/TimesOfIndia_20170928220426_00020.csv.gz 28\n",
      "93 ./csv/TimesOfIndia_20170928210308_00019.csv.gz 29\n",
      "94 ./csv/TimesOfIndia_20170928223336_00018.csv.gz 29\n",
      "95 ./csv/TimesOfIndia_20170928211541_00007.csv.gz 28\n",
      "96 ./csv/TimesOfIndia_20170928211645_00009.csv.gz 28\n",
      "97 ./csv/TimesOfIndia_20170928215023_00022.csv.gz 29\n",
      "98 ./csv/TimesOfIndia_20170928223233_00016.csv.gz 29\n",
      "99 ./csv/TimesOfIndia_20170928220740_00030.csv.gz 29\n",
      "100 ./csv/TimesOfIndia_20170928211747_00011.csv.gz 28\n",
      "101 ./csv/TimesOfIndia_20170928223129_00012.csv.gz 29\n",
      "102 ./csv/TimesOfIndia_20170928221935_00022.csv.gz 29\n",
      "103 ./csv/TimesOfIndia_20170928210410_00020.csv.gz 28\n",
      "104 ./csv/TimesOfIndia_20170928212214_00027.csv.gz 28\n",
      "105 ./csv/TimesOfIndia_20170928214042_00041.csv.gz 29\n",
      "106 ./csv/TimesOfIndia_20170928222508_00043.csv.gz 29\n",
      "107 ./csv/TimesOfIndia_20170928223022_00007.csv.gz 29\n",
      "108 ./csv/TimesOfIndia_20170928212843_00002.csv.gz 28\n",
      "109 ./csv/TimesOfIndia_20170928214709_00013.csv.gz 29\n",
      "110 ./csv/TimesOfIndia_20170928214706_00010.csv.gz 28\n",
      "111 ./csv/TimesOfIndia_20170928213938_00038.csv.gz 28\n",
      "112 ./csv/TimesOfIndia_20170928223442_00021.csv.gz 29\n",
      "113 ./csv/TimesOfIndia_20170928210619_00027.csv.gz 28\n",
      "114 ./csv/TimesOfIndia_20170928212111_00025.csv.gz 28\n",
      "115 ./csv/TimesOfIndia_20170928222356_00034.csv.gz 29\n",
      "116 ./csv/TimesOfIndia_20170928214604_00008.csv.gz 29\n",
      "117 ./csv/TimesOfIndia_20170928212215_00028.csv.gz 28\n",
      "118 ./csv/TimesOfIndia_20170928214708_00012.csv.gz 28\n",
      "119 ./csv/TimesOfIndia_20170928212320_00031.csv.gz 28\n",
      "120 ./csv/TimesOfIndia_20170928215234_00029.csv.gz 28\n",
      "121 ./csv/TimesOfIndia_20170928212945_00049.csv.gz 29\n",
      "122 ./csv/TimesOfIndia_20170928223545_00024.csv.gz 29\n",
      "123 ./csv/TimesOfIndia_20170928210202_00016.csv.gz 28\n",
      "124 ./csv/TimesOfIndia_20170928215237_00030.csv.gz 29\n",
      "125 ./csv/TimesOfIndia_20170928220737_00027.csv.gz 28\n",
      "126 ./csv/TimesOfIndia_20170928215653_00042.csv.gz 29\n",
      "127 ./csv/TimesOfIndia_20170928212005_00022.csv.gz 28\n",
      "128 ./csv/TimesOfIndia_20170928221514_00007.csv.gz 29\n",
      "129 ./csv/TimesOfIndia_20170928214919_00019.csv.gz 29\n",
      "130 ./csv/TimesOfIndia_20170928215339_00031.csv.gz 28\n",
      "131 ./csv/TimesOfIndia_20170928210514_00024.csv.gz 29\n",
      "132 ./csv/TimesOfIndia_20170928221409_00004.csv.gz 28\n",
      "133 ./csv/TimesOfIndia_20170928214602_00007.csv.gz 28\n",
      "134 ./csv/TimesOfIndia_20170928213259_00012.csv.gz 29\n",
      "135 ./csv/TimesOfIndia_20170928215959_00003.csv.gz 29\n",
      "136 ./csv/TimesOfIndia_20170928220215_00013.csv.gz 29\n",
      "137 ./csv/TimesOfIndia_20170928212632_00041.csv.gz 28\n",
      "138 ./csv/TimesOfIndia_20170928221157_00042.csv.gz 28\n",
      "139 ./csv/TimesOfIndia_20170928213411_00019.csv.gz 29\n",
      "140 ./csv/TimesOfIndia_20170928220216_00014.csv.gz 29\n",
      "141 ./csv/TimesOfIndia_20170928222815_00002.csv.gz 29\n",
      "142 ./csv/TimesOfIndia_20170928223649_00027.csv.gz 29\n",
      "143 ./csv/TimesOfIndia_20170928212947_00004.csv.gz 29\n",
      "144 ./csv/TimesOfIndia_20170928222507_00042.csv.gz 28\n",
      "145 ./csv/TimesOfIndia_20170928221933_00020.csv.gz 28\n",
      "146 ./csv/TimesOfIndia_20170928223130_00013.csv.gz 29\n",
      "147 ./csv/TimesOfIndia_20170928213517_00024.csv.gz 29\n",
      "148 ./csv/TimesOfIndia_20170928214707_00011.csv.gz 29\n",
      "149 ./csv/TimesOfIndia_20170928222143_00027.csv.gz 29\n",
      "150 ./csv/TimesOfIndia_20170928220948_00035.csv.gz 28\n",
      "151 ./csv/TimesOfIndia_20170928223804_00036.csv.gz 29\n",
      "152 ./csv/TimesOfIndia_20170928212630_00039.csv.gz 28\n",
      "153 ./csv/TimesOfIndia_20170928220844_00032.csv.gz 29\n",
      "154 ./csv/TimesOfIndia_20170928211958_00017.csv.gz 29\n",
      "155 ./csv/TimesOfIndia_20170928215445_00035.csv.gz 29\n",
      "156 ./csv/TimesOfIndia_20170928205842_00003.csv.gz 29\n",
      "157 ./csv/TimesOfIndia_20170928223026_00010.csv.gz 29\n",
      "158 ./csv/TimesOfIndia_20170928214251_00047.csv.gz 28\n",
      "159 ./csv/TimesOfIndia_20170928222357_00035.csv.gz 29\n",
      "160 ./csv/TimesOfIndia_20170928211959_00018.csv.gz 29\n",
      "161 ./csv/TimesOfIndia_20170928222252_00032.csv.gz 29\n",
      "162 ./csv/TimesOfIndia_20170928221826_00016.csv.gz 29\n",
      "163 ./csv/TimesOfIndia_20170928214605_00009.csv.gz 29\n",
      "164 ./csv/TimesOfIndia_20170928213157_00010.csv.gz 29\n",
      "165 ./csv/TimesOfIndia_20170928213726_00030.csv.gz 29\n",
      "166 ./csv/TimesOfIndia_20170928210259_00014.csv.gz 28\n",
      "167 ./csv/TimesOfIndia_20170928210052_00009.csv.gz 28\n",
      "168 ./csv/TimesOfIndia_20170928212529_00038.csv.gz 28\n",
      "169 ./csv/TimesOfIndia_20170928210047_00007.csv.gz 28\n",
      "170 ./csv/TimesOfIndia_20170928222505_00041.csv.gz 28\n",
      "171 ./csv/TimesOfIndia_20170928214814_00016.csv.gz 28\n",
      "172 ./csv/TimesOfIndia_20170928212838_00046.csv.gz 28\n",
      "173 ./csv/TimesOfIndia_20170928221725_00015.csv.gz 28\n",
      "174 ./csv/TimesOfIndia_20170928223652_00029.csv.gz 28\n",
      "175 ./csv/TimesOfIndia_20170928215551_00040.csv.gz 28\n",
      "176 ./csv/TimesOfIndia_20170928214145_00043.csv.gz 28\n",
      "177 ./csv/TimesOfIndia_20170928213158_00011.csv.gz 28\n",
      "178 ./csv/TimesOfIndia_20170928221408_00003.csv.gz 28\n",
      "179 ./csv/TimesOfIndia_20170928221618_00010.csv.gz 29\n",
      "180 ./csv/TimesOfIndia_20170928213514_00022.csv.gz 29\n",
      "181 ./csv/TimesOfIndia_20170928211539_00005.csv.gz 28\n",
      "182 ./csv/TimesOfIndia_20170928222402_00038.csv.gz 29\n",
      "183 ./csv/TimesOfIndia_20170928215900_00004.csv.gz 28\n",
      "184 ./csv/TimesOfIndia_20170928220528_00022.csv.gz 28\n",
      "185 ./csv/TimesOfIndia_20170928213048_00005.csv.gz 28\n",
      "186 ./csv/TimesOfIndia_20170928211853_00015.csv.gz 29\n",
      "187 ./csv/TimesOfIndia_20170928222918_00004.csv.gz 29\n",
      "188 ./csv/TimesOfIndia_20170928210156_00012.csv.gz 29\n",
      "189 ./csv/TimesOfIndia_20170928221617_00009.csv.gz 29\n",
      "190 ./csv/TimesOfIndia_20170928212003_00021.csv.gz 29\n",
      "191 ./csv/TimesOfIndia_20170928210157_00013.csv.gz 28\n",
      "192 ./csv/TimesOfIndia_20170928215233_00028.csv.gz 29\n",
      "193 ./csv/TimesOfIndia_20170928220530_00023.csv.gz 28\n",
      "194 ./csv/TimesOfIndia_20170928220214_00012.csv.gz 29\n",
      "195 ./csv/TimesOfIndia_20170928213053_00007.csv.gz 29\n",
      "196 ./csv/TimesOfIndia_20170928214149_00045.csv.gz 28\n",
      "197 ./csv/TimesOfIndia_20170928212946_00003.csv.gz 28\n",
      "198 ./csv/TimesOfIndia_20170928221721_00013.csv.gz 29\n",
      "199 ./csv/TimesOfIndia_20170928220212_00011.csv.gz 28\n",
      "200 ./csv/TimesOfIndia_20170928223444_00023.csv.gz 29\n",
      "201 ./csv/TimesOfIndia_20170928223547_00025.csv.gz 28\n",
      "202 ./csv/TimesOfIndia_20170928223335_00017.csv.gz 29\n",
      "203 ./csv/TimesOfIndia_20170928212425_00035.csv.gz 28\n",
      "204 ./csv/TimesOfIndia_20170928222403_00039.csv.gz 28\n",
      "205 ./csv/TimesOfIndia_20170928213305_00016.csv.gz 29\n",
      "206 ./csv/TimesOfIndia_20170928205839_00001.csv.gz 28\n",
      "207 ./csv/TimesOfIndia_20170928212736_00044.csv.gz 29\n",
      "208 ./csv/TimesOfIndia_20170928223802_00035.csv.gz 28\n",
      "209 ./csv/TimesOfIndia_20170928213724_00028.csv.gz 29\n",
      "210 ./csv/TimesOfIndia_20170928211748_00012.csv.gz 28\n",
      "211 ./csv/TimesOfIndia_20170928221052_00038.csv.gz 28\n",
      "212 ./csv/TimesOfIndia_20170928220738_00028.csv.gz 28\n",
      "213 ./csv/TimesOfIndia_20170928213833_00034.csv.gz 28\n",
      "214 ./csv/TimesOfIndia_20170928211857_00016.csv.gz 29\n",
      "215 ./csv/TimesOfIndia_20170928215447_00037.csv.gz 29\n",
      "216 ./csv/TimesOfIndia_20170928213052_00006.csv.gz 29\n",
      "217 ./csv/TimesOfIndia_20170928223025_00009.csv.gz 29\n",
      "218 ./csv/TimesOfIndia_20170928212735_00043.csv.gz 28\n",
      "219 ./csv/TimesOfIndia_20170928213622_00027.csv.gz 29\n",
      "220 ./csv/TimesOfIndia_20170928213310_00018.csv.gz 28\n",
      "221 ./csv/TimesOfIndia_20170928220110_00009.csv.gz 29\n",
      "222 ./csv/TimesOfIndia_20170928214456_00002.csv.gz 29\n",
      "223 ./csv/TimesOfIndia_20170928213729_00031.csv.gz 29\n",
      "224 ./csv/TimesOfIndia_20170928211435_00003.csv.gz 28\n",
      "225 ./csv/TimesOfIndia_20170928215550_00039.csv.gz 28\n",
      "226 ./csv/TimesOfIndia_20170928220318_00015.csv.gz 29\n",
      "227 ./csv/TimesOfIndia_20170928223700_00034.csv.gz 29\n",
      "228 ./csv/TimesOfIndia_20170928210051_00008.csv.gz 28\n",
      "229 ./csv/TimesOfIndia_20170928212001_00020.csv.gz 29\n",
      "230 ./csv/TimesOfIndia_20170928213306_00017.csv.gz 29\n",
      "231 ./csv/TimesOfIndia_20170928220001_00005.csv.gz 29\n",
      "232 ./csv/TimesOfIndia_20170928221513_00006.csv.gz 28\n",
      "233 ./csv/TimesOfIndia_20170928220527_00021.csv.gz 29\n",
      "234 ./csv/TimesOfIndia_20170928211332_00001.csv.gz 28\n",
      "235 ./csv/TimesOfIndia_20170928214144_00042.csv.gz 28\n",
      "236 ./csv/TimesOfIndia_20170928221158_00043.csv.gz 28\n",
      "237 ./csv/TimesOfIndia_20170928212527_00037.csv.gz 28\n",
      "238 ./csv/TimesOfIndia_20170928215443_00034.csv.gz 28\n",
      "239 ./csv/TimesOfIndia_20170928220951_00037.csv.gz 28\n",
      "240 ./csv/TimesOfIndia_20170928221934_00021.csv.gz 28\n",
      "241 ./csv/TimesOfIndia_20170928212107_00023.csv.gz 28\n",
      "242 ./csv/TimesOfIndia_20170928220843_00031.csv.gz 29\n",
      "243 ./csv/TimesOfIndia_20170928221305_00001.csv.gz 28\n",
      "244 ./csv/TimesOfIndia_20170928222917_00003.csv.gz 29\n",
      "245 ./csv/TimesOfIndia_20170928210620_00028.csv.gz 28\n",
      "246 ./csv/TimesOfIndia_20170928214457_00003.csv.gz 29\n",
      "247 ./csv/TimesOfIndia_20170928211434_00002.csv.gz 28\n",
      "248 ./csv/TimesOfIndia_20170928220319_00016.csv.gz 29\n",
      "249 ./csv/TimesOfIndia_20170928222355_00033.csv.gz 29\n",
      "250 ./csv/TimesOfIndia_20170928205841_00002.csv.gz 29\n",
      "251 ./csv/TimesOfIndia_20170928212213_00026.csv.gz 28\n",
      "252 ./csv/TimesOfIndia_20170928222036_00023.csv.gz 29\n",
      "253 ./csv/TimesOfIndia_20170928213725_00029.csv.gz 29\n",
      "254 ./csv/TimesOfIndia_20170928222145_00028.csv.gz 28\n",
      "255 ./csv/TimesOfIndia_20170928221510_00005.csv.gz 29\n",
      "256 ./csv/TimesOfIndia_20170928220425_00019.csv.gz 29\n",
      "257 ./csv/TimesOfIndia_20170928221619_00011.csv.gz 28\n",
      "258 ./csv/TimesOfIndia_20170928211646_00010.csv.gz 28\n",
      "259 ./csv/TimesOfIndia_20170928215548_00038.csv.gz 28\n",
      "260 ./csv/TimesOfIndia_20170928214501_00006.csv.gz 29\n",
      "261 ./csv/TimesOfIndia_20170928215128_00025.csv.gz 28\n",
      "262 ./csv/TimesOfIndia_20170928220947_00034.csv.gz 28\n",
      "263 ./csv/TimesOfIndia_20170928214917_00017.csv.gz 28\n",
      "264 ./csv/TimesOfIndia_20170928220006_00006.csv.gz 29\n",
      "265 ./csv/TimesOfIndia_20170928223757_00032.csv.gz 29\n",
      "266 ./csv/TimesOfIndia_20170928215129_00026.csv.gz 29\n",
      "267 ./csv/TimesOfIndia_20170928221830_00018.csv.gz 29\n",
      "268 ./csv/TimesOfIndia_20170928223441_00020.csv.gz 29\n",
      "269 ./csv/TimesOfIndia_20170928223758_00033.csv.gz 29\n",
      "270 ./csv/TimesOfIndia_20170928210513_00023.csv.gz 29\n",
      "271 ./csv/TimesOfIndia_20170928215126_00024.csv.gz 28\n",
      "272 ./csv/TimesOfIndia_20170928205946_00006.csv.gz 28\n",
      "273 ./csv/TimesOfIndia_20170928212526_00036.csv.gz 28\n",
      "274 ./csv/TimesOfIndia_20170928212634_00042.csv.gz 29\n",
      "275 ./csv/TimesOfIndia_20170928210154_00011.csv.gz 28\n",
      "276 ./csv/TimesOfIndia_20170928214354_00001.csv.gz 29\n",
      "277 ./csv/TimesOfIndia_20170928222249_00030.csv.gz 29\n",
      "278 ./csv/TimesOfIndia_20170928215856_00001.csv.gz 29\n",
      "279 ./csv/TimesOfIndia_20170928214920_00020.csv.gz 29\n",
      "280 ./csv/TimesOfIndia_20170928223231_00014.csv.gz 29\n",
      "281 ./csv/TimesOfIndia_20170928212318_00029.csv.gz 28\n",
      "282 ./csv/TimesOfIndia_20170928212631_00040.csv.gz 28\n",
      "283 ./csv/TimesOfIndia_20170928213934_00035.csv.gz 29\n",
      "284 ./csv/TimesOfIndia_20170928212424_00034.csv.gz 28\n",
      "285 ./csv/TimesOfIndia_20170928214918_00018.csv.gz 29\n",
      "286 ./csv/TimesOfIndia_20170928213200_00013.csv.gz 28\n",
      "287 ./csv/TimesOfIndia_20170928213937_00037.csv.gz 28\n",
      "288 ./csv/TimesOfIndia_20170928220950_00036.csv.gz 29\n",
      "289 ./csv/TimesOfIndia_20170928210411_00021.csv.gz 29\n",
      "290 ./csv/TimesOfIndia_20170928223548_00026.csv.gz 29\n"
     ]
    }
   ],
   "source": [
    "cols = set()\n",
    "for i, fn in enumerate(glob('./data/*.zip')):\n",
    "    with ZipFile(fn, 'r') as zo:\n",
    "        # Get list of files names in zip\n",
    "        files = zo.namelist()\n",
    "        rows = []\n",
    "        for zfn in files:\n",
    "            with zo.open(zfn) as f:\n",
    "                d = xmltodict.parse(f)\n",
    "                rows.append(d['Record'])\n",
    "        df = pd.DataFrame(pd.json_normalize(rows))\n",
    "        ofn = fn.replace('data', 'csv').replace('zip', 'csv.gz')\n",
    "        cols.update(df.columns)\n",
    "        df.to_csv(ofn, index=False, compression='gzip')\n",
    "        print(i, ofn, len(df.columns))\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3f74eef9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29,\n",
       " {'Abstract',\n",
       "  'ActionCode',\n",
       "  'AlphaPubDate',\n",
       "  'Contributor',\n",
       "  'Contributor.ContribRole',\n",
       "  'Contributor.FirstName',\n",
       "  'Contributor.LastName',\n",
       "  'Contributor.MiddleName',\n",
       "  'Contributor.NameSuffix',\n",
       "  'Contributor.OrganizationName',\n",
       "  'Contributor.OriginalForm',\n",
       "  'Contributor.PersonName',\n",
       "  'Contributor.PersonTitle',\n",
       "  'DateTimeStamp',\n",
       "  'FullText',\n",
       "  'LanguageCode',\n",
       "  'NumericPubDate',\n",
       "  'ObjectType',\n",
       "  'Products.Product',\n",
       "  'Publication.PublicationID',\n",
       "  'Publication.Qualifier',\n",
       "  'Publication.Title',\n",
       "  'Publisher',\n",
       "  'RecordID',\n",
       "  'RecordTitle',\n",
       "  'SourceType',\n",
       "  'StartPage',\n",
       "  'URLDocView',\n",
       "  'Version'})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show list of columns from all zip files\n",
    "len(cols), cols"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "434399f6",
   "metadata": {},
   "source": [
    "### Step 2: Merge all csv files to a csv file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "0d5c31d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 csv/TimesOfIndia_20170928220631_00024.csv.gz 29\n",
      "1 csv/TimesOfIndia_20170928215232_00027.csv.gz 29\n",
      "2 csv/TimesOfIndia_20170928210052_00009.csv.gz 29\n",
      "3 csv/TimesOfIndia_20170928213935_00036.csv.gz 29\n",
      "4 csv/TimesOfIndia_20170928214811_00014.csv.gz 29\n",
      "5 csv/TimesOfIndia_20170928213156_00009.csv.gz 29\n",
      "6 csv/TimesOfIndia_20170928220633_00025.csv.gz 29\n",
      "7 csv/TimesOfIndia_20170928223548_00026.csv.gz 29\n",
      "8 csv/TimesOfIndia_20170928221409_00004.csv.gz 29\n",
      "9 csv/TimesOfIndia_20170928210047_00007.csv.gz 29\n",
      "10 csv/TimesOfIndia_20170928213154_00008.csv.gz 29\n",
      "11 csv/TimesOfIndia_20170928223652_00029.csv.gz 29\n",
      "12 csv/TimesOfIndia_20170928213306_00017.csv.gz 29\n",
      "13 csv/TimesOfIndia_20170928213259_00012.csv.gz 29\n",
      "14 csv/TimesOfIndia_20170928221933_00020.csv.gz 29\n",
      "15 csv/TimesOfIndia_20170928212320_00031.csv.gz 29\n",
      "16 csv/TimesOfIndia_20170928215234_00029.csv.gz 29\n",
      "17 csv/TimesOfIndia_20170928210259_00014.csv.gz 29\n",
      "18 csv/TimesOfIndia_20170928212947_00004.csv.gz 29\n",
      "19 csv/TimesOfIndia_20170928213304_00015.csv.gz 29\n",
      "20 csv/TimesOfIndia_20170928212422_00033.csv.gz 29\n",
      "21 csv/TimesOfIndia_20170928212843_00002.csv.gz 29\n",
      "22 csv/TimesOfIndia_20170928211900_00019.csv.gz 29\n",
      "23 csv/TimesOfIndia_20170928223022_00007.csv.gz 29\n",
      "24 csv/TimesOfIndia_20170928222357_00035.csv.gz 29\n",
      "25 csv/TimesOfIndia_20170928210304_00017.csv.gz 29\n",
      "26 csv/TimesOfIndia_20170928215653_00042.csv.gz 29\n",
      "27 csv/TimesOfIndia_20170928223758_00033.csv.gz 29\n",
      "28 csv/TimesOfIndia_20170928211853_00015.csv.gz 29\n",
      "29 csv/TimesOfIndia_20170928223340_00019.csv.gz 29\n",
      "30 csv/TimesOfIndia_20170928223757_00032.csv.gz 29\n",
      "31 csv/TimesOfIndia_20170928223545_00024.csv.gz 29\n",
      "32 csv/TimesOfIndia_20170928214707_00011.csv.gz 29\n",
      "33 csv/TimesOfIndia_20170928223129_00012.csv.gz 29\n",
      "34 csv/TimesOfIndia_20170928222147_00029.csv.gz 29\n",
      "35 csv/TimesOfIndia_20170928213833_00034.csv.gz 29\n",
      "36 csv/TimesOfIndia_20170928221052_00038.csv.gz 29\n",
      "37 csv/TimesOfIndia_20170928214042_00041.csv.gz 29\n",
      "38 csv/TimesOfIndia_20170928215445_00035.csv.gz 29\n",
      "39 csv/TimesOfIndia_20170928223336_00018.csv.gz 29\n",
      "40 csv/TimesOfIndia_20170928212529_00038.csv.gz 29\n",
      "41 csv/TimesOfIndia_20170928215443_00034.csv.gz 29\n",
      "42 csv/TimesOfIndia_20170928215233_00028.csv.gz 29\n",
      "43 csv/TimesOfIndia_20170928222917_00003.csv.gz 29\n",
      "44 csv/TimesOfIndia_20170928211645_00009.csv.gz 29\n",
      "45 csv/TimesOfIndia_20170928214814_00016.csv.gz 29\n",
      "46 csv/TimesOfIndia_20170928213157_00010.csv.gz 29\n",
      "47 csv/TimesOfIndia_20170928210202_00016.csv.gz 29\n",
      "48 csv/TimesOfIndia_20170928211748_00012.csv.gz 29\n",
      "49 csv/TimesOfIndia_20170928213200_00013.csv.gz 29\n",
      "50 csv/TimesOfIndia_20170928221618_00010.csv.gz 29\n",
      "51 csv/TimesOfIndia_20170928214917_00017.csv.gz 29\n",
      "52 csv/TimesOfIndia_20170928213411_00019.csv.gz 29\n",
      "53 csv/TimesOfIndia_20170928212526_00036.csv.gz 29\n",
      "54 csv/TimesOfIndia_20170928214706_00010.csv.gz 29\n",
      "55 csv/TimesOfIndia_20170928223802_00035.csv.gz 29\n",
      "56 csv/TimesOfIndia_20170928220001_00005.csv.gz 29\n",
      "57 csv/TimesOfIndia_20170928213726_00030.csv.gz 29\n",
      "58 csv/TimesOfIndia_20170928210619_00027.csv.gz 29\n",
      "59 csv/TimesOfIndia_20170928220426_00020.csv.gz 29\n",
      "60 csv/TimesOfIndia_20170928223025_00009.csv.gz 29\n",
      "61 csv/TimesOfIndia_20170928221157_00042.csv.gz 29\n",
      "62 csv/TimesOfIndia_20170928210308_00019.csv.gz 29\n",
      "63 csv/TimesOfIndia_20170928220216_00014.csv.gz 29\n",
      "64 csv/TimesOfIndia_20170928222814_00001.csv.gz 29\n",
      "65 csv/TimesOfIndia_20170928210516_00025.csv.gz 29\n",
      "66 csv/TimesOfIndia_20170928220950_00036.csv.gz 29\n",
      "67 csv/TimesOfIndia_20170928221514_00007.csv.gz 29\n",
      "68 csv/TimesOfIndia_20170928215654_00043.csv.gz 29\n",
      "69 csv/TimesOfIndia_20170928210514_00024.csv.gz 29\n",
      "70 csv/TimesOfIndia_20170928213514_00022.csv.gz 29\n",
      "71 csv/TimesOfIndia_20170928214456_00002.csv.gz 29\n",
      "72 csv/TimesOfIndia_20170928221721_00013.csv.gz 29\n",
      "73 csv/TimesOfIndia_20170928211541_00007.csv.gz 29\n",
      "74 csv/TimesOfIndia_20170928213938_00038.csv.gz 29\n",
      "75 csv/TimesOfIndia_20170928214145_00043.csv.gz 29\n",
      "76 csv/TimesOfIndia_20170928210513_00023.csv.gz 29\n",
      "77 csv/TimesOfIndia_20170928212005_00022.csv.gz 29\n",
      "78 csv/TimesOfIndia_20170928221720_00012.csv.gz 29\n",
      "79 csv/TimesOfIndia_20170928211851_00013.csv.gz 29\n",
      "80 csv/TimesOfIndia_20170928215339_00031.csv.gz 29\n",
      "81 csv/TimesOfIndia_20170928220323_00017.csv.gz 29\n",
      "82 csv/TimesOfIndia_20170928215550_00039.csv.gz 29\n",
      "83 csv/TimesOfIndia_20170928213619_00025.csv.gz 29\n",
      "84 csv/TimesOfIndia_20170928222403_00039.csv.gz 29\n",
      "85 csv/TimesOfIndia_20170928214812_00015.csv.gz 29\n",
      "86 csv/TimesOfIndia_20170928220530_00023.csv.gz 29\n",
      "87 csv/TimesOfIndia_20170928223804_00036.csv.gz 29\n",
      "88 csv/TimesOfIndia_20170928221202_00044.csv.gz 29\n",
      "89 csv/TimesOfIndia_20170928220739_00029.csv.gz 29\n",
      "90 csv/TimesOfIndia_20170928221934_00021.csv.gz 29\n",
      "91 csv/TimesOfIndia_20170928210518_00026.csv.gz 29\n",
      "92 csv/TimesOfIndia_20170928211959_00018.csv.gz 29\n",
      "93 csv/TimesOfIndia_20170928221158_00043.csv.gz 29\n",
      "94 csv/TimesOfIndia_20170928211436_00004.csv.gz 29\n",
      "95 csv/TimesOfIndia_20170928213413_00021.csv.gz 29\n",
      "96 csv/TimesOfIndia_20170928214918_00018.csv.gz 29\n",
      "97 csv/TimesOfIndia_20170928220215_00013.csv.gz 29\n",
      "98 csv/TimesOfIndia_20170928221054_00039.csv.gz 29\n",
      "99 csv/TimesOfIndia_20170928213725_00029.csv.gz 29\n",
      "100 csv/TimesOfIndia_20170928210620_00028.csv.gz 29\n",
      "101 csv/TimesOfIndia_20170928220846_00033.csv.gz 29\n",
      "102 csv/TimesOfIndia_20170928223443_00022.csv.gz 29\n",
      "103 csv/TimesOfIndia_20170928211434_00002.csv.gz 29\n",
      "104 csv/TimesOfIndia_20170928213517_00024.csv.gz 29\n",
      "105 csv/TimesOfIndia_20170928213305_00016.csv.gz 29\n",
      "106 csv/TimesOfIndia_20170928215023_00022.csv.gz 29\n",
      "107 csv/TimesOfIndia_20170928214501_00006.csv.gz 29\n",
      "108 csv/TimesOfIndia_20170928211857_00016.csv.gz 29\n",
      "109 csv/TimesOfIndia_20170928220110_00009.csv.gz 29\n",
      "110 csv/TimesOfIndia_20170928215340_00032.csv.gz 29\n",
      "111 csv/TimesOfIndia_20170928214500_00005.csv.gz 29\n",
      "112 csv/TimesOfIndia_20170928212424_00034.csv.gz 29\n",
      "113 csv/TimesOfIndia_20170928212001_00020.csv.gz 29\n",
      "114 csv/TimesOfIndia_20170928223547_00025.csv.gz 29\n",
      "115 csv/TimesOfIndia_20170928220007_00007.csv.gz 29\n",
      "116 csv/TimesOfIndia_20170928212003_00021.csv.gz 29\n",
      "117 csv/TimesOfIndia_20170928220737_00027.csv.gz 29\n",
      "118 csv/TimesOfIndia_20170928212318_00029.csv.gz 29\n",
      "119 csv/TimesOfIndia_20170928212213_00026.csv.gz 29\n",
      "120 csv/TimesOfIndia_20170928214041_00040.csv.gz 29\n",
      "121 csv/TimesOfIndia_20170928222919_00005.csv.gz 29\n",
      "122 csv/TimesOfIndia_20170928223335_00017.csv.gz 29\n",
      "123 csv/TimesOfIndia_20170928213831_00033.csv.gz 29\n",
      "124 csv/TimesOfIndia_20170928221619_00011.csv.gz 29\n",
      "125 csv/TimesOfIndia_20170928210154_00011.csv.gz 29\n",
      "126 csv/TimesOfIndia_20170928214604_00008.csv.gz 29\n",
      "127 csv/TimesOfIndia_20170928220425_00019.csv.gz 29\n",
      "128 csv/TimesOfIndia_20170928210305_00018.csv.gz 29\n",
      "129 csv/TimesOfIndia_20170928212111_00025.csv.gz 29\n",
      "130 csv/TimesOfIndia_20170928212319_00030.csv.gz 29\n",
      "131 csv/TimesOfIndia_20170928210153_00010.csv.gz 29\n",
      "132 csv/TimesOfIndia_20170928214602_00007.csv.gz 29\n",
      "133 csv/TimesOfIndia_20170928213934_00035.csv.gz 29\n",
      "134 csv/TimesOfIndia_20170928213937_00037.csv.gz 29\n",
      "135 csv/TimesOfIndia_20170928205841_00002.csv.gz 29\n",
      "136 csv/TimesOfIndia_20170928222359_00036.csv.gz 29\n",
      "137 csv/TimesOfIndia_20170928222609_00044.csv.gz 29\n",
      "138 csv/TimesOfIndia_20170928214039_00039.csv.gz 29\n",
      "139 csv/TimesOfIndia_20170928213158_00011.csv.gz 29\n",
      "140 csv/TimesOfIndia_20170928220214_00012.csv.gz 29\n",
      "141 csv/TimesOfIndia_20170928222249_00030.csv.gz 29\n",
      "142 csv/TimesOfIndia_20170928210157_00013.csv.gz 29\n",
      "143 csv/TimesOfIndia_20170928220947_00034.csv.gz 29\n",
      "144 csv/TimesOfIndia_20170928222141_00026.csv.gz 29\n",
      "145 csv/TimesOfIndia_20170928221827_00017.csv.gz 29\n",
      "146 csv/TimesOfIndia_20170928212946_00003.csv.gz 29\n",
      "147 csv/TimesOfIndia_20170928213310_00018.csv.gz 29\n",
      "148 csv/TimesOfIndia_20170928213724_00028.csv.gz 29\n",
      "149 csv/TimesOfIndia_20170928212840_00001.csv.gz 29\n",
      "150 csv/TimesOfIndia_20170928214558_00004.csv.gz 29\n",
      "151 csv/TimesOfIndia_20170928215959_00003.csv.gz 29\n",
      "152 csv/TimesOfIndia_20170928222400_00037.csv.gz 29\n",
      "153 csv/TimesOfIndia_20170928215126_00024.csv.gz 29\n",
      "154 csv/TimesOfIndia_20170928213830_00032.csv.gz 29\n",
      "155 csv/TimesOfIndia_20170928212527_00037.csv.gz 29\n",
      "156 csv/TimesOfIndia_20170928211644_00008.csv.gz 29\n",
      "157 csv/TimesOfIndia_20170928221617_00009.csv.gz 29\n",
      "158 csv/TimesOfIndia_20170928212631_00040.csv.gz 29\n",
      "159 csv/TimesOfIndia_20170928215342_00033.csv.gz 29\n",
      "160 csv/TimesOfIndia_20170928221826_00016.csv.gz 29\n",
      "161 csv/TimesOfIndia_20170928222505_00041.csv.gz 29\n",
      "162 csv/TimesOfIndia_20170928220319_00016.csv.gz 29\n",
      "163 csv/TimesOfIndia_20170928215551_00040.csv.gz 29\n",
      "164 csv/TimesOfIndia_20170928205945_00005.csv.gz 29\n",
      "165 csv/TimesOfIndia_20170928222250_00031.csv.gz 29\n",
      "166 csv/TimesOfIndia_20170928214920_00020.csv.gz 29\n",
      "167 csv/TimesOfIndia_20170928220843_00031.csv.gz 29\n",
      "168 csv/TimesOfIndia_20170928222037_00024.csv.gz 29\n",
      "169 csv/TimesOfIndia_20170928214251_00047.csv.gz 29\n",
      "170 csv/TimesOfIndia_20170928214709_00013.csv.gz 29\n",
      "171 csv/TimesOfIndia_20170928212215_00028.csv.gz 29\n",
      "172 csv/TimesOfIndia_20170928214149_00045.csv.gz 29\n",
      "173 csv/TimesOfIndia_20170928222920_00006.csv.gz 29\n",
      "174 csv/TimesOfIndia_20170928213202_00014.csv.gz 29\n",
      "175 csv/TimesOfIndia_20170928222815_00002.csv.gz 29\n",
      "176 csv/TimesOfIndia_20170928205946_00006.csv.gz 29\n",
      "177 csv/TimesOfIndia_20170928215024_00023.csv.gz 29\n",
      "178 csv/TimesOfIndia_20170928220527_00021.csv.gz 29\n",
      "179 csv/TimesOfIndia_20170928220528_00022.csv.gz 29\n",
      "180 csv/TimesOfIndia_20170928212842_00048.csv.gz 29\n",
      "181 csv/TimesOfIndia_20170928212736_00044.csv.gz 29\n",
      "182 csv/TimesOfIndia_20170928221932_00019.csv.gz 29\n",
      "183 csv/TimesOfIndia_20170928220844_00032.csv.gz 29\n",
      "184 csv/TimesOfIndia_20170928205944_00004.csv.gz 29\n",
      "185 csv/TimesOfIndia_20170928220951_00037.csv.gz 29\n",
      "186 csv/TimesOfIndia_20170928214708_00012.csv.gz 29\n",
      "187 csv/TimesOfIndia_20170928214250_00046.csv.gz 29\n",
      "188 csv/TimesOfIndia_20170928223441_00020.csv.gz 29\n",
      "189 csv/TimesOfIndia_20170928223700_00034.csv.gz 29\n",
      "190 csv/TimesOfIndia_20170928221156_00041.csv.gz 29\n",
      "191 csv/TimesOfIndia_20170928222402_00038.csv.gz 29\n",
      "192 csv/TimesOfIndia_20170928221305_00001.csv.gz 29\n",
      "193 csv/TimesOfIndia_20170928223756_00031.csv.gz 29\n",
      "194 csv/TimesOfIndia_20170928210411_00021.csv.gz 29\n",
      "195 csv/TimesOfIndia_20170928210410_00020.csv.gz 29\n",
      "196 csv/TimesOfIndia_20170928221407_00002.csv.gz 29\n",
      "197 csv/TimesOfIndia_20170928222252_00032.csv.gz 29\n",
      "198 csv/TimesOfIndia_20170928222145_00028.csv.gz 29\n",
      "199 csv/TimesOfIndia_20170928221510_00005.csv.gz 29\n",
      "200 csv/TimesOfIndia_20170928212838_00046.csv.gz 29\n",
      "201 csv/TimesOfIndia_20170928213729_00031.csv.gz 29\n",
      "202 csv/TimesOfIndia_20170928215129_00026.csv.gz 29\n",
      "203 csv/TimesOfIndia_20170928211539_00005.csv.gz 29\n",
      "204 csv/TimesOfIndia_20170928214354_00001.csv.gz 29\n",
      "205 csv/TimesOfIndia_20170928222143_00027.csv.gz 29\n",
      "206 csv/TimesOfIndia_20170928223649_00027.csv.gz 29\n",
      "207 csv/TimesOfIndia_20170928221725_00015.csv.gz 29\n",
      "208 csv/TimesOfIndia_20170928223128_00011.csv.gz 29\n",
      "209 csv/TimesOfIndia_20170928222610_00045.csv.gz 29\n",
      "210 csv/TimesOfIndia_20170928220212_00011.csv.gz 29\n",
      "211 csv/TimesOfIndia_20170928215652_00041.csv.gz 29\n",
      "212 csv/TimesOfIndia_20170928220634_00026.csv.gz 29\n",
      "213 csv/TimesOfIndia_20170928212109_00024.csv.gz 29\n",
      "214 csv/TimesOfIndia_20170928221935_00022.csv.gz 29\n",
      "215 csv/TimesOfIndia_20170928221723_00014.csv.gz 29\n",
      "216 csv/TimesOfIndia_20170928214919_00019.csv.gz 29\n",
      "217 csv/TimesOfIndia_20170928210200_00015.csv.gz 29\n",
      "218 csv/TimesOfIndia_20170928223231_00014.csv.gz 29\n",
      "219 csv/TimesOfIndia_20170928223026_00010.csv.gz 29\n",
      "220 csv/TimesOfIndia_20170928212214_00027.csv.gz 29\n",
      "221 csv/TimesOfIndia_20170928220738_00028.csv.gz 29\n",
      "222 csv/TimesOfIndia_20170928215021_00021.csv.gz 29\n",
      "223 csv/TimesOfIndia_20170928222355_00033.csv.gz 29\n",
      "224 csv/TimesOfIndia_20170928222504_00040.csv.gz 29\n",
      "225 csv/TimesOfIndia_20170928221513_00006.csv.gz 29\n",
      "226 csv/TimesOfIndia_20170928212107_00023.csv.gz 29\n",
      "227 csv/TimesOfIndia_20170928220318_00015.csv.gz 29\n",
      "228 csv/TimesOfIndia_20170928205842_00003.csv.gz 29\n",
      "229 csv/TimesOfIndia_20170928212841_00047.csv.gz 29\n",
      "230 csv/TimesOfIndia_20170928212632_00041.csv.gz 29\n",
      "231 csv/TimesOfIndia_20170928210051_00008.csv.gz 29\n",
      "232 csv/TimesOfIndia_20170928212425_00035.csv.gz 29\n",
      "233 csv/TimesOfIndia_20170928223651_00028.csv.gz 29\n",
      "234 csv/TimesOfIndia_20170928212634_00042.csv.gz 29\n",
      "235 csv/TimesOfIndia_20170928213620_00026.csv.gz 29\n",
      "236 csv/TimesOfIndia_20170928210156_00012.csv.gz 29\n",
      "237 csv/TimesOfIndia_20170928212945_00049.csv.gz 29\n",
      "238 csv/TimesOfIndia_20170928211958_00017.csv.gz 29\n",
      "239 csv/TimesOfIndia_20170928212737_00045.csv.gz 29\n",
      "240 csv/TimesOfIndia_20170928213048_00005.csv.gz 29\n",
      "241 csv/TimesOfIndia_20170928211646_00010.csv.gz 29\n",
      "242 csv/TimesOfIndia_20170928205839_00001.csv.gz 29\n",
      "243 csv/TimesOfIndia_20170928222356_00034.csv.gz 29\n",
      "244 csv/TimesOfIndia_20170928213052_00006.csv.gz 29\n",
      "245 csv/TimesOfIndia_20170928215858_00002.csv.gz 29\n",
      "246 csv/TimesOfIndia_20170928214605_00009.csv.gz 29\n",
      "247 csv/TimesOfIndia_20170928223444_00023.csv.gz 29\n",
      "248 csv/TimesOfIndia_20170928215447_00037.csv.gz 29\n",
      "249 csv/TimesOfIndia_20170928215128_00025.csv.gz 29\n",
      "250 csv/TimesOfIndia_20170928223024_00008.csv.gz 29\n",
      "251 csv/TimesOfIndia_20170928210412_00022.csv.gz 29\n",
      "252 csv/TimesOfIndia_20170928212421_00032.csv.gz 29\n",
      "253 csv/TimesOfIndia_20170928211540_00006.csv.gz 29\n",
      "254 csv/TimesOfIndia_20170928220740_00030.csv.gz 29\n",
      "255 csv/TimesOfIndia_20170928221830_00018.csv.gz 29\n",
      "256 csv/TimesOfIndia_20170928223233_00016.csv.gz 29\n",
      "257 csv/TimesOfIndia_20170928220948_00035.csv.gz 29\n",
      "258 csv/TimesOfIndia_20170928211435_00003.csv.gz 29\n",
      "259 csv/TimesOfIndia_20170928220006_00006.csv.gz 29\n",
      "260 csv/TimesOfIndia_20170928221515_00008.csv.gz 29\n",
      "261 csv/TimesOfIndia_20170928221055_00040.csv.gz 29\n",
      "262 csv/TimesOfIndia_20170928214144_00042.csv.gz 29\n",
      "263 csv/TimesOfIndia_20170928220424_00018.csv.gz 29\n",
      "264 csv/TimesOfIndia_20170928221408_00003.csv.gz 29\n",
      "265 csv/TimesOfIndia_20170928211747_00011.csv.gz 29\n",
      "266 csv/TimesOfIndia_20170928223232_00015.csv.gz 29\n",
      "267 csv/TimesOfIndia_20170928223442_00021.csv.gz 29\n",
      "268 csv/TimesOfIndia_20170928213622_00027.csv.gz 29\n",
      "269 csv/TimesOfIndia_20170928222918_00004.csv.gz 29\n",
      "270 csv/TimesOfIndia_20170928212630_00039.csv.gz 29\n",
      "271 csv/TimesOfIndia_20170928214457_00003.csv.gz 29\n",
      "272 csv/TimesOfIndia_20170928222508_00043.csv.gz 29\n",
      "273 csv/TimesOfIndia_20170928222036_00023.csv.gz 29\n",
      "274 csv/TimesOfIndia_20170928215856_00001.csv.gz 29\n",
      "275 csv/TimesOfIndia_20170928211332_00001.csv.gz 29\n",
      "276 csv/TimesOfIndia_20170928215548_00038.csv.gz 29\n",
      "277 csv/TimesOfIndia_20170928223654_00030.csv.gz 29\n",
      "278 csv/TimesOfIndia_20170928211852_00014.csv.gz 29\n",
      "279 csv/TimesOfIndia_20170928213412_00020.csv.gz 29\n",
      "280 csv/TimesOfIndia_20170928215446_00036.csv.gz 29\n",
      "281 csv/TimesOfIndia_20170928212735_00043.csv.gz 29\n",
      "282 csv/TimesOfIndia_20170928222040_00025.csv.gz 29\n",
      "283 csv/TimesOfIndia_20170928222507_00042.csv.gz 29\n",
      "284 csv/TimesOfIndia_20170928220111_00010.csv.gz 29\n",
      "285 csv/TimesOfIndia_20170928215900_00004.csv.gz 29\n",
      "286 csv/TimesOfIndia_20170928223130_00013.csv.gz 29\n",
      "287 csv/TimesOfIndia_20170928214146_00044.csv.gz 29\n",
      "288 csv/TimesOfIndia_20170928215237_00030.csv.gz 29\n",
      "289 csv/TimesOfIndia_20170928213053_00007.csv.gz 29\n",
      "290 csv/TimesOfIndia_20170928220108_00008.csv.gz 29\n"
     ]
    }
   ],
   "source": [
    "header = True\n",
    "for i, fn in enumerate(glob('csv/*.csv.gz')):\n",
    "    #df = pd.read_csv(fn, nrows=100)\n",
    "    df = pd.read_csv(fn)\n",
    "    dcols = list(cols - set(df.columns))\n",
    "    for c in dcols:\n",
    "        df[c] = None\n",
    "    df[list(cols)].to_csv(output_file, mode='w' if header else 'a', header=header, index=False, compression='gzip')\n",
    "    header = False\n",
    "    print(i, fn, len(df.columns))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "343ed3a5",
   "metadata": {},
   "source": [
    "### Step 3: Check output file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ed03c4ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9af936ac",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SourceType</th>\n",
       "      <th>Publication.PublicationID</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Contributor.PersonName</th>\n",
       "      <th>Contributor.LastName</th>\n",
       "      <th>URLDocView</th>\n",
       "      <th>DateTimeStamp</th>\n",
       "      <th>Publication.Qualifier</th>\n",
       "      <th>AlphaPubDate</th>\n",
       "      <th>ActionCode</th>\n",
       "      <th>...</th>\n",
       "      <th>Contributor.NameSuffix</th>\n",
       "      <th>Contributor.MiddleName</th>\n",
       "      <th>FullText</th>\n",
       "      <th>ObjectType</th>\n",
       "      <th>Contributor.OrganizationName</th>\n",
       "      <th>Publication.Title</th>\n",
       "      <th>Publisher</th>\n",
       "      <th>Contributor.OriginalForm</th>\n",
       "      <th>LanguageCode</th>\n",
       "      <th>NumericPubDate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54644</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/608500584/</td>\n",
       "      <td>20170928172148</td>\n",
       "      <td>New Delhi, India</td>\n",
       "      <td>Feb 24, 1998</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SJqSwldni8 vold apf f181 CflSyZTS Theleadergua...</td>\n",
       "      <td>Credit/Acknowledgement</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Times of India (1861-current)</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENG</td>\n",
       "      <td>19980224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54644</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/608869332/</td>\n",
       "      <td>20170928172126</td>\n",
       "      <td>New Delhi, India</td>\n",
       "      <td>May 22, 1996</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>JHHQk                 guest Akshay Page 3</td>\n",
       "      <td>['Table of Contents', 'Front Matter']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Times of India (1861-current)</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENG</td>\n",
       "      <td>19960522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54644</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/608623689/</td>\n",
       "      <td>20170928172151</td>\n",
       "      <td>New Delhi, India</td>\n",
       "      <td>Sep 17, 1997</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UB Baseball basketball and football have their...</td>\n",
       "      <td>Advertisement</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Times of India (1861-current)</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENG</td>\n",
       "      <td>19970917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54644</td>\n",
       "      <td>MUMBAI : Chief minister Manohar Joshi will see...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/608797730/</td>\n",
       "      <td>20170928172250</td>\n",
       "      <td>New Delhi, India</td>\n",
       "      <td>Sep 16, 1998</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>By Vldyadhar Date The Times of India News Serv...</td>\n",
       "      <td>['Feature', 'Article']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Times of India (1861-current)</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>Vidyadhar Date The Times of India News Service</td>\n",
       "      <td>ENG</td>\n",
       "      <td>19980916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54694</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/233771878/</td>\n",
       "      <td>20170928172211</td>\n",
       "      <td>Bombay, India</td>\n",
       "      <td>Aug 3, 1839</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Printed and Published for the Bombay Times by ...</td>\n",
       "      <td>General Information</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Bombay Times and Journal of Commerce (1838...</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENG</td>\n",
       "      <td>18390803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54644</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/741170447/</td>\n",
       "      <td>20170928173348</td>\n",
       "      <td>New Delhi, India</td>\n",
       "      <td>Jan 14, 1969</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>German team on way to Rourkela                ...</td>\n",
       "      <td>Front Page/Cover Story</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Times of India (1861-current)</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENG</td>\n",
       "      <td>19690114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54644</td>\n",
       "      <td>MUMBAI: An unbeaten halt-century by Yogesh Raj...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/741230807/</td>\n",
       "      <td>20170928173243</td>\n",
       "      <td>New Delhi, India</td>\n",
       "      <td>Feb 10, 2000</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CITY SPORT                 BTTTCxnnrm         ...</td>\n",
       "      <td>['Feature', 'Article']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Times of India (1861-current)</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>Our Correspondents</td>\n",
       "      <td>ENG</td>\n",
       "      <td>20000210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54644</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/614079844/</td>\n",
       "      <td>20170928173321</td>\n",
       "      <td>New Delhi, India</td>\n",
       "      <td>Jul 17, 1967</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AfFnrmijiffJriii- BOMBAY DELHI MAURA</td>\n",
       "      <td>Advertisement</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Times of India (1861-current)</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENG</td>\n",
       "      <td>19670717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54644</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/741150898/</td>\n",
       "      <td>20170928173354</td>\n",
       "      <td>New Delhi, India</td>\n",
       "      <td>Oct 1, 1966</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TTie Statesman of Calcutta and Delhi in statem...</td>\n",
       "      <td>['Feature', 'Article']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Times of India (1861-current)</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENG</td>\n",
       "      <td>19661001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>Historical Newspapers</td>\n",
       "      <td>54644</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://search.proquest.com/docview/613860310/</td>\n",
       "      <td>20170928173242</td>\n",
       "      <td>New Delhi, India</td>\n",
       "      <td>Nov 22, 1939</td>\n",
       "      <td>change</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mr Hlnrichs ex-President of the Western India ...</td>\n",
       "      <td>Image/Photograph</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Times of India (1861-current)</td>\n",
       "      <td>Bennett, Coleman &amp; Company Limited</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENG</td>\n",
       "      <td>19391122</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 SourceType  Publication.PublicationID  \\\n",
       "0     Historical Newspapers                      54644   \n",
       "1     Historical Newspapers                      54644   \n",
       "2     Historical Newspapers                      54644   \n",
       "3     Historical Newspapers                      54644   \n",
       "4     Historical Newspapers                      54694   \n",
       "...                     ...                        ...   \n",
       "9995  Historical Newspapers                      54644   \n",
       "9996  Historical Newspapers                      54644   \n",
       "9997  Historical Newspapers                      54644   \n",
       "9998  Historical Newspapers                      54644   \n",
       "9999  Historical Newspapers                      54644   \n",
       "\n",
       "                                               Abstract  \\\n",
       "0                                                   NaN   \n",
       "1                                                   NaN   \n",
       "2                                                   NaN   \n",
       "3     MUMBAI : Chief minister Manohar Joshi will see...   \n",
       "4                                                   NaN   \n",
       "...                                                 ...   \n",
       "9995                                                NaN   \n",
       "9996  MUMBAI: An unbeaten halt-century by Yogesh Raj...   \n",
       "9997                                                NaN   \n",
       "9998                                                NaN   \n",
       "9999                                                NaN   \n",
       "\n",
       "     Contributor.PersonName Contributor.LastName  \\\n",
       "0                       NaN                  NaN   \n",
       "1                       NaN                  NaN   \n",
       "2                       NaN                  NaN   \n",
       "3                       NaN                  NaN   \n",
       "4                       NaN                  NaN   \n",
       "...                     ...                  ...   \n",
       "9995                    NaN                  NaN   \n",
       "9996                    NaN                  NaN   \n",
       "9997                    NaN                  NaN   \n",
       "9998                    NaN                  NaN   \n",
       "9999                    NaN                  NaN   \n",
       "\n",
       "                                         URLDocView   DateTimeStamp  \\\n",
       "0     http://search.proquest.com/docview/608500584/  20170928172148   \n",
       "1     http://search.proquest.com/docview/608869332/  20170928172126   \n",
       "2     http://search.proquest.com/docview/608623689/  20170928172151   \n",
       "3     http://search.proquest.com/docview/608797730/  20170928172250   \n",
       "4     http://search.proquest.com/docview/233771878/  20170928172211   \n",
       "...                                             ...             ...   \n",
       "9995  http://search.proquest.com/docview/741170447/  20170928173348   \n",
       "9996  http://search.proquest.com/docview/741230807/  20170928173243   \n",
       "9997  http://search.proquest.com/docview/614079844/  20170928173321   \n",
       "9998  http://search.proquest.com/docview/741150898/  20170928173354   \n",
       "9999  http://search.proquest.com/docview/613860310/  20170928173242   \n",
       "\n",
       "     Publication.Qualifier  AlphaPubDate ActionCode  ...  \\\n",
       "0         New Delhi, India  Feb 24, 1998     change  ...   \n",
       "1         New Delhi, India  May 22, 1996     change  ...   \n",
       "2         New Delhi, India  Sep 17, 1997     change  ...   \n",
       "3         New Delhi, India  Sep 16, 1998     change  ...   \n",
       "4            Bombay, India   Aug 3, 1839     change  ...   \n",
       "...                    ...           ...        ...  ...   \n",
       "9995      New Delhi, India  Jan 14, 1969     change  ...   \n",
       "9996      New Delhi, India  Feb 10, 2000     change  ...   \n",
       "9997      New Delhi, India  Jul 17, 1967     change  ...   \n",
       "9998      New Delhi, India   Oct 1, 1966     change  ...   \n",
       "9999      New Delhi, India  Nov 22, 1939     change  ...   \n",
       "\n",
       "     Contributor.NameSuffix Contributor.MiddleName  \\\n",
       "0                       NaN                    NaN   \n",
       "1                       NaN                    NaN   \n",
       "2                       NaN                    NaN   \n",
       "3                       NaN                    NaN   \n",
       "4                       NaN                    NaN   \n",
       "...                     ...                    ...   \n",
       "9995                    NaN                    NaN   \n",
       "9996                    NaN                    NaN   \n",
       "9997                    NaN                    NaN   \n",
       "9998                    NaN                    NaN   \n",
       "9999                    NaN                    NaN   \n",
       "\n",
       "                                               FullText  \\\n",
       "0     SJqSwldni8 vold apf f181 CflSyZTS Theleadergua...   \n",
       "1             JHHQk                 guest Akshay Page 3   \n",
       "2     UB Baseball basketball and football have their...   \n",
       "3     By Vldyadhar Date The Times of India News Serv...   \n",
       "4     Printed and Published for the Bombay Times by ...   \n",
       "...                                                 ...   \n",
       "9995  German team on way to Rourkela                ...   \n",
       "9996  CITY SPORT                 BTTTCxnnrm         ...   \n",
       "9997               AfFnrmijiffJriii- BOMBAY DELHI MAURA   \n",
       "9998  TTie Statesman of Calcutta and Delhi in statem...   \n",
       "9999  Mr Hlnrichs ex-President of the Western India ...   \n",
       "\n",
       "                                 ObjectType Contributor.OrganizationName  \\\n",
       "0                    Credit/Acknowledgement                          NaN   \n",
       "1     ['Table of Contents', 'Front Matter']                          NaN   \n",
       "2                             Advertisement                          NaN   \n",
       "3                    ['Feature', 'Article']                          NaN   \n",
       "4                       General Information                          NaN   \n",
       "...                                     ...                          ...   \n",
       "9995                 Front Page/Cover Story                          NaN   \n",
       "9996                 ['Feature', 'Article']                          NaN   \n",
       "9997                          Advertisement                          NaN   \n",
       "9998                 ['Feature', 'Article']                          NaN   \n",
       "9999                       Image/Photograph                          NaN   \n",
       "\n",
       "                                      Publication.Title  \\\n",
       "0                     The Times of India (1861-current)   \n",
       "1                     The Times of India (1861-current)   \n",
       "2                     The Times of India (1861-current)   \n",
       "3                     The Times of India (1861-current)   \n",
       "4     The Bombay Times and Journal of Commerce (1838...   \n",
       "...                                                 ...   \n",
       "9995                  The Times of India (1861-current)   \n",
       "9996                  The Times of India (1861-current)   \n",
       "9997                  The Times of India (1861-current)   \n",
       "9998                  The Times of India (1861-current)   \n",
       "9999                  The Times of India (1861-current)   \n",
       "\n",
       "                               Publisher  \\\n",
       "0     Bennett, Coleman & Company Limited   \n",
       "1     Bennett, Coleman & Company Limited   \n",
       "2     Bennett, Coleman & Company Limited   \n",
       "3     Bennett, Coleman & Company Limited   \n",
       "4     Bennett, Coleman & Company Limited   \n",
       "...                                  ...   \n",
       "9995  Bennett, Coleman & Company Limited   \n",
       "9996  Bennett, Coleman & Company Limited   \n",
       "9997  Bennett, Coleman & Company Limited   \n",
       "9998  Bennett, Coleman & Company Limited   \n",
       "9999  Bennett, Coleman & Company Limited   \n",
       "\n",
       "                            Contributor.OriginalForm LanguageCode  \\\n",
       "0                                                NaN          ENG   \n",
       "1                                                NaN          ENG   \n",
       "2                                                NaN          ENG   \n",
       "3     Vidyadhar Date The Times of India News Service          ENG   \n",
       "4                                                NaN          ENG   \n",
       "...                                              ...          ...   \n",
       "9995                                             NaN          ENG   \n",
       "9996                              Our Correspondents          ENG   \n",
       "9997                                             NaN          ENG   \n",
       "9998                                             NaN          ENG   \n",
       "9999                                             NaN          ENG   \n",
       "\n",
       "      NumericPubDate  \n",
       "0           19980224  \n",
       "1           19960522  \n",
       "2           19970917  \n",
       "3           19980916  \n",
       "4           18390803  \n",
       "...              ...  \n",
       "9995        19690114  \n",
       "9996        20000210  \n",
       "9997        19670717  \n",
       "9998        19661001  \n",
       "9999        19391122  \n",
       "\n",
       "[10000 rows x 29 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"toi_parsed_all.csv.gz\", nrows = 10000)\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}