RecognisingSpeciesAndLocations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Imports\" data-toc-modified-id=\"Imports-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Imports</a></span></li><li><span><a href=\"#Loading-Text-File\" data-toc-modified-id=\"Loading-Text-File-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Loading Text File</a></span></li><li><span><a href=\"#Tokenising-Document\" data-toc-modified-id=\"Tokenising-Document-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Tokenising Document</a></span></li><li><span><a href=\"#Named-Entity-Recognition\" data-toc-modified-id=\"Named-Entity-Recognition-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>Named Entity Recognition</a></span><ul class=\"toc-item\"><li><span><a href=\"#Using-small-model\" data-toc-modified-id=\"Using-small-model-4.1\"><span class=\"toc-item-num\">4.1&nbsp;&nbsp;</span>Using small model</a></span></li><li><span><a href=\"#Using-medium-model\" data-toc-modified-id=\"Using-medium-model-4.2\"><span class=\"toc-item-num\">4.2&nbsp;&nbsp;</span>Using medium model</a></span></li><li><span><a href=\"#Using-large-model\" data-toc-modified-id=\"Using-large-model-4.3\"><span class=\"toc-item-num\">4.3&nbsp;&nbsp;</span>Using large model</a></span></li><li><span><a href=\"#Comparing-Location-identification-between-Spacy-models\" data-toc-modified-id=\"Comparing-Location-identification-between-Spacy-models-4.4\"><span class=\"toc-item-num\">4.4&nbsp;&nbsp;</span>Comparing Location-identification between Spacy models</a></span></li></ul></li><li><span><a href=\"#Extracting-Locations\" data-toc-modified-id=\"Extracting-Locations-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>Extracting Locations</a></span><ul class=\"toc-item\"><li><span><a href=\"#Gazetteers\" data-toc-modified-id=\"Gazetteers-5.1\"><span class=\"toc-item-num\">5.1&nbsp;&nbsp;</span>Gazetteers</a></span></li><li><span><a href=\"#Locations\" data-toc-modified-id=\"Locations-5.2\"><span class=\"toc-item-num\">5.2&nbsp;&nbsp;</span>Locations</a></span><ul class=\"toc-item\"><li><span><a href=\"#Using-Spacy's-Small-Model\" data-toc-modified-id=\"Using-Spacy's-Small-Model-5.2.1\"><span class=\"toc-item-num\">5.2.1&nbsp;&nbsp;</span>Using Spacy's Small Model</a></span></li><li><span><a href=\"#Using-Spacy's-Medium-Model\" data-toc-modified-id=\"Using-Spacy's-Medium-Model-5.2.2\"><span class=\"toc-item-num\">5.2.2&nbsp;&nbsp;</span>Using Spacy's Medium Model</a></span></li><li><span><a href=\"#Using-Spacy's-Large-Model\" data-toc-modified-id=\"Using-Spacy's-Large-Model-5.2.3\"><span class=\"toc-item-num\">5.2.3&nbsp;&nbsp;</span>Using Spacy's Large Model</a></span></li></ul></li><li><span><a href=\"#Extracting-and-Saving-Locations-for-each-Document-in-a-Folder\" data-toc-modified-id=\"Extracting-and-Saving-Locations-for-each-Document-in-a-Folder-5.3\"><span class=\"toc-item-num\">5.3&nbsp;&nbsp;</span>Extracting and Saving Locations for each Document in a Folder</a></span></li><li><span><a href=\"#Finding-Locations-in-Collection-of-Documents\" data-toc-modified-id=\"Finding-Locations-in-Collection-of-Documents-5.4\"><span class=\"toc-item-num\">5.4&nbsp;&nbsp;</span>Finding Locations in Collection of Documents</a></span></li></ul></li><li><span><a href=\"#Species\" data-toc-modified-id=\"Species-6\"><span class=\"toc-item-num\">6&nbsp;&nbsp;</span>Species</a></span><ul class=\"toc-item\"><li><span><a href=\"#Fraser's-Comprehensive-Species-List\" data-toc-modified-id=\"Fraser's-Comprehensive-Species-List-6.1\"><span class=\"toc-item-num\">6.1&nbsp;&nbsp;</span>Fraser's Comprehensive Species List</a></span></li><li><span><a href=\"#Locating-Genus-and-Species-Mentions-in-Document\" data-toc-modified-id=\"Locating-Genus-and-Species-Mentions-in-Document-6.2\"><span class=\"toc-item-num\">6.2&nbsp;&nbsp;</span>Locating Genus and Species Mentions in Document</a></span></li><li><span><a href=\"#Code\" data-toc-modified-id=\"Code-6.3\"><span class=\"toc-item-num\">6.3&nbsp;&nbsp;</span>Code</a></span></li><li><span><a href=\"#Second-Document\" data-toc-modified-id=\"Second-Document-6.4\"><span class=\"toc-item-num\">6.4&nbsp;&nbsp;</span>Second Document</a></span></li><li><span><a href=\"#Extracting-and-Saving-Species-for-each-Document-in-a-Folder\" data-toc-modified-id=\"Extracting-and-Saving-Species-for-each-Document-in-a-Folder-6.5\"><span class=\"toc-item-num\">6.5&nbsp;&nbsp;</span>Extracting and Saving Species for each Document in a Folder</a></span></li></ul></li><li><span><a href=\"#Combining-Species-and-Location-into-Matrix\" data-toc-modified-id=\"Combining-Species-and-Location-into-Matrix-7\"><span class=\"toc-item-num\">7&nbsp;&nbsp;</span>Combining Species and Location into Matrix</a></span></li><li><span><a href=\"#Sandbox\" data-toc-modified-id=\"Sandbox-8\"><span class=\"toc-item-num\">8&nbsp;&nbsp;</span>Sandbox</a></span><ul class=\"toc-item\"><li><span><a href=\"#String-Matching\" data-toc-modified-id=\"String-Matching-8.1\"><span class=\"toc-item-num\">8.1&nbsp;&nbsp;</span>String Matching</a></span></li><li><span><a href=\"#Comparing-String-Matching-Metrics\" data-toc-modified-id=\"Comparing-String-Matching-Metrics-8.2\"><span class=\"toc-item-num\">8.2&nbsp;&nbsp;</span>Comparing String Matching Metrics</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:09:23.029698Z",
     "start_time": "2021-02-08T03:09:23.014592Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:80% !important; } p, ul {max-width:␣\n",
       ",→40em;} .rendered_html table { margin-left: 0; } .output_subarea.output_png {␣\n",
       ",→display: flex; justify-content: center;}</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML, Markdown as md\n",
    "display(HTML(\"\"\"<style>.container { width:80% !important; } p, ul {max-width:␣\n",
    ",→40em;} .rendered_html table { margin-left: 0; } .output_subarea.output_png {␣\n",
    ",→display: flex; justify-content: center;}</style>\"\"\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2021-03-10T22:38:44.950Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-05T00:43:33.129280Z",
     "start_time": "2021-03-05T00:43:33.127142Z"
    }
   },
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-05T00:43:44.263302Z",
     "start_time": "2021-03-05T00:43:38.415771Z"
    }
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy import displacy\n",
    "from collections import Counter\n",
    "import en_core_web_sm\n",
    "#using small Spacy model\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:09:46.872269Z",
     "start_time": "2021-02-08T03:09:29.279318Z"
    }
   },
   "outputs": [],
   "source": [
    "import en_core_web_md\n",
    "nlp_md = spacy.load('en_core_web_md')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-05T00:43:47.111796Z",
     "start_time": "2021-03-05T00:43:44.273473Z"
    }
   },
   "outputs": [],
   "source": [
    "import en_core_web_lg\n",
    "nlp_lg = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Text File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T04:15:54.835829Z",
     "start_time": "2021-02-26T04:15:54.820227Z"
    }
   },
   "outputs": [],
   "source": [
    "file = open(\"smallPDFs/10.1007@s00300-017-2230-0.txt\")\n",
    "doc = file.read()\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T04:15:56.417882Z",
     "start_time": "2021-02-26T04:15:56.411251Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://doi.org/10.1007/s00300-017-2230-0 ORIGINAL PAPER @® CrossMark Collembola of Barrientos Island, Antarctica: first census and assessment of environmental factors determining springtail distribution Natalia Enriquez\\' - Pablo Tejedo? - Javier Benayas? - Belén Albertos? - Maria José Luciafiez\\' Received: 21 October 2016 / Revised: 13 August 2017 / Accepted: 3 December 2017 © Springer-Verlag GmbH Germany, part of Springer Nature 2017 Abstract Barrientos Island is a small islet in the South Shetland archipelago frequently visited by Antarctic tourists. Collembola were recently used in another study developed in this site to assess the environmental conditions of two paths used by visitors, showing the importance of this soil faunal community. This motivated the realization of the first comprehensive census of Collembola from Barrientos Island. Fifty-six samples were recorded over three seasons, 2011-2013, from eight different substrate types. During the last campaign, 39 soil samples were also taken to analyze different physical, chemical, biological, microbiological, and biochemical variables in order to put into relation all these environmental conditions with abundance and richness of Collembola species. A total of ten species were recorded, all of them previously cited from Antarctica. The native species Cryptopygus antarcticus antarcticus Willem (Collembola: Isotomidae) is the most abundant in the island. Two cosmopolitan species—Hypogastrura viatica (Tullberg) (Hypogastruridae) and Mesaphorura macrochaeta Rusek (Tullbergiidae)—were also identified. Many of the species on this island can be considered as eurytopic, appearing in most of the analyzed substrate types. Collembola distribution was found to be influenced by elevation and, to a lesser extent, by phosphorous content and slope. Other variables that are usually cited in the specialized literature did not significantly influence the Collembola distribution in this island. Our results show that the distribution and abundance of Collembola in Antarctic locations is not directly dependent on a single factor. The environmental characteristics of each studied site will determine what factors are driving the composition and structure of this soil community. Keywords Aitcho Islands - Non-indigenous species - Hypogastruridae - Basic collembolan species set - Inter-annual resampling Introduction Barrientos Island (62°24’S, 59°47\\'W) is located between Robert and Greenwich Islands, in the South Shetland archi Electronic supplementary material The online version of this pelago, Antarctica (Fig. 1). The island is an ice-free area in supplementary material, which is available to authorized users. b<| Natalia Enriquez Departamento de Biologia, Universidad Aut6noma de natalia.enriquez.miranda@ gmail.com Madrid, C/ Darwin 2, 28049 Madrid, Spain Pablo Tejedo 2 Departamento de Ecologia, Universidad Aut6noma de pablo.tejedo @uam.es Madrid, C/ Darwin 2, 28049 Madrid, Spain Javier Benayas Departamento de Botanica y Geologia, Universidad javier.benayas @uam.es de Valencia, Avda. Vicente Andrés Estellés s/n, Belén Albertos 46100 Buryjassot, Spain belen.albertos @uv.es Maria José Luciaiiez myjose.luciannez @uam.es Published online: 13 December 2017 a Springer 357500 3078000 [| Closed area C Penguin Rookery Chinstrap Penguins Chinstrap and Gentoo Penguins Gentoo Penguins 3077500 SS a5\" se, 13.43/44. 11 A 3 3077000 357500   358000 358500 Graphic Scale 0 80 160 240 es \\\\Victers UTM Projection WGS 84 datum - Zone 21S 3078000 he ou! a Ad . \\' 1 sa’ FS yrpe . BY 4 of een le 3077500 3077000 Pacific Ocean Ui ctose@ ee) eld Prd | Aue 358000 358500 Fig.1 Map of Barrientos Island showing the main footpaths, some selected features of the island and all sample sites for 2011 (Sites11-01 to 11-06), 2012 (Sites12-01 to 12-11), and 2013 (Sites 13-01 to 13-39) most visited terrestrial sites in Antarctica, receiving 8663 visitors in the 2015—2016 austral summer season (International Association of Antarctica Tour Operators 2015). A set of site guidelines for visitors was adopted by the Antarctic Treaty Consultative Meeting (ATCM) in 2005 after validation by the ATCM’s Committee for Environmental Protection (CEP). Several areas, including nesting areas for southern giant petrels, monitoring sites for chinstrap penguins, and extensive moss carpets vulnerable to trampling, are closed to visitors. This document also includes several paths, which allow visiting different areas of the island. Two of them were recently assessed using Collembola as an indicator in order to compare their environmental conditions (Tejedo et al. 2016). Based on this study, it is decided to carry out a complete census of this taxonomic group in the island to enlarge the work developed, which is presented in this paper. Collembola in Antarctica are relatively well known due to numerous past studies focused on this taxon (e.g., Salmon 1949; Wise 1964, 1971; Greenslade and Wise 1984; Potapov 1991; Greenslade 1995; Convey et al. 2000). Particularly, Collembola from maritime Antarctica, the region comprising the western side of the Antarctic Peninsula and nearby islands, have been studied intensively (e.g., Willem 1901; Wahlgren 1906; Carpenter and Evans 1906; Weiner 1980; Greenslade 1995, 2010; Greenslade et al. 2012; Russell et al. 2014). However, the edaphic fauna of Barrientos Island has not been previously studied. Collembola fauna usually presents a patchy distribution, which depends on some environmental factors (Hertzberg et al. 1994; Ims et al. 2004). Soil moisture is often a limiting factor affecting the distribution and abundance of this taxon (e.g., Hopkin 1997; Sinclair et al. 2006; Day et al. 2009). Thus, Antarctic Collembola usually show higher densities and species richness at higher soil moistures (Russell et al. 2014). Another important habitat factor for Collembola is soil organic matter, which can be assessed through total amounts, concentrations of C and N, or C/N-relationship (e.g., Adams et al. 2006; Russell et al. 2014). Vegetation cover also plays an important role in the collembolan distribution. Usually, the denser the vegetation cover is the more individual-rich usually are the collembolan communities (Russell et al. 2014). At the local scale of patchy environments within the Antarctic, all these habitat factors appear to be more important than biotic interactions in influencing microarthropod assemblages (Adams et al. 2006; Hogg et al. 2006). These physical, chemical, and biological variables are related to other microbiological and biochemical soil properties that could affect to the soil quality, such as microbial biomass carbon, respiration, or different enzymatic activities, including urease, phosphatase, B-glucosidase (related to the cycles of N, P, and C, respectively), or dehydrogenase (which is used as an indicator of overall soil microbial activity). These soil variables are sensitive indicators of soil quality (Bastida et al. 2008) and for this reason were included in this work. The aim of this study is to identify the Collembola species on Barrientos Island, studying the substrate preferences and the relationships between Collembola distribution and environmental factors in order to contribute new knowledge about the ecology of these arthropods in maritime Antarctica. Materials and methods Study area The island’s northern coast is 1.5 km long and is dominated by steep cliffs reaching a height of 70 m. From there, a gentle slope extends down to the southern coast. The eastern and western ends of Barrientos are covered by black sand and cobbled beaches. Columnar basalt outcrops are found at the western end of the island. Extensive moss carpets cover the center of the island, with Sanionia as the dominant genus. Sanionia uncinata (Hedw.) Loeske and S. georgicouncinata (Mill. Hal.) Ochyra & Hedenis are present. Sanionia uncinata forms monospecific communities or mixed communities with Polytrichastrum alpinum (Hedw.) G.L. Sm. and Bryum pseudotriquetrum (Hedw.) G. Gaertn., B. Mey. & Scherb., together with other secondary species. Sanionia georgicouncinata is often found with Warnstorfia fontinaliopsis (Miill. Hal.) Ochyra, but forms extent monospecific carpets as well. Different crustose lichen species, such as Xanthoria spp. or Caloplaca spp., are also present in Barrientos. The green alga Prasiola crispa (Lightfoot) Kiitzing is widespread, especially in the most nitrified environments. The local fauna is diverse and includes gentoo penguins (Pygoscelis papua (Forster, 1781)), chinstrap penguins (Pygoscelis antarcticus (Forster, 1781)), southern giant petrels (Macronectes giganteus (Gmelin, 1789)), kelp gulls (Larus dominicanus Lichtenstein, 1823), skuas (Catharacta spp.), southern elephant seals (Mirounga leonina (Linnaeus, 1758)), Weddell seals (Leptonychotes weddellii (Lesson, 1826)), and Antarctic fur seals (Arctocephalus gazella (Peters, 1875)) (Antarctic Treaty Secretariat 2015). Collection, extraction, preparation, and identification of Collembola samples The study was conducted in three phases. An exploratory study was developed on February 7, 2011, whereby samples were taken from six sites (Sites 11-01 to 11-06) in different types of substrate (Fig. 1). The second visit to the island was on February 16, 2012. Samples from 11 sites (Sites 12-01 to 12-11) were collected. Six of them were located along two footpaths that cross the island. The results from the second visit expanded the list of known Collembola species on the island. A third study employing stratified sampling was designed to include all types of substrates found on the island. Thirty-nine sites were sampled (Sites 13-01 to 13-39) during February 2013, including nine samples from the footpaths. At each site, additional soil samples with about | to 1.5 kg were collected for physical, chemical, and microbiological analyses. These samples were packaged in plastic freezer bags, double labeled, and stored at 4 °C until processing in the laboratory. A brief description of the surrounding environment, including identification of nearby plant communities was also recorded (Online Resource 1). A total of eight types of substrate were sampled along the three campaigns (Online Resource 2), in an effort to reflect the maximum diversity of environments on the island. A cylinder of 500 cm? was used to sample the top 8 cm of soil to capture the edaphic fauna. If present, surface vegetation was included in the sample. Where possible, the extracted material was replaced with soil and small pebbles from the nearby area to avoid creating permanent visual alteration. The samples were packaged in 1-L plastic freezer bags, double labeled, and stored at 4 °C until extraction in the laboratory. For each sample, geographical coordinates, approximate elevation above sea level and slope range were noted. The first two were obtained with a GPS Garmin eTrex Vista HCx, and the slope was measured in grades with a clinometer Suunto PM-5/360. Collembola were obtained by Tullgren extraction of the samples, carried out within 2—3 days after collection Animals collected were preserved in 99% ethanol before being sorted under a dissecting microscope. Specimens were cleared in lactic acid over a 1-week period, although this lapse was prolonged slightly for those individuals with dark pigmentation. Collembola were mounted on slides using Hoyer’s solution and identified to species level under a phase contrast microscope. Antarctic species were identified following Salmon (1962, 1974a, b), Wise (1967, 1970a, b, 1971), Weiner (1980), Deharveng (1981), Lubbock (1876), Convey et al. (1999), and Greenslade (1995, 2006, 2010). The cosmopolitan species Hypogastrura viatica (Tullberg, 1872) and Mesaphorura macrochaeta Rusek, 1976 were classified according to Jordana et al. (1997), Jordana and Arbea (1989), Potapov (2001), and Greenslade et al. (2014) Physical—chemical and microbiological variables Soil samples were sieved (< 2 mm) and all debris and plant remains were removed before analysis in the laboratory. Electrical conductivity and pH were measured in a 1/5 w/v aqueous solution using a pH-meter (Navi Horiba model). Total organic carbon was determined by wet oxidation with K,CrO- and titration of dichromate excess with Mohr’s salt (Yeomans and Bremner 1988) while the organic matter was inferred by multiplying the total organic carbon content by 1.728 (Lucas-Borja et al. 2010). Total carbonates were measured by evaluation of the HCl excess with NaOH according to the method proposed by Baena et al. (2013). Total nitrogen was determined using Kjeldahl’s method modified by Bremner (1965). The total phosphorus content was determined by the method of Olsen et al. (1954). Soil moisture was determined using the gravimetric method, drying the sample at about 115 °C during 24 h and measuring the mass difference before and after drying (Baena et al. 2013). The microbial biomass carbon was determined following the method adapted by Garcia et al. (2003) from Vance et al. (1987). Urease activity was determined by staining the ammonium released into an incubation solution which was kept at 37 °C for 2 h (Kandeler et al. 1999). Alkaline phosphatase and B-glucosidase activities were determined following the methods reported by Tabatabai (1982). Soil dehydrogenase activity was determined by measuring the reduction of p-iodonitrotetrazolium chloride (INT) to p-iodonitrotetrazolium formazan in 1 g of soil, a method reported by Hedo et al. (2015). Basal soil respiration was analyzed by moistening 15 g of soil to 50-60% of its waterholding capacity (water potential: 0.055 Mpa), then letting it incubate in hermetically sealed flasks for 20 days at 28 °C. The CO, released was periodically measured (every day for the first 4 days after this, weekly) using an infrared gas analyzer (Toray PG-100, Toray Engineering Co. Ltd., Japan). The data were summed to give a cumulative amount of CO, released after 20 days of incubation. For each variable, two  replicates were analyzed for each sample site. The average of the values from the replicates was used in the statistical analyses. Statistical analyses Collembolan presence per sample data were transformed to abundance per surface area (individuals/m7) in order to facilitate comparisons with previous studies. Mean and standard error of mean were used to summarize a number of results (Table 1). All species data and most environmental data were log-transformed to obtain normalized and smooth distributions. Relationships between species abundance, Collembola population abundance, species richness, and environmental variables were analyzed using Spearman’s rank correlations. Kruskal—Wallis test was applied for comparisons of collembolan species richness and abundance between the different substrate types. A Canonical Correspondence Analysis was performed in R with the “Vegan” package to assess the variability of springtail communities with respect to environmental variables. Significance level for null hypothesis rejection was established at 0.05 for all tests. The software used for the statistical analysis were IBM© SPSS© Statistics 21 (SPSS, Inc.) and R software (R Core Team 2017). Results Fauna survey A total of ten Collembola species and 8570 specimens were recorded (Online Resource 2). They were sampled at 56 sites during three field campaigns. Eight species are native in Antarctica, Friesea bispinosa Deharveng, 1981, Friesea grisea (Schaffer, 1891), Friesea woyciechowskii Weiner, 1980, Tullbergia mixta Wahlgren, 1906, Archisotoma brucei (Carpenter, 1907), Cryptopygus antarcticus antarcticus Willem, 1901, Cryptopygus badasa Greenslade, 1995, and Folsomotoma octooculata (Willem, 1901). Two species are non-indigenous in Antarctica, Hypogastrura viatica and Mesaphorura macrochaeta (Online Resource 3). All species have already been recorded in Antarctica (Deharveng 1981; Greenslade et al. 2012). The collembolan total richness for Barrientos has been changing along the 3 years of the study. During the 2011 campaign, six sites and four of the eight substrate types of the island were sampled, obtaining six species (Online Resource 2). In 2012 campaign the number of sampling sites (11) and substrate types (four, but being 3 new) increased, obtaining eight species, with two new records F. grisea and A. brucei (Online Resource 2). Finally, in 2013 campaign, 39 samples were obtained in six substrate types (1 new), cl 81 sojdures JO N Cl + 62 C78S+TEIL LOCI + PLOT Li +8 voy + CL9 SIL + ELI 6bb + P6S C687 + €60b LELI + 9S07 CLET + O8E7 Co + BIZ LOT + POE 6 + OFT OSTI + POET 099 + 106 DID]NIOOJIO psppvq puojowosjog snskdojdkiy 8 ZLV6 F LOL\\'OT 6S + cL LET Iv ¥ OLS\\'8h LOL + pZ6€ T801 + 87ST bv + 601 PEV\\'8T F ESO\\'SS T98T + OLOE SNITNIADJUD SNITNIADJUD sndkdojdkuy DwojosiyoAy vIssaq]yN[, véansoydvsapy C 691 + SLI Ot + 9¢ 19 INA 6$ + cL OST + €0€ 98¢ + €L8 80S + 669 6vI + 781 Ot + 9¢ cE8 + 6IOL OS + OS Cy + Cel OTT + v9 co + 601 68 + 601 DIXIU yDJIDYIOAIDUL 11ySMOYIAaIIKOM DASA J DaSids Vasals] soroeds uevyjodoursos C6 + COE LL + 601 8LT + 817 c6 + 601 psouidsiq 4 DINIDIA pasaiy vinsspsodky osuel oyensqns yyedjooy seas yueydeyo uzoyynos Aq posn MOTTE Auojoo umnsudg (syooururnty) suorysnd SSOUW Te], syeul pure sjodieo oydydoArg suorysno pure syn} Ssoul 04S yOoys e3[V soyoreq /ou0Z [epHscju] od} oyesqns UMOUS Ose ore add} aJeINSQNs YoRd UI popsOdal soydures JO Jaquunu oy} pue ‘yWosaid st satoads yoRa aIDyM sadA} ayejsqns jo Joquinu “ol ‘esues soyensqns oy, ‘odA} oyesqns pue sorseds yowo 10F JoLe prepueys pue (Wi Jod s[enprAIpul Jo sioquinu) sotoods ejoquis[[oD Jo souvpunge ueo| | a/qeL pringer QD) § obtaining seven species, with a new record F. bispinosa (Online Resource 2). Distribution and basic notes on identified taxa ecology are provided in Online Resource 3. Basic environmental data for each substrate type and for the island as a whole are given in Online Resource 4. There are four species in the island occupied a wide range of substrates (> 6), hereafter represented in parenthesis by number of habitats with species presence/total of habitats analyze. These species are Cryptopygus antarcticus antarcticus (8/8), Folsomotoma octooculata (8/8), Cryptopygus badasa (7/8), and Friesea grisea (6/8) (Table 1). Tullbergia mixta (5/8) and Hypogastrura viatica (5/8) form a second group occupied five substrates types. Species with a narrow range of substrates (< 2) would be considered less common at the island. This group is formed by Archisotoma brucei, Friesea bispinosa, Friesea woyciechowskii, and Mesaphorura macrochaeta. Hypogastrura viatica was the only species of the family Hypogastruridae, always presenting low densities barely reaching 300 individuals/m7. This species is not included in the study focused on the environmental requirements of the collembolan species since it was not recorded in 2013, the campaign in which soil samples were taken for this purpose. The same happens with F. woyciechowskii and M. macrochaeta, which are discussed later. Belonging to the family Neanuridae, three species were recorded in Barrientos: Friesea bispinosa, F. grisea, and F. woyciechowskii. The first one was present only in a footpath with very low mean density (seven individuals/m”), not being correlated with any environmental studied variable. Friesea grisea can be considered as eurytopic since it was recorded in a high number of different substrates. However, it had low mean densities, except for wallows where above 1000 individuals/m” were recorded (Table 1). A negative correlation is observed between microbial biomass C and Fgrisea abundance (Spearman rank correlation, rs = — 0.411, n = 39, p = 0.009), hence it was not abundant in those areas that were more favorable for microorganisms. The abundance of this species correlated positively with slope (Spearman rank correlation, rs = 0.382, n = 39, p = 0.017). Finally, F. woyciechowskii only was found in bryophyte carpets and mats with low mean density (164 individuals/m7). The family Tullbergiidae was represented by Mesaphorura macrochaeta, present only in a wallow with low mean density (36 individuals/m7), and Tullbergia mixta, which appeared in five substrates but with limited mean density (< 873 individuals/m7). The abundance of the later species was positively correlated with the amount of carbonates (Spearman rank correlation, rs = 0.414, n = 34, p = 0.015). The other four recorded species belong to the family Isotomidae. Archisotoma brucei only was found in two samples, one from a footpath and other from a wallow (Table 1).  Its abundance was not correlated with any environmental studied variable. Cryptopygus antarcticus antarcticus was found in all eight types of sampled substrates, indicating eurytopicity. This species presents the highest densities in the island, with means above 48,000 individuals/m7 (Table 1) in areas occupied by the algae Prasiola crispa and penguin colonies. The least favorable substrates for C. a. antarcticus on Barrientos Island were wallows and areas covered by short moss turf and cushions, with very low densities, under 120 individuals/m7. Electrical conductivity had a negative correlation with the abundance of C. a. antarcticus (Spearman rank correlation, rs = — 0.434, n = 34, p = 0.010), as well as microbial biomass C (Spearman rank correlation, rs = — 0.395, n = 39, p = 0.013). Its abundance is positively correlated with slope (Spearman rank correlation, rs = 0.348, n = 39, p = 0.030). Cryptopygus badasa also appeared in a high variety of substrates, with the exception of footpaths. The abundance of this species correlated positively with slope (Spearman rank correlation, rs = 0.343, n = 39, p = 0.033). The last species from this family is Folsomotoma octooculata. It was also present in all substrates, with high abundances in wallows and in areas with bryophyte carpets and mats, although without reaching the densities of C. a. antarcticus. Elevation was positively correlated with the abundance of F: octooculata (Spearman rank correlation, rs = 0.350, n = 39, p = 0.029), as well as slope (Spearman rank correlation, rs = 0.467, n = 39, p = 0.003). However, pH correlates negatively with the abundance of this species (Spearman rank correlation, rs = — 0.414, n = 39, p = 0.009). Variation of Collembolan species richness and abundance with substrate type and environmental variables Median species richness for Collembola was higher in areas occupied by alga sheets, bryophyte carpets and mats, and short moss turf and cushions (Fig. 2). By contrast, median species richness was usually low in samples taken from penguin colonies, wallow used by southern elephant seals and footpaths. In fact, there are significant differences between species richness and substrate type (Kruskal—Wallis test, H56 = 16.540, p = 0.021). The total number of recorded species in each substrate ranged between four and seven, with penguin colonies and intertidal zone/beaches being the least favorable substrate types for collembolan richness (Table 1). Within the recorded environmental variables, only elevation (Spearman rank correlation, rs = 0.403, n = 39, p = 0.011) and slope (Spearman rank correlation, rs = 0.645, n = 39, p = 0.000) were positively correlated with the collembolan species richness. However, the microbial biomass C appeared to adversely affect the collembolan species  6 5 “ “ c < 4 oO = “ @ ° oO 2 Ww 2 1 0 ov Oo © 6 cS a 5 s 5 cS Cc Qe 4 i oo B+ Ss aD 3 oo o© SS 2 °o 2 £ a 1 © © 0 “) “” ” ~~ => ” ior » 3 5 8 g E 5 = Oo s 2 = o = v a. G ’ o s 2 3 He 3 2 S 3 5 E c 5 e ® < 3 v 2 D 5 o o 2 = 5 2 IN & ” m D _ 2 “ S a < 5 3 » 7 $s = 2 ’ = 3 3 = ° 2. ”~ 7 e 2g % Ad 5 o E x hes = $ WY 2 % > 2 oo S Substrate type Fig. 2 Boxplot of species richness and collembolan population abundance (log-transformed numbers of individuals per m”) by substrate type. Median (bold line), interquartile range (box), highest and low richness (Spearman rank correlation, rs = — 0.399, n = 39, p = 0.012). Regarding the total abundance of collembolan, higher densities were found in areas occupied by P. crispa, bryophyte carpets and mats and, to a lesser extent, tall moss est values which are no greater than 1.5 times the interquartile range (whiskers), outliers (circles), and extreme values (stars) are shown cushions (hummocks), and short moss turf and cushions (Fig. 2). At these locations, the presence of large amounts of C. a. antarcticus in certain samples was responsible for these high mean abundances. However, there are not significant differences between collembolan abundanc and substrate type (Kruskal—Wallis test, H56 = 6.744, p = 0.456). Only two environmental variables were correlated with the collembolan abundance, slope positively (Spearman rank correlation, rs = 0.543, n = 39, p = 0.000), and microbial biomass C negatively (Spearman rank correlation, rs = — 0.365, n = 39, p = 0.022). In order to strengthen the previous analysis, a Canonical Correspondence Analysis was performed. It shows that basically three environmental variables explain the collembolan distribution in the island: elevation, slope, and phosphorous content (p = 0.014), although only one variable, elevation, was significant at 95% (p = 0.032), meanwhile the phosphorous content was significant at 90% (p = 0.095). The ordination diagram (Online Resource 5) shows that C. badasa is likely to be found in substrates with a medium to high content of phosphorous, such as penguin colonies and some vegetated habitats. Folsomotoma octooculata and F. bispinosa are favored by elevation, since they were recorded mainly in sample sites away from the seashore. The other four species included in this test (C. a. antarcticus, T. mixta, M. macrochaeta, and F. grisea) do not show a clear trend and therefore occupy the center of the ordination diagram. Discussion Barrientos Island’s species census This is the first census of Collembola species on Barrientos Island, with a total of ten species recorded during three field campaigns. All species found in this study coincided with the “basic species set” proposed by Russell et al. (2014) for the maritime Antarctica region, which includes the following: C. a. antarcticus, C. badasa, Friesea grisea, F. woyciechowskii, Folsomotoma octooculata, and Tullbergia mixta. In addition, four other species—Friesea bispinosa, Archisotoma brucei, Hypogastrura viatica, and Mesaphorura macrochaeta—were identified, but these are also habitually present in this region. The most abundant species was Cryptopygus antarcticus antarcticus (Online Resource 2). The species census has increased in this study with each campaign, showing the importance of sampling at different times and, maybe even more important, in different environments. More species are likely to be identified following a higher sampling effort. Accordingly, recent monitoring activities in maritime Antarctica and the northern Antarctic Peninsula have revealed a higher abundance and distribution of non-indigenous species than was previously known (Greenslade et al. 2012; Russell et al. 2014).  Recorded non-indigenous species Two cosmopolitan species—Hypogastrura viatica and Mesaphorura macrochaeta—were found during two different field campaigns (Online Resource 2), expanding existing knowledge of non-indigenous species in maritime Antarctica (Hughes et al. 2015), and suggesting that biosecurity practices employed to date may not be completely adequate. The first species, H. viatica, is widely distributed in the maritime Antarctic (Frenot et al. 2005; Greenslade and Convey 2012; Hughes et al. 2015). Its expansion throughout this region is favored because it is osmoconformity at higher external values of salt, being able to hyper-regulate the osmotic and 1onic concentration of the hemolymph acclimated for at least 7 days (Witteveen et al. 1987). Its presence could affect the native Collembola, since this species seems to be able to outcompete C. a. antarcticus on coastal sites on South Georgia (Convey et al. 1999). On Barrientos Island, H. viatica was recorded in five types of substrates (from a total of eight), while M. macrochaeta was only present in one (Table 1). The presence of H. viatica could be a consequence of human activity on the island, since it was recorded in the vicinity of footpaths and on beaches visited by tourists. The presence of M. macrochaeta may be associated with the presence of birds or marine mammals that have acted as dispersion vectors (Panagis 1985; Lebedeva and Krivolutsky 2003; Krivolutsky et al. 2004). However, neither hypothesis can be tested with our data. The absence of cosmopolitan species in the samples collected in 2013 could be explained by sampling issues and highly patchy distribution of collembolans (Hertzberg et al. 1994; Ims et al. 2004). In the case of Antarctic Collembola, it is common to find large differences in species richness and abundance from one sample site to another (Tilbrook 1967b) and, therefore, this result is logical and plausible. Collembola is the dominant non-indigenous taxon that has been introduced into different environments within the Antarctic Peninsula and Scotia Arc (Hughes et al. 2015), and it established becoming invasive on sub-Antarctic islands (Frenot et al. 2005). Introductions of non-indigenous fauna could trigger alterations of native terrestrial fauna, owing to isolation, high levels of endemism, and a general lack of inter-species competition (Chown and Convey 2007; Convey 2008). The first non-indigenous invertebrate to be reported in the South Shetland Islands in 1949 was the collembolan H. viatica found on Deception Island (Hack 1949). This species was recorded even before, in 1901, on sub-Antarctic Macquarie Island (Carpenter 1909). These findings suggest that the introduction and at least transient establishment of non-indigenous species are likely to have been occurring for as long as humans have been inhabiting suitable Antarctic locations, and possibly since the early nineteenth century when sealers first visited the northern Antarctic Peninsula and South Shetland Islands (Dibbern 2010). The species M. macrochaeta was reported for the first ttme on Macquarie Island in 2008 (Greenslade 2010). It was probably originally introduced to the Southern Hemisphere and northern sites by human intervention in importations of soil, plants, and mosses (Greenslade 1987, 2006). All locations colonized by non-indigenous springtails within Antarctica are subject to high levels of national operator and tourist activity, as is in the case of Barrientos Island (International Association of Antarctica Tour Operators 2015). Tourist visits are concentrated mainly during a 5-month period, from November to March. Visits ashore are generally of short duration and moderate intensity, in groups of fewer than 100 people. Ships that usually anchor at this location carry 200 or fewer passengers and only one ship can disembark passengers at a time, with a maximum of two ships per day. With the exception of closed areas, visitors can roam freely under supervision (Antarctic Treaty Secretariat 2015). Researchers also visit the island along the austral summer season, although in a much smaller number that varies according to the needs of the research projects developed each year. The risk of bioinvasions between different remote Antarctic locations may be increasing due to the continuous movement of scientists and tourists, facilitating undesirable and unintended translocations. In this context, there is a real possibility of spreading collembolan species around Barrientos Island through the movement of visitors. However, this hypothesis needs to be tested. To reduce the impact of non-indigenous species within the Antarctic Treaty area, the measures proposed by Hughes et al. (2015) need to be applied. These include better education of all Antarctic visitors, biosecurity practices in all Antarctic operations, monitoring of high activity sites for non-indigenous introductions, development of contingency plans, including eradication protocols, and inclusion of information on how to respond to a non-indigenous species introduction within protected area management plans. Influence of environmental conditions on Collembolan species We already commented that Collembola presence depends on different environmental factors, mainly soil moisture, organic matter, and vegetation cover (Tilbrook 1967a; Hop kin 1997; Adams et al. 2006; Sinclair et al. 2006; Day et al. 2009; Russell et al. 2014). However, our findings do not point to the two first variables as key factors for the distribution and abundance of Barrientos’ Collembola. This could be because soil moisture is not usually a problem on this island because there is abundant rainfall in liquid form throughout the austral summer. In addition, the small size of the island facilitates the influence of moisture-laden sea breezes, as a result soils here rarely suffer water stress. Soil organic matter neither is a limiting factor for Collembola at this site owing to its high concentrations in the study sites, especially in vegetated areas (Online Resource 4). The third environmental variable, vegetation cover, cannot be considered as a limitation problem in an island that precisely stands out because of its high coverage of vegetation communities. However, other environmental variables, such as elevation or phosphorous content, are important for Collembola on this site. Regarding elevation, a certain altitude in Barrientos means a greater abundance of vegetation, because of the island there is a greater availability of habitats far from the coast, since the shoreline is vegetation-free except for some algae. This trend is not common on other larger Antarctic islands where glaciers immediately appear as one moves inland, and favorable environments for Collembola disappear rapidly. In the case of phosphorous, a higher concentration of this nutrient can also contributes to a greater food availability for Collembola, mainly algae. On Barrientos Island, environments dominated by the algae Prasiola crispa, which usually is present close to nutrient-rich areas such as penguin colonies, appeared to favor Collembola owing to these sites have a highest species richness and density. Cryptopygus antarcticus antarcticus has a marked preference for this algae, although it feeds on a wide variety of food, being able to take advantage of different nutrients present in algae, mosses, and lichens, but also filamentous fungi (Broady 1979) and dead organic material (Bokhorst et al. 2007). Behind its predilection for P. crispa is the greater nitrogen concentration present in this algae regarding lichens. Although this nutrient is present in mosses in similar concentrations, they contain inhibitory and refractory compounds which are difficult for soil arthropods to digest. Therefore, mosses are less consumed by C. a. antarcticus when both types of food, algae and mosses, are present in the environment (Davidson and Broady 1996). This explains the high presence of this species in alga sheet (Online Resource 2). This species plays an essential role in the nutrient flow of the environments in the maritime Antarctica which are dominated by algae and mosses (Bokhorst et al. 2007). Cryptopygus antarcticus antarcticus Was very common in our samples, being alga sheet and bryophyte carpets and mats the substrates with higher average species richness. Data suggest that other non-recorded environmental variables could be important for certain species. For example, Cryptopygus badasa, the other species of the genus Cryptopygus present on the island, appeared in most of the sampled substrate types with the exception of footpaths. This may be because it does not tolerate excess surface temperature (Russell et al. 2014) as is in the case of exposed bare soil in footpaths. Although this variable was not recorded during sampling, the observed abundance pattern supports this possibility In addition to the environmental factors, other variables must be taken into account when interpreting the abundance results obtained for each habitat. This is the case of the trophic strategy of the Collembola. The genus Friesea is a perfect example of this question, since species belonging to it are usually predators. On Barrientos Island, the species with the largest representation for this genus was F. grisea. Its maximum presence (3056 specimens/ m7”) was detected in a sample from a wallow. Wallows used by southern elephant seals are terrestrial nutrient enrichment areas through deposits of feces, urine, and molted skin from them (Panagis 1985). Nematodes are strongly related to soil organic matter since many of them are detritivores (Yergeau et al. 2007). Hence, the probability of finding nematodes in wallows is high. In fact, the presence of these invertebrates was observed when wallows’ samples were processed (unpublished data). Therefore, it is not surprising that F. grisea abundance was elevated in this site. This species was present in almost all types of substrates, although with low densities for substrates other than wallows. This indicates that F. grisea was widely distributed on Barrientos Island, although with low densities typical for a predator. Friesea bispinosa and F. woyciechowskii, the other two species of this genus, were less abundant than F. grisea, only 109 and 327 individuals/m’, respectively, were recorded during three field campaigns. Therefore their relative importance in the community of Collembola on this island is much lower. Morphological issues also may be considered to explain the collembolan distribution. Folsomotoma octooculata is an epedaphic species that has a longer furca than other species recorded in this study, allowing it to have greater ability to colonize less accessible areas, including steeper areas, hence elevation and slope favor this species. Therefore, it is not surprising that this species is present in all the habitats analyzed on the island. Even taking into account all the issues raised above, we must emphasize that we have tried to be cautious with the interpretation of the results concerning the distribution of springtails in the different environments of the island. The reason is that a stratified sampling was used, with sampling efforts that varied according to the type of substrate and their representativeness and importance on Barrientos Island. Besides, samples for the environmental study were only recorded in 2013 campaign, resulting in a slated representation of substrate types (6/8) and species (7/10). Therefore, results should be compared with caution since they are not based on a similar sampling effort. Besides, Collembola have a natural tendency for aggregation and patchy distribution (Ims et al. 2004), as well as population fluctuations from year to year (Pollierer and Scheu 2017). All this makes difficult to identify trends in their distribution, discouraging the assumption of strict axioms regarding this issue.  Finally, our study has also generated unexpected results regarding the limitations on the presence of certain species because of environmental conditions. Archisotoma brucei is usually classified as strictly marine-littoral species, like all species in this genus, although individuals have been recorded in large numbers in traps set supralittorally on Heard Island and in Tasmania presumably blown in sea spray (Greenslade 2010). In Barrientos, it was recorded in 2012 on a footpath about 100 meters away from the shoreline. While this is an unusual location for the species, other studies have also found specimens of A. brucei at some distance from the shore. For example, Gressitt (1967) collected one specimen from rocks on a raised terrace immediately around a chinstrap penguin nest. Tilbrook (1967b) cited some specimens on an ash plane near shore, but not on the beach. More recently, Russell et al. (2014) found specimens on Devil Island in soil substrates. Accordingly, it is possible that this species will be able to colonize areas at some distance from the shoreline if soil texture and other environmental factors are suitable, not being so restricted to the shoreline as was thought (Greenslade and van Klinken 1994). Future studies may be developed to test this hypothesis. Conclusions The present study increases the knowledge on the distribution of Antarctic Collembola, generating the first list of species for Barrientos Island, a location that has not been subject to a systematic census until now despite being a traditional tourist destination in the South Shetland Islands. Although all recorded species had already been cited in Antarctica, two non-indigenous species were found—Hypogastrura viatica and Mesaphorura macrochaeta—expanding the list of Antarctic sites in which exotic collembolans have been located. In spite of these presences, the local Collembola community seems to be relatively healthy in view of the obtained results, with a high richness (ten species) for Antarctic standards. The environmental information obtained was limited and did not allow going deeper in the patterns that govern the distribution of the springtails in the island. Nevertheless, certain environmental variables have increased the list of factors that may be important for these organisms, such as elevation, phosphorous content, or slope. These factors are probably directly related to the number of available habitats for Collembola. This confirms the importance of sampling in different environments, and at different times if possible, in order to obtain a more complete species census when you work in Antarctic sites. Acknowledgements Logistical support for this work was provided by the Ecuadorian and the Spanish National Antarctic Programs, and the Spanish Navy. Fieldwork and analysis for this paper were developed under three projects: EVA-ANTARCTICA (CTM2009-06604-E) and ALIENANT (CTM2013-47381-P), which were supported by the Spanish Government, and Proposal for Tourism Management of Barrientos Island and Surroundings (2011-2013), funded by the Ecuadorian Government. The authors would like to thank José Olmedo Moran, Executive Director of the Ecuadorian National Antarctic Program, for his advice and support for this work; Daniela Cajiao for her help in the fieldwork during the 2012 and 2013 campaigns; Dr. Luis R. Pertierra for his collaboration in the 2012 campaign; Laura Mufioz for her help in the identification of mosses; Dr. Manuela Andrés-Abellan and Consuelo Wic for the analysis of soil samples in the laboratories of the Universidad de Castilla-La Mancha; and Dr.Tina Tin for her constructive and helpful comments during the editing process. We also would like to thank the anonymous reviewers and the Editor for their helpful comments, which have improved this work. Compliance with ethical standards Conflict of interest The authors declare that they have no conflict of interest. References Adams B, Bardgett RD, Ayres E, Wall DH, Aislabie J, Bamforth S, Bargagli R, Cary C, Cavacini P, Connell L, Convey P, Fell J, Frati F, Hogg I, Newsham N, O’ Donnell A, Russell N, Seppelt R, Stevens MI (2006) Diversity and distribution of Victoria Land biota. Soil Biol Biochem 38:3003—3018. https://doi.org/10.1016/j. soilbio.2006.04.030 Antarctic Treaty Secretariat (2015) Site Guidelines for Visitors. Barrientos Island. http://www.ats.aq/e/ats_other_siteguidelines.htm. Accessed 10 Feb 2015 Baena CW, Andrés-Abellan M, Lucas-Borja ME, Martinez-Garcia E, Garcia-Morote FA, Rubio E, Lépez-Serrano FR (2013) Thinning and recovery effects on soil properties in two sites of a Mediterranean forest, in Cuenca Mountain (South-eastern of Spain). For Ecol Manage 308:223-230. https://doi.org/10.1016/j. foreco.2013.06.065 Bastida F, Kandeler E, Moreno JL, Ros M, Garcia C, Hernandez T (2008) Application of fresh and composted organic wastes modifies structure, size and activity of soil microbial community under semiarid climate. App! Soil Ecol 40:318—329. https://doi. org/10.1016/j.apsoil.2008.05.007 Bokhorst S, Ronfort C, Huiskes A, Convey P, Aerts R (2007) Food choice of Antarctic soil arthropods clarified by stable isotope signatures. Polar Biol 30:983-990. https://doi.org/10.1007/ s00300-007-0256-4 Bremner JM (1965) Nitrogen availability indexes. In: Black CA et al (eds) Methods of Soil Analysis, Part 2 Agronomy 9. American Society of Agronomy, Madison, pp 1324-1345 Broady PA (1979) Feeding studies on the Collembolan Cryptopygus antarcticus Willem at Signy Island, South Orkney Islands. Br Antarct Surv Bull 48:37-46 Carpenter GH (1909) On some subantarctic Collembola. Article XVII. In: Chilton C (ed) The subantarctic islands of New Zealand. Philosophical Institute of Canterbury, Christchurch, pp 377-383 Carpenter GH, Evans W (1906) Scottish National Antarctic Expedition. “Scotia” collections. Collembola from the South Orkney Islands. P Roy Soc Edinb 26:473-483. https://doi.org/10.1017/ S0370164600024718 Chown SL, Convey P (2007) Spatial and temporal variability across life’s hierarchies in the terrestrial Antarctic. Phil Trans Roy Soc ser B 362:2307—2331. https://doi.org/10.1098/rstb.2006.1949 Convey P (2008) Non-native species in Antarctic terrestrial and freshwater environments: presence, sources, impacts and predictions. In: Rogan-Finnemore M (ed) Non-native species in the Antarctic, proceedings. Gateway Antarctica, Christchurch, pp 97-130 Convey P, Greenslade P, Arnold RJ, Block W (1999) Collembola of sub-Antarctic South Georgia. Polar Biol 22:1-6. https://doi. org/10.1007/s003000050383 Convey P, Greenslade P, Pugh PJA (2000) The terrestrial microarthropod fauna of the South Sandwich Islands. J Nat Hist 34:597-609. https://doi.org/10.1080/002229300299462 Davidson MM, Broady PA (1996) Analysis of gut contents of Gomphiocephalus hodgsoni carpenter (Collembola: Hypogastruridae) at Cape Geology, Antarctica. Polar Biol 16:463—467. https://doi.org/10.1007/BF02329064 Day TA, Ruhland CT, Strauss SL, Park JH, Krieg ML, Krna MA, Bryant DM (2009) Response of plants and the dominant microarthropod, Cryptopygus antarcticus, to warming and contrasting precipitation regimes in Antarctic tundra. Glob Chang Biol 15:1640-1651. https://doi. org/10.1111/j.1365-2486.2009.01919.x Deharveng L (1981) Collemboles des Iles Subantarctiques de 1°Océan Indien mission J. Travé 1972-1973. Comité National Francais des Recherches Antartiques 48:33-108 Dibbern JS (2010) Fur seals, whales and tourists: a commercial history of Deception Island, Antarctica. Polar Rec 46:210—221. https:// doi.org/10.1017/S003224740900865 1 Frenot Y, Chown SL, Whinam J, Selkirk PM, Convey P, Skotnicki M, Bergstrom DM (2005) Biological invasions in the Antarctic: extent, impacts and implications. Biol Rev 80:45—72. https://doi. org/10.1017/S 1464793 104006542 Garcia C, Gil F, Hernandez T, Trasar C (2003) Técnicas de analisis de parametros bioquimicos en suelos: Medida de actividades enzimaticas y biomasa microbiana. CEBAS-CSIC, Murcia Greenslade P (1987) Invertebrate conservation in the Antarctic and subantarctic. In: Major JD (ed) The Role of Invertebrates in Conservation and Biological Survey. Western Australian Department of Conservation and Land Management Report, Kensington, pp 119-121 Greenslade P (1995) Collembola from the Scotia Arc and Antarctic Peninsula including descriptions of two new species and notes on biogeography. Pol Pismo Entomol 64:305-—319 Greenslade P (2006) The Invertebrates of Macquarie Island. Australian Antarctic Division, Department of Environment and Heritage, Kingston Greenslade P (2010) Collembola fauna of the South Shetland Islands revisited. Antarct Sci 22:233-242. https://doi.org/10.1017/ S095410200999071X Greenslade P, Convey P (2012) Exotic Collembola on subantarctic islands: pathways, origins and biology. Biol Invasions 14:405-— 417. https://doi.org/10.1007/s10530-01 1-0086-8 Greenslade P, van Klinken RD (1994) Marine littoral and coastal Collembola on the subantarctic Macquarie Island. Acta Zool Fenn 195:58-61 Greenslade P, Wise KAJ (1984) Additions to the collembolan fauna of the Antarctic. Trans R Soc South Aust 108:203—205 Greenslade P, Potapov M, Russell D, Convey P (2012) Global Collembola on Deception Island. J Insect Sci 12:1—16. https://doi. org/10.1673/031.012.11101 Greenslade P, Ireson J, Skarzyriski D (2014) Biology and key to the Australian species of Hypogastrura and Ceratophysella (Collembola: Hypogastruridae). Austral Entomol 53:53-—74. https://doi. org/10.1111/aen.12048 Gressitt JL (1967) Notes on arthropod populations in the Antarctic Peninsula -South Shetland Islands- South Orkney Islands area. Antarct Res Ser 10:373-391. https://doi.org/10.1029/ARO10p037 Hack WH (1949) Nota sobre un Colémbolo de la Antartida Argentina Achorutes viaticus Tullberg. Notas del Museo de la Plata 14:211-212 Hedo J, Lucas-Borja ME, Wic C, Andrés-Abellan M, De Las Heras J (2015) Soil microbiological properties and enzymatic activities of long-term post-fire recovery in dry and semiarid Aleppo pine (Pinus halepensis M.) forest stands. Solid Earth 6:243-252. https://doi.org/10.5194/se-6-243-2015 Hertzberg K, Leinaas HP, Ims RA (1994) Patterns of abundance and demography: Collembola in a habitat patch gradient. Ecography 17:349-359. https://doi.org/10.1111/j.1600-0587.1994. tb00112.x Hogg ID, Cary SC, Convey P, Newsham K, O’Donnell T, Adams BJ, Aislabie J, Frati FF, Stevens MI, Wall DH (2006) Biotic interactions in Antarctic terrestrial ecosystems: are they a factor? Soil Biol Biochem 38:3035-—3040. https://doi.org/10.1016/j. soilbio.2006.04.026 Hopkin SP (1997) Biology of the Springtails (Insecta: Collembola). Oxford University Press, Oxford Hughes KA, Pertierra LR, Molina-Montenegro MA, Convey P (2015) Biological invasions in terrestrial Antarctica: what is the current status and can we respond? Biodivers Conserv 24:1031-— 1055. https://doi.org/10.1007/s 1053 1-015-0896-6 Ims RA, Leinaas HP, Coulson S (2004) Spatial and temporal variation in patch occupancy and population density in a model system of an arctic Collembola species assemblage. Oikos 105:89-100. https://doi.org/10.1111/j.0030-1299.2004.12634.x International Association of Antarctica Tour Operators (2015) Tourism statistics. http://iaato.org/tourism-statistics. Accessed 5 Aug 2016 Jordana R, Arbea JI (1989) Clave de identificacion de los géneros de Colémbolos de Espafia (Insecta: Collembola). Publicaciones de biologia de la Universidad de Navarra. Serie Zoolégica 19:1—32 Jordana R, Arbea JI, Simon C, Lucidfiez MJ (1997) Collembola, Poduromorpha. In: Ramos MA et al (eds) Fauna Ibérica. Museo Nacional de Ciencias Naturales. CSIC, Madrid Kandeler E, Stemmer M, Klimanek E (1999) Response of soil microbial biomass, urease and xylanase within particle size fractions to long-term soil management. Soil Biol Biochem 31:261-—273. https://doi.org/10.1016/S0038-0717(98)00115-1 Krivolutsky DA, Lebedeva NV, Gavrilo MV (2004) Soil microarthropods in the feathers of Antarctic birds. Dokl Biol Sci 397:342— 345. https://doi.org/10.1023/B:DOBS.0000039712.34478 .f6 Lebedeva NV, Krivolutsky DA (2003) Birds spread soil microarthropods to Arctic islands. Dokl Biol Sci 391:329-332. https://doi. org/10.1023/A:1025150500875 Lubbock J (1876) On a new genus and species of Collembola from Kerguelen Island. Ann Mag Nat Hist 18:324 Lucas-Borja ME, Bastida F, Nicolas C, Moreno JL, Del Cerro A, Andrés M (2010) Influence of forest cover and herbaceous vegetation on the microbiological and biochemical properties of soil under Mediterranean humid climate. Eur J Soil Biol 46:273-279. https://doi.org/10.1016/j.ejsobi.2010.05.003 Olsen SR, Cole CV, Watanabe FS, Dean LA (1954) Estimation of available phosphorus in soils by extraction with sodium bicarbonate. United States Department of Agriculture, Washington, DC Panagis K (1985) The influence of elephant seals on the terrestrial ecosystem at Marion Island. In: Siegfried WR, Condy PR, Laws RM (eds) Antarctic nutrient cycles and food web. Springer, Berlin, pp 173-179 Pollierer MM, Scheu S (2017) Driving factors and temporal fluctuation of Collembola communities and reproductive mode across forest types and regions. Ecol Evol 7:4390-4403. https://doi. org/10.1002/ece3.3035  Potapov M (1991) Antarctophorus—a new genus of Isotomidae (Collembola) from Antarctica. Rev Ecol Biol Sol 28:491-495 Potapov M (2001) Synopses on Palaearctic Collembola: Isotomidae. Abhandlungen und Berichte des Naturkundemuseums GoGrlitz, Gorlitz R Core Team (2017) R: A language and environment for statistical computing. R Foundation for Statistical Computing, Vienna, Austria. http://www.R-project.org/. Accessed 2017 Russell DJ, Hohberg K, Potapov M, Bruckner A, Otte V, Christian A (2014) Native terrestrial invertebrate fauna from the northern Antarctic Peninsula: new records, state of current knowledge and ecological preferences—Summary of a German federal study. Soil Organ 86:1-58 Salmon JT (1949) The Collembola of the United States Antarctic Service Expedition, 1939-41. Proc Roy Entomol Soc Lond ser B 18:161-162. https://doi.org/10.1111/j.1365-3113.1949. tb01440.x Salmon JT (1962) A new species and redescriptions of Collembola from Antarctica. Pac Insects 4:887—-894 Salmon JT (1974a) Notes and drawings from type material of Collembola. Zool Publ Vic University Wellingt 66:1—41 Salmon JT (1974b) Concerning the Collembola Tullbergiinae. Zool Publ Vic University Wellingt 67:1—19 Sinclair BJ, Scott MB, Klok CJ, Terblanche JS, Marshall DJ, Reyers B, Chown SL (2006) Determinants of terrestrial arthropod community composition at Cape Hallett, Antarctica. Antarct Sci 18:303-312. https://doi.org/10.1017/S0954 102006000356 Tabatabai MA (1982) Soil enzymes. In: Page AL, Miller EM, Keeney DR (eds) Methods of soil analysis, Part 2. Chemical and microbiological properties. Soil Science Society of America, Inc., Madison, pp 903-947 Tejedo P, Benayas J, Cajiao D, Albertos B, Lara F, Pertierra LR, Andrés-Abellan M, Wic C, Luciafiez MJ, Enriquez N, Justel A, Reck GK (2016) Assessing environmental conditions of Antarctic footpaths to support management decisions. J Environ Manag 177:320-330. https://doi.org/10.1016/j.jenvman.2016.04.032 Tilbrook PJ (1967a) Arthropod ecology in the maritime Antarctic. Antarct Res Ser 10:331-356. https://doi.org/10.1029/ ARO10p0331 Tilbrook PJ (1967b) The terrestrial invertebrate fauna of the maritime Antarctic. Phil Trans Roy Soc Lond ser B 252:261-278 Vance ED, Brookes PC, Jenkinson DS (1987) An extraction method for measuring soil microbial biomass C. Soil Biol Biochem 19:703-707. https://doi.org/10.1016/0038-07 17(87)90052-6 Wahlgren E (1906) Antarktische und Subantarktische Collembolen gesammelt von der schwedischen Sudpolarexpedition. Wiss Ergeb Schwed Sudpolar Exped 9:1—22 Weiner WM (1980) Friesea woyciechowskii sp. n. (Collembola, Neanuridae) from King George Island (South Shetland Islands, Antarctica). Pol Pismo Entomol 50:491—495 Willem V (1901) Les Collemboles recueillis par 1“Expédition antarctique belge. Annales de la Société Entomologique de Belgique 45:260-262 Wise KAJ (1964) New records of Collembola and Acarina in Antarctica. Pac Insects 6:522—-523 Wise KAJ (1967) Collembola (Springtails). Antarct Res Ser 10:123147. https://doi.org/10.1029/AR010p0123 Wise KAJ (1970a) Collembola of South Georgia. Pac Insects Monogr 23:183-—208 Wise KAJ (1970b) Collembola of Heard Island. Pac Insects Monogr 23:209-215 Wise KAJ (1971) The Collembola of Antarctica. Pac Insects Monogr 25:57-74 Witteveen J, Verhoef HA, Letschert JPW (1987) Osmotic and ionic regulation in marine littoral Collembola. J Insect Physiol 33:5966. https://doi.org/10.1016/0022-1910(87)90105-3 Yeomans J, Bremner JM (1988) A rapid and precise method for routine determination of organic carbon in soil. Commun Soil Sci Plant Anal 19:1467-1476. https://doi. org/10.1080/00103628809368027 Yergeau E, Bokhorst S, Huiskes AH, Boschker HT, Aerts R, Kowalchuk GA (2007) Size and structure of bacterial, fungal and nematode communities along an Antarctic environmental gradient. FEMS Microbiol Ecol 59:436—451. https://doi. org/10.1111/).1574-6941.2006.00200.'"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenising Document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T00:51:27.224565Z",
     "start_time": "2021-02-11T00:51:25.547334Z"
    }
   },
   "outputs": [],
   "source": [
    "#using small Spacy model\n",
    "document = nlp(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T00:51:31.479689Z",
     "start_time": "2021-02-11T00:51:28.575937Z"
    }
   },
   "outputs": [],
   "source": [
    "#using medium Spacy model\n",
    "document_med = nlp_md(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T04:16:05.834009Z",
     "start_time": "2021-02-26T04:16:03.943338Z"
    }
   },
   "outputs": [],
   "source": [
    "#using large Spacy model\n",
    "document_lg = nlp_lg(doc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Named Entity Recognition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using small model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T22:38:37.138584Z",
     "start_time": "2021-01-26T22:38:37.133310Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Barrientos Island', 'GPE'), ('Antarctica', 'LOC'), ('first', 'ORDINAL'), (\"Natalia Enriquez'\", 'ORG'), ('Javier Benayas', 'PERSON'), (\"Maria José Luciafiez'\", 'PERSON'), ('21', 'CARDINAL'), ('13', 'CARDINAL'), ('3', 'CARDINAL'), ('Springer', 'ORG'), ('Barrientos Island', 'GPE'), ('South Shetland', 'LOC'), ('Antarctic', 'NORP'), ('two', 'CARDINAL'), ('first', 'ORDINAL'), ('Collembola', 'PERSON'), ('Barrientos Island', 'FAC'), ('Fifty-six', 'CARDINAL'), ('three seasons,', 'DATE'), ('2011-2013', 'DATE'), ('eight', 'CARDINAL'), ('39', 'CARDINAL'), ('Collembola', 'ORG'), ('ten', 'CARDINAL'), ('Antarctica', 'LOC'), ('Cryptopygus', 'ORG'), ('Willem', 'PERSON'), ('Collembola', 'PERSON'), ('Two', 'CARDINAL'), ('Tullberg', 'GPE'), ('Hypogastruridae', 'NORP'), ('Mesaphorura', 'PERSON'), ('Collembola', 'PERSON'), ('Collembola', 'ORG'), ('Antarctic', 'NORP'), ('Keywords Aitcho Islands - Non', 'PERSON'), ('Barrientos Island', 'GPE'), ('62', 'CARDINAL'), ('59', 'CARDINAL'), (\"°47'W\", 'PERSON'), ('Robert', 'PERSON'), ('Greenwich Islands', 'GPE'), ('South Shetland', 'GPE'), ('Electronic', 'GPE'), ('Antarctica', 'LOC'), ('Fig', 'PERSON'), ('1', 'CARDINAL'), ('Natalia Enriquez Departamento de Biologia', 'WORK_OF_ART'), ('Universidad', 'GPE'), ('Madrid', 'GPE'), ('C/ Darwin 2,', 'PERSON'), ('28049', 'DATE'), ('Madrid', 'GPE'), ('Spain', 'GPE'), ('Departamento de Ecologia', 'ORG'), ('Universidad', 'GPE'), ('Madrid', 'GPE'), ('C/ Darwin 2,', 'PERSON'), ('Madrid', 'GPE'), ('Spain', 'GPE'), ('Javier Benayas', 'PERSON'), ('Departamento de Botanica', 'ORG'), ('Geologia', 'GPE'), ('Universidad', 'GPE'), ('@uam.es de Valencia', 'PERSON'), ('Avda', 'ORG'), ('Vicente Andrés', 'PERSON'), ('Spain', 'GPE'), ('Maria José Luciaiiez', 'PERSON'), ('13', 'CARDINAL'), ('357500', 'CARDINAL'), ('3078000', 'CARDINAL'), ('Chinstrap Penguins', 'PERSON'), ('Chinstrap', 'ORG'), ('Gentoo Penguins', 'PERSON'), ('Gentoo Penguins', 'PERSON'), ('3077500', 'CARDINAL'), ('13.43/44', 'CARDINAL'), ('11', 'CARDINAL'), ('3', 'CARDINAL'), ('3077000', 'CARDINAL'), ('357500', 'CARDINAL'), ('358000', 'CARDINAL'), ('UTM Projection', 'ORG'), ('3078000', 'CARDINAL'), ('1', 'CARDINAL'), ('4', 'CARDINAL'), ('3077500', 'CARDINAL'), ('3077000', 'CARDINAL'), ('Pacific Ocean', 'LOC'), ('Fig.1', 'ORG'), ('Barrientos Island', 'GPE'), ('2011', 'DATE'), ('2012', 'DATE'), ('2013', 'DATE'), ('13-01', 'CARDINAL'), ('13', 'CARDINAL'), ('Antarctica', 'LOC'), ('8663', 'CARDINAL'), ('2015—2016', 'GPE'), ('austral summer season', 'DATE'), ('International Association of Antarctica Tour Operators 2015', 'ORG'), ('the Antarctic Treaty Consultative Meeting', 'ORG'), ('ATCM', 'ORG'), ('2005', 'DATE'), ('the ATCM’s Committee for Environmental Protection', 'ORG'), ('CEP', 'ORG'), ('Two', 'CARDINAL'), ('Collembola', 'PERSON'), ('2016', 'DATE'), ('Antarctica', 'LOC'), ('Salmon 1949', 'PERSON'), ('1964', 'DATE'), ('1971', 'DATE'), ('Greenslade', 'ORG'), ('Wise 1984', 'DATE'), ('Potapov 1991', 'EVENT'), ('Greenslade 1995', 'EVENT'), ('Convey', 'ORG'), ('2000', 'DATE'), ('Collembola', 'PERSON'), ('Antarctica', 'PRODUCT'), ('the Antarctic Peninsula', 'LOC'), ('Willem 1901', 'PERSON'), ('Wahlgren 1906', 'EVENT'), ('Carpenter', 'PERSON'), ('Evans 1906', 'ORG'), ('Weiner 1980', 'PERSON'), ('1995', 'DATE'), ('2010', 'DATE'), ('Greenslade et al. 2012', 'PERSON'), ('Russell et al.', 'PERSON'), ('2014', 'DATE'), ('Barrientos Island', 'LOC')]\n"
     ]
    }
   ],
   "source": [
    "#Spacy finds these ENTS in the first 1000 characters of the document\n",
    "print([(X.text, X.label_) for X in document[:1000].ents])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T22:38:46.748311Z",
     "start_time": "2021-01-26T22:38:46.742291Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'GPE': 97,\n",
       "         'LOC': 65,\n",
       "         'ORDINAL': 14,\n",
       "         'ORG': 231,\n",
       "         'PERSON': 280,\n",
       "         'CARDINAL': 245,\n",
       "         'NORP': 53,\n",
       "         'FAC': 7,\n",
       "         'DATE': 196,\n",
       "         'WORK_OF_ART': 3,\n",
       "         'EVENT': 3,\n",
       "         'PRODUCT': 15,\n",
       "         'QUANTITY': 7,\n",
       "         'PERCENT': 5,\n",
       "         'LAW': 1})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Which types of named entites are recognised in the document?\n",
    "labels = [x.label_ for x in document.ents]\n",
    "Counter(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T22:38:49.180114Z",
     "start_time": "2021-01-26T22:38:49.171788Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Collembola', 49), ('Antarctic', 29), ('Antarctica', 24)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#The most common words labelled as named entities \n",
    "items = [x.text for x in document.ents]\n",
    "Counter(items).most_common(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T22:38:58.552252Z",
     "start_time": "2021-01-26T22:38:58.521673Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ORDINAL \n",
      " {'second', 'first', 'third'} \n",
      "\n",
      "NORP \n",
      " {'Collembolan', 'Yeomans', 'Cryptopygus', 'Hypogastruridae', 'Median', 'German', 'Ecuadorian', 'Lucas', 'C.', 'Australian', 'M.', 'Pygoscelis', 'Antarctic', 'Macronectes', 'Birds', 'Tullbergiidae'} \n",
      "\n",
      "GPE \n",
      " {'South Shetland', 'Spain', 'H. viatica', 'the Universidad de Castilla-La Mancha', 'Antarctica', 'Austria', 'Tabatabai MA', 'Lichtenstein', 'Mpa', 'Mesaphorura', 'Biotic', 'Electronic', 'Suunto', 'H.', 'Rogan', 'Luciafiez', 'Vienna', '2015—2016', 'Japan', 'Barrientos', 'Vance et al.', 'Arbea', 'South Orkney Islands', 'New Zealand', 'Leptonychotes', 'Tasmania', 'Tullberg', 'Baena', 'Barrientos Island', 'Results Fauna', 'Bamforth S', 'Tabatabai', 'Berlin', 'Victoria', 'Geologia', 'Tullbergia', 'The Invertebrates of Macquarie Island', 'Adams BJ', 'Prasiola', 'Washington, DC', 'Forster', 'Greenwich Islands', 'Deception Island', 'Nota', 'Antarct', 'Lépez-Serrano', 'Oxford', 'Verhoef HA', 'South Shetland Islands', 'Madrid', 'Online', 'Universidad', 'Wahlgren', 'Kerguelen Island', 'Devil Island', 'Thinning', 'Hypogastrura', 'Berichte', 'Georgia', 'Acta Zool', 'Kensington'} \n",
      "\n",
      "CARDINAL \n",
      " {'357500', '200', '11', 'only 109', '1.5', '7/8', '669', '15', '2000', 'Fifty-six', '601', '300', '601 68', '2 3', '0.029', '417', '13', '11-01', '3078000', '817', '0.010', '16.540', '377', '0.414', '98¢', 'Two', '8570', '81', 'three', '12-01', '106', '2014', '0.000', '1/5', '0.095', 'five', '1970b', 'ten', 'nine', 'six', '0.011', '873', '3056', 'one', '781', '0.017', 'Only two', '164', '0.645', '13-01', '1000', '8', '0.348', '3', 'eight', '28', '5', '1“Expédition', '13.43/44', '5/8', '0.543', 'seven', '62', '7', 'four', '0.033', 'only one', '1-0086', '4', '0.021', '6 5', '358000', '21', '0.013', '0.456', '903', '120', '12', 'Thirty-nine', '0.012', 'Eight', '39', '8663', '59', '6/8', '105:89-100', '0.467', '97-130', '6941.2006.00200', '8/8', '3 3', 'about 115', '3077000', '0.009', 'between four and seven', '48:37', '46:210—221', '1', '119', '0.014', '37', '0.343', '48,000', '0.05', '3077500', '0.032', '70', '7/10', '332', '1onic', '0.015', '5 2', 'fewer than 100', '6.744', '0.350', '0.022', '362:2307—2331', '0.055', '56', '173', '19', '6', '2017', '0.030', 'two', '36', '327', '2', '0.382', '34', '0.003', 'Six', '0', '1.728'} \n",
      "\n",
      "EVENT \n",
      " {'Wahlgren 1906', 'Greenslade 1995', 'Potapov 1991'} \n",
      "\n",
      "DATE \n",
      " {'2004', '1964', '1876', '2000', '2003', '1995', 'Peters, 1875', 'at least 7 days', '1962, 1974a', '2011-2013', '2015', 'November to March', '2013', 'February 7, 2011', '20 days', '1995, 2006', 'Broady 1979', '1985', '102006000356', 'weekly', '1997', '1053 1-015-0896-6', '2014', '1972-1973', '1988', '10 Feb 2015', '2012', '1781', '1982', '1984', '2005', '1906', '397:342— 345', '13-01', '1981', '1967, 1970a', 'the first 4 days', 'each year', '2001', '12:1—16', '2011', '2009', '1965', 'three seasons,', '1823', 'Deharveng 1981', '1939-41', '2006', 'Bremner 1988', '1991', '1996', '1970a', '2016', '2007', '1826', '1907', '1789', 'kin 1997', '1954', 'Day et al. 2009', '24', '17:349-359', 'austral summer season', '1967', 'the early nineteenth century', '1976', '1967b', 'Tilbrook 1967b', '1464793 104006542', '1989', '1872', '28049', '1949', '1971', 'February 16, 2012', '2010', '1909', '2—3 days', '1987', 'Table 1', '2008', '1891', '1999', '1901', '1994', 'Wise 1984', '2017', 'the 3 years', 'Panagis 1985', 'Tilbrook 1967a', 'February 2013', '1980'} \n",
      "\n",
      "LAW \n",
      " {'Antarctic Treaty Secretariat'} \n",
      "\n",
      "LOC \n",
      " {'South Shetland', 'Antarctica', 'Macquarie Island', 'Osmotic', 'Heard Island', 'Antarctic Treaty', 'the Southern Hemisphere', 'South Orkney Islands', 'Antarctic Peninsula', 'Mediterranean', 'Barrientos Island', 'Pacific Ocean', 'Marion Island', 'Signy Island', 'Arctic', 'Deception Island', 'Antarctic Macquarie Island', 'South Shetland Islands', 'Cuenca Mountain', 'South Georgia', 'the South Orkney Islands', 'South Sandwich Islands', 'the Antarctic Peninsula'} \n",
      "\n",
      "WORK_OF_ART \n",
      " {'’ Donnell A,', 'Polytrichastrum', 'Natalia Enriquez Departamento de Biologia'} \n",
      "\n",
      "FAC \n",
      " set() \n",
      "\n",
      "PERCENT \n",
      " {'90%', '50-60%', '99%', '2 %', '95%'} \n",
      "\n",
      "QUANTITY \n",
      " {'about 100 meters', '8 cm', '2 £', '500 cm', '1.5 kg', '2 mm', '1.5 km'} \n",
      "\n",
      "PERSON \n",
      " {'Salmon', 'Russell N', 'Nicolas C', 'Mesaphorura', 'Neanuridae', 'Hertzberg K', 'Murcia\\n', 'Frenot et al. 2005', 'Hernandez T', 'Strauss SL', '°C', 'Naturkundemuseums GoGrlitz', 'Deharveng', 'Hopkin 1997', 'Newsham K', 'Chinstrap Penguins', 'Warnstorfia', 'Exotic Collembola', 'Li', 'Andrés-Abellan M', 'Salmon JT (', 'Cape Hallett', 'Soil Biol', 'Sanionia', 'Loeske', 'Phil Trans Roy Soc Lond', 'Ros M', 'Garcia C', 'Maria José Luciaiiez', 'Wahlgren E (', 'Bargagli R', 'Carpenter', '@uam.es de Valencia', 'Bastida et al. 2008', 'Laura Mufioz', 'Roy Soc Edinb 26:473-483', 'S.', 'Martinez-Garcia', 'Chown', 'Arbea JI', 'Gentoo Penguins', 'J. Travé', 'Salmon 1949', 'Kandeler E', 'Witteveen et al.', 'Jo', 'Deharveng L', 'Gressitt JL', 'JO N', 'Consuelo Wic', 'Tina Tin', 'Serie Zoolégica', 'Aleppo', 'Hohberg K', 'Davidson MM', 'Wise KAJ', 'A.', 'Ims et al.', 'Hack WH', 'Boschker HT', 'Cajiao D,', 'Jordana et al.', 'Folsomotoma', 'Joquinu', 'Daniela Cajiao', 'Garcia-Morote FA', 'Scott MB', 'Gil F', 'Arthropod', 'Caloplaca', 'Block W', 'Lara F', 'Skotnicki M', 'Collembola', 'C/ Darwin 2,', 'G.L. Sm', 'Fig', 'Javier Benayas', 'Park JH', 'Sinclair et al. 2006', 'Scheu S', 'The Collembola of Antarctica', \"°47'W\", 'Garcia et al.', 'Kandeler E,', 'Gateway Antarctica', 'B. Mey', 'Deharveng,', 'JoLe', 'Arnold RJ', 'Phil Trans Roy Soc', 'Weiner', 'Hedo et al.', 'JO Jaquunu', 'Newsham N', 'Russell et al.', 'Yergeau et al.', 'Lesson', \"Maria José Luciafiez'\", 'Lubbock J', 'Olsen SR', 'Kowalchuk GA', 'Greenslade', 'Hopkin SP', 'Potapov', 'M.', 'al', 'Klok CJ', 'Potapov M', 'Keywords Aitcho Islands - Non', 'Hertzberg et al.', 'Bruckner A', 'Biological', 'Adams et al. 2006', 'Robert', 'Krivolutsky', 'Bastida F', 'Willem 1901', 'Krivolutsky et al.', 'Ochyra', 'G', 'Hughes et al.', 'P. crispa', 'Ireson J', 'Aislabie J', 'Bokhorst et al.', 'Chown SL', 'Environ Manag 177:320-330', 'Wic C', 'G. Gaertn', 'Panagis K', 'Bokhorst S', 'De Las Heras J', 'Wi Jod', 'Madison', 'Acarina', 'Wiss Ergeb Schwed', 'Lightfoot', 'Olsen et al.', 'Ims et al. 2004', 'Evans W', 'Xanthoria', 'Greenslade et al. 2012', 'Weiner 1980', 'Luis R. Pertierra', 'Ann Mag Nat', 'Ceratophysella', 'Kandeler et al.', 'Trans R Soc South Aust', 'Springer', 'Jenkinson DS', 'Res Ser', 'F.', 'Rusek', 'Adams B', 'Fauna Ibérica', 'C. Soil', 'Vicente Andrés', 'Davidson', 'Pol Pismo', 'Evol 7:4390-4403', 'Jordana R', 'Frenot Y', 'Lubbock', 'Jordana', 'The Collembola', 'Glob Chang Biol', 'Archisotoma', 'Manuela Andrés-Abellan', 'Dokl Biol', 'Proc Roy Entomol Soc Lond', 'Witteveen J', 'Medida de actividades', 'José Olmedo Moran', 'Hogg et al.', 'Gressitt', 'Bremner', 'Biology', 'Hedo J', 'Willem', 'Hal', 'Greenslade et al.', 'Seppelt R'} \n",
      "\n",
      "ORG \n",
      " {'FEMS Microbiol Ecol 59:436—451', 'Baena CW', 'Mirounga', 'Mesaphorura', 'Scottish National Antarctic Expedition', 'Gavrilo MV (2004', 'Tilbrook PJ (1967a', 'the ATCM’s Committee for Environmental Protection', 'Antarctic (Frenot et al.', 'Andrés', 'CEP', '°C', 'Russell D,', 'Krieg ML', 'Siegfried WR', 'Stevens MI', 'Museo Nacional de Ciencias Naturales', 'MM', 'Gmelin', 'DINIDIA', 'Gorlitz', 'Mohr’s', 'Lebedeva', 'Tullgren', 'Dean LA', 'Barrientos’ Collembola', 'Departamento de Botanica', 'Broady PA (', 'Antarctic Collembola', 'Kjeldahl', 'Leinaas HP', 'Mill', 'Australian Antarctic Division', 'NaOH', 'the Spanish National Antarctic Programs', 'Evans 1906', 'UI', 'Online Resource 3', 'Notas del Museo de la Plata', 'Carpenter', 'Cryptopygus', 'Comité National Francais des Recherches', 'Bryum', 'Pertierra LR', 'Convey P', 'Pygoscelis', 'CSIC', 'Cary SC', 'Zool Publ Vic University', 'Abhandlungen', 'SPSS', 'Connell L,', 'the Spanish Government', 'Cape Geology', \"F LOL'OT\", 'Tullbergia', 'Lebedeva NV', 'WY', 'Letschert JPW', 'CO', 'INT', 'Fgrisea', 'Whinam J', 'Watanabe FS', 'Christchurch', 'the Ecuadorian National Antarctic Program', 'Benayas J', 'Collembola', 'C/N-relationship', 'Friesea', 'Huiskes', 'OS', 'Poduromorpha', 'Online Resource 4', 'Major JD', 'Fell J', 'the United States Antarctic Service Expedition', 'Philosophical Institute of Canterbury', 'SPSS, Inc.', 'Lucidfiez MJ (1997', 'Terblanche JS', 'Western Australian Department of Conservation and Land Management Report', 'SNITNIADJUD SNITNIADJUD', 'Online Resource 2', 'Fig.1', 'Stemmer M', 'Barrientos Island (', 'Yergeau E', 'Frati FF', 'Greenslade', 'Toray', 'ATCM', 'CTM2013-47381-P', 'the Ecuadorian Government', 'Departamento de Ecologia', 'Bremner JM', 'United States Department of Agriculture', 'Chinstrap', 'Soil Science Society of America, Inc.', 'Antarktische', 'Miller EM', 'R Foundation for Statistical Computing', 'Insect Sci', 'Barrientos', 'DaSids Vasals', 'Animals', 'Hogg ID', 'Kingston\\nGreenslade', 'Sudpolarexpedition', 'Weddell', 'International Association of Antarctica Tour Operators', 'the Spanish Navy', 'Online Resource 5', 'Hughes KA', 'Justel A', 'Lucas-Borja ME', 'Chown SL (', 'Spearman’s', 'Kruskal—Wallis', 'Krna MA', 'MOTTE', 'K', 'Bryant DM', 'Bokhorst S', 'Condy PR', 'Rubio E', 'Ronfort C', 'Subantarktische Collembolen', 'Ochyra & Hedenis', 'Chilton C', 'Springer', 'Schaffer', 'Marshall DJ', 'Carpenter GH', 'Keeney DR', 'Russell DJ', 'Chemical', 'Avda', 'Selkirk', 'ALIENANT', 'Oxford University Press', 'Coulson S', 'Vance ED', 'Biol Invasions', 'UTM Projection', 'Conservation and Biological Survey', 'the Antarctic Treaty Consultative Meeting', 'Hoyer', \"Natalia Enriquez'\", 'Miill', 'International Association of Antarctica Tour Operators 2015', 'Archisotoma', 'UMOUS Ose', '& Scherb', 'Polar Rec', 'Toray Engineering Co. Ltd.', 'Simon C', 'de Colémbolos de Espafia', 'Proposal for Tourism Management of Barrientos Island', 'Convey', 'Aerts R', 'Cary C', 'Ruhland CT', 'IBM', 'Sinclair BJ', 'Enriquez N', 'Larus', 'Navi Horiba', 'Willem', 'XVII', 'Department of Environment and Heritage', 'Colémbolo de la Antartida Argentina Achorutes', 'Nitrogen'} \n",
      "\n",
      "PRODUCT \n",
      " {'Prasiola', 'Surroundings', 'Online Resource 2', 'Del Cerro A', 'Antarctica', 'Online Resource 4', 'Cole', 'Polar Biol 22:1-6', 'Notes', 'Friesea', 'Soil', 'Laws RM', 'Soil Biol Biochem 31:261-—273', 'Soil Biol Biochem'} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "#A more detailed look at which words are being picked up as certain types of entites.\n",
    "# There are some obvious errors, especially as how species are being classed\n",
    "\n",
    "for entity in list(set(labels)):\n",
    "    print(entity,'\\n', set([x.text for x in document.ents if x.label_ is entity]), '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using medium model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T22:39:10.507936Z",
     "start_time": "2021-01-26T22:39:10.502663Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'ORG': 236,\n",
       "         'ORDINAL': 9,\n",
       "         'GPE': 29,\n",
       "         'CARDINAL': 286,\n",
       "         'PRODUCT': 46,\n",
       "         'DATE': 74,\n",
       "         'PERSON': 115,\n",
       "         'WORK_OF_ART': 12,\n",
       "         'NORP': 11,\n",
       "         'FAC': 3,\n",
       "         'LAW': 4,\n",
       "         'EVENT': 3,\n",
       "         'QUANTITY': 1,\n",
       "         'PERCENT': 6,\n",
       "         'TIME': 2,\n",
       "         'LOC': 2})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Which types of named entites are recognised in the document?\n",
    "labels_med = [x.label_ for x in document_med.ents]\n",
    "Counter(labels_med)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T22:39:22.534521Z",
     "start_time": "2021-01-26T22:39:22.506123Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ORDINAL \n",
      " {'first', 'third'} \n",
      "\n",
      "NORP \n",
      " {'Murcia', 'German', 'Arthropod', 'Ecuadorian', 'Mirounga', 'Pacific', 'Relationships', 'Antarctic', '%'} \n",
      "\n",
      "CARDINAL \n",
      " {'357500', '25:57', '2—3', '200', '11', 'only 109', '1964', '1.5', '1053', '669', '15', '1939', '2000', '0.403', '2003', '1995', '0.365', '67:1—19', '300', '2 3', '0.029', '3078000', '817', '10', '6vI', '0.010', '0.414', '0.434', '3 December 2017', 'a5', '102006000356', '0.399', '98¢', '20', 'at least', 'Two', '1967a', '81', '278', '12-01', '106', '2 1', '2015—2016', '2014', '0.000', '691', '1/5', '1988', '0.095', '2012', '12-11', '873', '68', '3056', 'one', '781', '66:1—41', '1906', '164', '0.645', '13-01', '1000', '8', '0.348', '3', '97', '28', '5', '391', '13.43/44', '345', '2011', '1962', '0.543', '500', '62', '7', 'Oo', '1823', '0.033', '1974b', '06', '2006', '4', '0.021', '21', '0.456', '120', '0.411', '16:463—467', '0.012', '39', '59', '6/8', '0.467', 'activities', '1954', '24', '8/8', '240', '1967b', '80', '11-06', 'about 115', '3077000', '0.009', '1989', '28049', '1', '1949', '0.014', '37', '0.343', '48,000', '0.05', 'about 100', '3077500', '0.032', '70', '903-947', '7/10', '105:89', '0.015', '3 5', '6.744', '0.350', '0.022', '362:2307—2331', '1999', '1901', 'Forster', '0.055', '56', '358500', '19', '6', '173', '2017', '0.030', 'two', '01', '36', '327', '119-', '84', '2', '0.382', '34', '0.003', 'Six', '0', '1.728'} \n",
      "\n",
      "GPE \n",
      " {'Spain', 'Biologia', 'Universidad', 'Mesaphorura', 'Acknowledgements', 'Tasmania', 'Tullberg', 'Baena', 'Pertierra LR', 'Received', 'Conflict', 'Berlin', 'Vienna', 'Madrid', 'Block', 'Published'} \n",
      "\n",
      "LAW \n",
      " {'Soil Science Society of America, Inc.', 'the Spanish Navy', 'February 7, 2011', 'February 16, 2012'} \n",
      "\n",
      "DATE \n",
      " {'1985', '1758', '1997', '1872', '1875', '1949', 'm7', '8570', '2010', 'Spearman', '1909', 'a lesser extent', '1987', '1996', '1781', '2007', '2008', '1907', '1891', '1995', '1982', '1984', '2005', '1901', '1994', '1906', '1789', 'strict', '8663', '1981', '1954', '2001', 'being', '2013', '358000 358500', '2011', '1976', '1980'} \n",
      "\n",
      "EVENT \n",
      " {'February 2013', 'Mediterranean humid climate', 'International Association of Antarctica Tour Operators'} \n",
      "\n",
      "LOC \n",
      " {'Adams', 'Birds'} \n",
      "\n",
      "WORK_OF_ART \n",
      " {'C. Soil', 'Macquarie Island', 'Scotia', 'Deception Island (Hack 1949', 'Methods of Soil Analysis, Part', 'Heard Island', 'José Olmedo Moran, Executive Director of the Ecuadorian National Antarctic Program', 'South Shetland archi Electronic', 'Yeomans and Bremner 1988', 'Deception Island', 'Daniela Cajiao', 'Pollierer and Scheu 2017'} \n",
      "\n",
      "TIME \n",
      " {'2013', '48:33-108'} \n",
      "\n",
      "FAC \n",
      " set() \n",
      "\n",
      "PERCENT \n",
      " {'90%', '50-60%', '99%', '2 %', '2 oo', '95%'} \n",
      "\n",
      "QUANTITY \n",
      " {'1.5 kg'} \n",
      "\n",
      "PERSON \n",
      " {'Compliance', 'Yergeau', 'Table', 'Austria', 'Aq', 'Nicolas C', 'J. Travé', 'Keeney', '2 Agronomy', 'Buryjassot', 'Therefore', 'Data', 'Rogan-Finnemore', 'wallow', 'Deharveng', 'Influence', 'Fauna Ibérica', 'Biological', 'Barrientos', 'Penguin Rookery', 'Austral Entomol', 'Ims', 'Arbea', 'Kandeler', 'Antarctic Peninsula', 'Li', 'Davidson MM', 'Cape Geology', 'Krivolutsky', 'Comité National Francais', 'Rev Ecol', 'Antarctic', 'Folsomotoma', 'Sanionia', 'Distribution', 'Hughes KA', 'Ochyra', 'Antarctic Collembola', 'Vista', 'Lubbock', 'Jordana', 'Leinaas HP', 'Manuela Andrés', 'Arctic', 'Collembola', '§', 'Wic C', 'Maria José Luciaiiez', 'Zool Publ', 'Antarct Sci', 'Wi Jod', 'Ireson', 'Madison', 'JD', 'Gressitt', 'B. Mey', 'Aislabie', 'Laura Mufioz', 'Xanthoria', 'Klinken', 'Antarct Res Ser', 'Plata', 'Enriquez Departamento', 'Wallows', 'Arbea JI'} \n",
      "\n",
      "ORG \n",
      " {'Russell N', 'Lépez-Serrano FR', 'Luciafiez MJ', 'Frati F', 'Scottish National Antarctic Expedition', 'Neanuridae', 'Gavrilo MV', 'Bardgett RD', 'Krieg ML', 'Stevens MI', 'G.L. Sm.', 'Gmelin', 'Exotic Collembola', 'Antarctic Peninsula', 'Mohr’s', 'Lucas', 'Lebedeva', 'Tullgren', 'Dean LA', 'Collembola of Heard Island', 'Canterbury,', 'Kjeldahl', 'CrossMark', 'Loeske and S.', 'Leinaas HP', 'Mill', 'Australian Antarctic Division', 'UI', 'Cary C, Cavacini P', 'Carpenter', 'Cryptopygus', 'Polar Biol 30:983-', 'ED', 'Bryum', 'Devil Island', 'Antarctophorus', 'Pertierra LR', 'Sinclair', 'Colémbolos de Espafia', 'Convey P', 'Ciencias Naturales', 'Pygoscelis', 'Gateway Antarctica, Christchurch', 'Cary SC', 'Arbea JI', 'Collembola and Acarina', 'Bokhorst', 'Gentoo Penguins', 'Bergstrom DM', 'the Spanish Government', 'Tilbrook', 'Heritage', 'Aleppo', 'Hohberg K', 'Baena', 'Marion Island', 'Bamforth S', 'Fig.1 Map of Barrientos Island', 'Hack WH', 'Macronectes', 'Boschker HT', 'Keywords', 'Tullbergia', 'Colémbolo', 'Lebedeva NV', 'Prasiola', 'Letschert JPW', 'Caloplaca', 'INT', 'Whinam J', 'Christchurch', 'Collembola', 'Dokl Biol Sci', 'Friesea', 'Huiskes AH', 'Poduromorpha', 'Cole CV', 'Kerguelen Island', 'DASA', '°', 'Arnold RJ', 'Pol Pismo Entomol', 'Weiner', 'Lesson', 'Online Resource 2', 'Osmotic', 'Philosophical Institute of', 'Frati FF', 'Greenslade', 'Toray', 'Hopkin SP', 'Linnaeus', 'M.', 'the Ecuadorian Government', 'Ann Mag', 'Bremner JM', 'Potapov M', 'Barrientos', 'DaSids Vasals', 'New Zealand', 'Moreno JL', 'Hogg ID', 'Sudpolarexpedition', 'Robert', 'Reck GK', 'Tabatabai', 'Hopkin', 'Justel A', 'Department of Environment', 'Evans', 'Kruskal—Wallis', 'Aislabie J', 'Krna MA', 'DH', 'Chown SL', 'Gorlitz\\n', 'Verhoef HA', 'Greenslade and Convey 2012', 'Castilla-La Mancha', 'Connell L', 'Cuenca Mountain', 'Bokhorst S', 'Bryant DM', 'Condy PR', 'Ronfort C', 'Researchers', 'the South Orkney Islands', 'the South Sandwich Islands', 'Hop', 'Ochyra & Hedenis', 'the Antarctic Peninsula', 'Marshall DJ', 'Carpenter GH', 'Notes', 'Seppelt', 'Peters', 'Insect Physiol', 'the South Shetland Islands', 'Chemical', 'Selkirk', 'ALIENANT', 'Benayas', 'October 2016 /', 'Kandeler', 'Coulson S', 'Biol Invasions', 'Microbiol Ecol 59:436—451', 'Barrientos Island', 'Antarctic', 'Antarctic (', 'Adams BJ', \"Natalia Enriquez'\", 'Miill', 'King George Island', 'Black CA', 'Frenot', 'UMOUS Ose', '& Scherb', 'Toray Engineering Co. Ltd.', 'Antarctic tourists', 'Simon C', 'Springer-Verlag GmbH Germany', 'Convey', 'De Las Heras', 'Lucidfiez MJ', 'Ruhland CT', 'Hypogastrura', 'IBM', 'Enriquez N', 'Navi Horiba', 'Christian A', 'Biology of the Springtails'} \n",
      "\n",
      "PRODUCT \n",
      " {'Online Resource 2', 'Antarctica', 'Hypogastruridae', 'Schaffer', 'Kowalchuk GA', 'Core Team', 'Core Team 2017', 'Soil Organ 86:1-58', '377-383', '2012', 'Soil', 'CJ', 'Polytrichastrum', 'Lara F', 'Skarzyriski', 'Online Resource 3', 'Antarct Sci', 'Del Cerro A', 'Insect Sci 12:1—16', 'Vegan', 'Convey P'} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "#A more detailed look at which words are being picked up as certain types of entites.\n",
    "# There are some obvious errors, especially as how species are being classed\n",
    "\n",
    "for entity in list(set(labels_med)):\n",
    "    print(entity,'\\n', set([x.text for x in document_med.ents if x.label_ is entity]), '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using large model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T22:39:32.187126Z",
     "start_time": "2021-01-26T22:39:32.180715Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'GPE': 112,\n",
       "         'ORDINAL': 19,\n",
       "         'PERSON': 351,\n",
       "         'CARDINAL': 228,\n",
       "         'DATE': 239,\n",
       "         'ORG': 181,\n",
       "         'PRODUCT': 32,\n",
       "         'LOC': 81,\n",
       "         'LANGUAGE': 4,\n",
       "         'WORK_OF_ART': 9,\n",
       "         'NORP': 22,\n",
       "         'MONEY': 8,\n",
       "         'QUANTITY': 11,\n",
       "         'FAC': 5,\n",
       "         'LAW': 2,\n",
       "         'EVENT': 4,\n",
       "         'PERCENT': 6,\n",
       "         'TIME': 1})"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Which types of named entites are recognised in the document?\n",
    "labels_lg = [x.label_ for x in document_lg.ents]\n",
    "Counter(labels_lg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T22:39:44.687283Z",
     "start_time": "2021-01-26T22:39:44.645416Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ORDINAL \n",
      " {'first', 'second', 'Oo © 6', 'third', 'Hypogastruridae - Basic', 'Tullbergiidae'} \n",
      "\n",
      "NORP \n",
      " {'Collembolan', 'FEMS', 'German', 'CO', 'Arthropod', 'Ecuadorian', 'Mirounga', 'Biotic', 'Scottish', 'Pollierer', 'Australian', 'Native', 'F.', 'Pygoscelis', 'Antarctic', 'Macronectes', 'Antarctic Treaty', 'Tullbergia'} \n",
      "\n",
      "CARDINAL \n",
      " {'5 3', '200', 'yoRa', '11', 'only 109', '1.5', '7/8', '1053', '669', '099', '15', 'Fifty-six', '0.403', '601', '22:1', '300', '0.029', '417', '13', '3078000', '817', '0.010', '377', '0.414', '0.434', 'Two', '1967a', '8570', '81', 'three', '106', '48:33', '0.000', '691', '1/5', '1-0086-8\\n', 'five', 'ten', 'nine', 'six', '0.011', '12-11', '873', '3056', 'one', '781', 'Only two', '164', '0.645', '1000', '0.348', '8', '3', 'eight', '40:318—329', '5', 'CrO-', '160', '345', '5/8', '0.543', 'seven', '62', '7', 'four', '0.033', '1974b', 'only one', '4', '0.021', '21S', '21', '0.456', '903', '120', '0.411', 'Thirty-nine', '0.012', '6S', 'Eight', '39', '8663', '6/8', '0.467', '97-130', '8/8', '240', '3 3', '1967b', '80', '0.009', 'between four and seven', '28049', '1', '119', '0.343', '48,000', '0.05', '3077500', '0.032', '9', '59:436—451', '7/10', '1974a', '1onic', '0.015', '3 5', 'fewer than 100', '6.744', '0.350', '0.022', '9¢', '0.055', '56', '173', '358500', '6', '19', '0.030', 'two', '36', '327', 'https://doi', '84', '2', '0.382', '0.003', 'Six', '0', '1.728', '1055'} \n",
      "\n",
      "GPE \n",
      " {'South Shetland', 'Spain', 'H. viatica', 'doi.org/10.1017/S003224740900865 1', 'Antarctica', 'Austria', 'Lichtenstein', 'Mesaphorura', 'Hertzberg K', 'colonies', 'Washington, DC\\nPanagis K', 'Seppelt', 'Keeney DR', 'Vienna', 'Japan', 'South Orkney Islands', 'Coulson S', 'New Zealand', 'Aleppo', 'Tasmania', 'Tullberg', 'Cape Geology', 'Baena et al', 'Barrientos Island', 'Bamforth S', 'Victoria Land', 'Hertzberg et al', 'Berlin', 'Cape Hallett', 'Geologia', 'Tullbergia', 'Kingston', 'Frenot et al.', 'Catharacta', 'Prasiola', 'Murcia', 'Scotia', 'Lubbock', 'Klimanek', 'Whinam J', 'Signy Island', 'Kruskal—Wallis', 'Aislabie J', 'Mill', 'Greenwich Islands', 'Collembola', 'Christchurch', 'Skotnicki M', 'Antarct', 'Oxford', 'Hedo et al', 'Frenot et al', 'South Shetland Islands', 'Cuenca Mountain', 'Madrid', 'Bargagli R', 'Bokhorst S', 'Scheu S', 'Del Cerro A', 'Madison', 'Universidad', 'Wahlgren', 'Kerguelen Island', 'Devil Island', 'Gateway Antarctica', 'Hypogastrura', 'Georgia', 'et al', 'Pygoscelis', 'Kensington'} \n",
      "\n",
      "LAW \n",
      " {'the Antarctic Treaty Consultative Meeting', 'Salmon 1949'} \n",
      "\n",
      "DATE \n",
      " {'1758', '1875', '2004', '1964', '1876', '2000', 'annual', '2003', '1995', '80S', 'Convey 2008', 'at least 7 days', '2011-2013', '2015', '11-01', 'November to March', '3078000', '2013', 'February 7, 2011', 'the 2015—2016 austral summer season', '20 days', '3 December 2017', '1985', '102006000356', 'weekly', '1997', '12-01', '2014', '1972-1973', '1988', '10 Feb 2015', 'a 1-week period', '2012', '1781', '5 Aug 2016', '1982', '1984', '2005', '1906', '13-01', '1981', 'Wise 1964, 1971', 'October 2016', 'the first 4 days', 'each year', '2001', '2011', '1962', '2009', '1965', '1823', '1939-41', '2006', 'three seasons', '6 5', 'the austral summer season', '1991', '1970a', '1996', '2016', '2007', '1826', '1979', '1907', '39', 'December 2017', '1954', '24', '1967', 'the early nineteenth century', '1976', 'Convey 2012', 'the austral summer', '1967b', 'yDJIDYIOAIDUL', '1464793 104006542', '1989', '1872', '1949', '1971', 'February 16, 2012', '2010', '1909', '2—3 days', '1987', '2008', '1891', '1999', '1901', '1994', 'Wise 1984', '2017', 'the 3 years', 'cL', '34', '2 1 0', 'February 2013', '1980'} \n",
      "\n",
      "LANGUAGE \n",
      " {'Hypogastruridae', 'Polytrichastrum'} \n",
      "\n",
      "LOC \n",
      " {'Antarctica', 'Macquarie Island', 'Heard Island', 'Potapov M', 'the South Shetland Islands', 'the Southern Hemisphere', 'South Orkney Islands', 'Antarctic Peninsula', 'Mediterranean', 'Marion Island', 'Antarctic', 'King George Island', 'Arctic', 'Deception Island', 'South Shetland Islands', 'Fig', 'Pinus', 'Antarctic South', 'South Georgia', 'the South Orkney Islands', 'the South Sandwich Islands', 'Navi Horiba', '°24’S', 'Solid Earth', 'the Antarctic Peninsula'} \n",
      "\n",
      "EVENT \n",
      " {'Mesaphorura', 'Potapov 1991'} \n",
      "\n",
      "MONEY \n",
      " {'6$ + cL', 'CEBAS-CSIC', '60b', '2 £', '37 °C', \"59°47'W\", '16.540', '1324-1345\\nBroady PA'} \n",
      "\n",
      "WORK_OF_ART \n",
      " {'Tilbrook', 'Vegan', 'Antarct Sci 22:233-242', 'Antarct Sci 18:303-312', 'The Role of Invertebrates in Conservation and Biological Survey', 'a Canonical Correspondence Analysis', 'Rusek (Tullbergiidae)—were', '2 IN &', 'Collemboles des Iles Subantarctiques de 1°Océan Indien'} \n",
      "\n",
      "TIME \n",
      " {'19:1—32\\n'} \n",
      "\n",
      "FAC \n",
      " set() \n",
      "\n",
      "PERCENT \n",
      " {'90%', '50-60%', '99%', '2 %', '9S07', '95%'} \n",
      "\n",
      "QUANTITY \n",
      " {'4 of een le\\n', '70 m.', 'about 100 meters', '4 °C', '8 cm', '500 cm', '28 °C', '1.5 kg', '2 mm', '1.5 km'} \n",
      "\n",
      "PERSON \n",
      " {'Yergeau', 'Nicolas C', 'Luciafiez MJ', 'Mesaphorura', 'yOoys e3[V\\n', 'Vicente Andrés Estellés', 'Pugh PJA', 'Garcia et al', 'Natalia Enriquez', 'Hernandez T', 'Sinclair et al', 'Bardgett RD', 'Deharveng', 'Hopkin 1997', 'Krieg ML', 'Krivolutsky DA', 'Stevens MI', 'Convey et al.', 'Siegfried WR', 'AL', 'Skarzyriski D', 'Andrés-Abellan M', 'Lebedeva', 'Tullgren', 'Russell et al', 'Mohr', 'Sanionia', 'Hughes et al', 'T. mixta', 'Aq\\n', 'Antarctic Collembola', 'Leinaas HP', 'Loeske', 'Phil Trans Roy Soc Lond', 'Ros M', 'Garcia C', 'Maria José Luciaiiez', 'Selkirk PM', 'Carpenter', 'Cryptopygus', '@uam.es de Valencia', 'JD', 'Bryum', 'Gomphiocephalus', 'Laura Mufioz', 'Dr', 'Cary SC', 'Pablo Tejedo', 'S.', 'Arbea JI', 'J. Travé', 'C.', 'van Klinken RD', 'Witteveen et al.', 'Jo', 'Bergstrom DM', 'JO N', 'Tilbrook', 'Consuelo Wic', 'Tina Tin', 'Serie Zoolégica', 'Belén Albertos', 'Universidad Aut6noma de pablo.tejedo', 'Hohberg K', 'de Biologia', 'Hack WH', 'Ims et al.', '1970b', 'Folsomotoma', 'Daniela Cajiao', 'Vance et al', 'Joquinu', 'Bokhorst et al', 'Garcia-Morote FA', 'Scott MB', 'Gil F', 'Watanabe FS', 'Abhandlungen und Berichte des', 'Collembola', 'Kiitzing', 'Biodivers Conserv', 'Fig', 'Friesea', 'Javier Benayas', 'Adams et al', 'Huiskes AH', 'Park JH', 'Otte V', 'B. Mey', 'Russell D', 'Arnold RJ', 'Phil Trans Roy Soc', 'Terblanche JS', 'Weiner', 'Albertos B', 'Donnell A', 'Pol Pismo Entomol', 'JO Jaquunu', 'Rogan-Finnemore M', 'Reyers B', \"Maria José Luciafiez'\", 'Exped', 'Universidad Aut6noma de\\n', 'Frati FF', 'Kowalchuk GA', 'Jordana et al', 'van Klinken', 'Greenslade', 'Yeomans J', 'Potapov', 'M.', 'Miller EM', 'Yergeau et al', 'Bruckner A', 'Barrientos', 'Glob Chang Biol 15:1640-1651', 'Arbea', 'Moreno JL', 'Martinez-Garcia E', 'KAJ', '1970a', 'Robert', 'Day et al', 'Krivolutsky', 'Monogr 23:209-215', 'Bastida F', 'Hughes KA', 'Willem 1901', 'Lucas-Borja ME', 'Krivolutsky et al.', 'Ochyra', 'Evans', 'Pac Insects Monogr', 'MOTTE', 'G. Gaertn', 'Huiskes A', 'Ramos MA et al', 'Bokhorst S', 'Ecol Evol 7:4390-4403', 'Condy PR', 'Rubio E', 'Wiss Ergeb Schwed', 'Global Collembola', 'Lightfoot', 'Olsen et al.', 'Hogg et al', 'Xanthoria', 'Evans W', '1967', 'Antarct Res Ser', 'Publicaciones de biologia de la Universidad de Navarra', 'Luis R. Pertierra', 'Ceratophysella', 'Ui ctose@ ee', 'Kandeler et al', 'Fenn 195:58-61\\nGreenslade P', 'Peters', 'Greenslade et al', 'Rusek', 'Russell DJ', 'Avda', 'Dibbern', 'Adams B', 'Fauna Ibérica', 'Acta', 'Ann Mag Nat Hist', 'Kandeler', 'Aitcho Islands', 'Davidson', 'EVA-ANTARCTICA', 'Palaearctic Collembola', 'Wi Jod s[enprAIpul Jo', 'Pac Insects 4:887—-894\\nSalmon JT', 'sadA', 'Adams BJ', 'Darwin 2', 'Greenslade P', 'Hoyer', \"Natalia Enriquez'\", 'Miill', 'Jordana', 'The Collembola', 'Forster', 'Hypogastrura viatica', '© ©', 'Archisotoma', 'UMOUS Ose', 'Manuela Andrés-Abellan', 'Simon C', 'Medida de actividades', 'José Olmedo Moran', 'Lucas-Borja et al', 'Gressitt', 'Wahlgren', 'Cary C', 'Lucidfiez MJ', 'Bremner', 'Molina-Montenegro MA', 'Sinclair BJ', 'Enriquez N', 'Willem', 'Bastida et al.', 'Hal', 'Ayres', 'Christian A'} \n",
      "\n",
      "ORG \n",
      " {'the Universidad de Castilla-La Mancha', 'Salmon', 'Russell N', 'Baena CW', 'Lépez-Serrano FR', 'Chilton C (ed', 'Strauss SL', 'CEP', 'Committee for Environmental Protection', 'Soil Biol Biochem', 'G.L. Sm.', 'Gmelin, 1789)', 'Newsham K', 'Museo Nacional de Ciencias Naturales', 'Warnstorfia', 'DINIDIA', 'Notas del', 'Andrés-Abellan M', 'SPSS© Statistics', 'Dean LA', 'COE\\nLL', 'Antarctic Collembola', 'Kjeldahl', 'Australian Antarctic Division', 'NaOH', 'the Spanish National Antarctic Programs', 'Collembola (Springtails', 'Co + BIZ LOT', 'Antarctophorus', 'Pertierra LR', 'Convey P', 'CSIC', 'Aerts', 'Broady 1979', 'Tabatabai MA', 'Collembola and Acarina', 'Trasar C', 'Técnicas de analisis de parametros', 'the Spanish Government', 'Weiner WM', 'Urease', 'Boschker HT', 'Tullbergia', 'Lebedeva NV', 'Folsomotoma', 'Proposal for Tourism Management of Barrientos Island and Surroundings', 'Tilbrook PJ', 'Letschert JPW', 'INT', 'Fgrisea', 'Block W', 'the Ecuadorian National Antarctic Program', 'Benayas J', 'Collembola', 'Broady PA', 'Dokl Biol Sci', 'Microbiol', 'American Society of Agronomy', 'Friesea', 'OS', 'Poduromorpha', 'INA', 'The Collembola of Antarctica', 'Fell J', 'Philosophical Institute of Canterbury', 'SPSS, Inc.', 'DASA J DaSids Vasals', 'the Scotia Arc', 'Brookes PC', 'Weiner, 1980', 'Western Australian Department of Conservation and Land Management Report', 'Frati F, Hogg I,', 'Lesson', 'Canonical Correspondence Analysis', 'Osmotic', 'Stemmer M', 'Hopkin SP', 'Toray', 'ATCM', 'Linnaeus', 'CTM2013-47381-P', 'Departamento de Ecologia', 'Suunto', 'CTM2009', 'the Ecuadorian Government', 'the Collembolan Cryptopygus', 'Potapov M', 'United States Department of Agriculture', 'Klok CJ', 'Soil Science Society of America, Inc.', 'R Foundation for Statistical Computing', 'CL9', 'Spearman', 'Ecol Manage', 'Isotomidae', 'Discussion Barrientos Island’s', 'Hogg ID', 'UTM Projection WGS', 'Sudpolarexpedition', 'Antarctic Treaty Secretariat', 'Tabatabai', 'Weddell', 'International Association of Antarctica Tour Operators', 'Reck GK', 'Bastida F', 'the Spanish Navy', 'Justel A', 'YoRd UI', 'Ireson J', 'Kruskal—Wallis', 'Krna MA', 'Chown SL', 'Verhoef HA', 'Wic C', 'Belén Albertos 46100 Buryjassot', 'Connell L', 'Birds', 'Bryant DM', 'De Las Heras J', 'Ronfort C', 'C/N', 'Salmon JT', 'Spatial', 'Ochyra & Hedenis', 'States Antarctic Service Expedition', 'Trans R Soc South Aust', 'Broady 1996', 'Schaffer', 'Springer', 'Marshall DJ', 'Carpenter GH', 'Jenkinson DS', 'Cavacini P', 'Comité National Francais des Recherches Antartiques', 'Clave de', 'Ims RA', 'Gorlitz\\nR Core Team (', 'O’Donnell T', 'Yeomans', 'Chemical', 'vIssaq]yN', 'C. Soil Biol Biochem', 'ALIENANT', 'Oxford University Press', 'Vance ED', 'CLET', 'Jordana R', 'Frenot Y', 'Toray Engineering Co. Ltd.', 'de Colémbolos de Espafia', 'Springer-Verlag GmbH Germany', 'Panagis 1985', 'Aerts R', 'Ruhland CT', 'Bremner', 'IBM', 'Department of Environment and Heritage'} \n",
      "\n",
      "PRODUCT \n",
      " {'Notes', 'Res Ser', 'Potapov M', 'Ss aD', 'Soil Organ 86:1-58', 'Soil Biol Biochem 31:261-—273', 'Soil Biol Biochem', 'Polar Biol 16:463—467', 'Spearman', '391:329-332', '© SS 2 °', 'Springer Nature 2017\\n', 'Prasiola', 'https://doi.org/10.1007/ s00300-007-0256-4\\nBremner JM', 'Cole', 'Lara F', 'Polar Biol 30:983-990', 'Garmin eTrex Vista', 'Commun Soil Sci Plant Anal 19:1467-1476', 'Dokl Biol Sci', '10:331-356', 'Convey P', 'Biol Rev 80:45—72'} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "#A more detailed look at which words are being picked up as certain types of entites.\n",
    "# There are some obvious errors, especially as how species are being classed\n",
    "\n",
    "for entity in list(set(labels_lg)):\n",
    "    print(entity,'\\n', set([x.text for x in document_lg.ents if x.label_ is entity]), '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-06T22:40:31.242589Z",
     "start_time": "2021-01-06T22:40:31.240436Z"
    }
   },
   "source": [
    "### Comparing Location-identification between Spacy models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T22:40:00.911230Z",
     "start_time": "2021-01-26T22:40:00.902830Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Small \n",
      " {'South Shetland', 'Antarctica', 'Macquarie Island', 'Osmotic', 'Heard Island', 'Antarctic Treaty', 'the Southern Hemisphere', 'South Orkney Islands', 'Antarctic Peninsula', 'Mediterranean', 'Barrientos Island', 'Pacific Ocean', 'Marion Island', 'Signy Island', 'Arctic', 'Deception Island', 'Antarctic Macquarie Island', 'South Shetland Islands', 'Cuenca Mountain', 'South Georgia', 'the South Orkney Islands', 'South Sandwich Islands', 'the Antarctic Peninsula'} \n",
      "\n",
      "Medium \n",
      " {'Adams', 'Birds'} \n",
      "\n",
      "Large \n",
      " {'Antarctica', 'Macquarie Island', 'Heard Island', 'Potapov M', 'the South Shetland Islands', 'the Southern Hemisphere', 'South Orkney Islands', 'Antarctic Peninsula', 'Mediterranean', 'Marion Island', 'Antarctic', 'King George Island', 'Arctic', 'Deception Island', 'South Shetland Islands', 'Fig', 'Pinus', 'Antarctic South', 'South Georgia', 'the South Orkney Islands', 'the South Sandwich Islands', 'Navi Horiba', '°24’S', 'Solid Earth', 'the Antarctic Peninsula'} \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model, name in zip([document, document_med, document_lg], ['Small', 'Medium', 'Large']):\n",
    "    print(name,'\\n', set([x.text for x in model.ents if x.label_ is 'LOC']), '\\n')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting Locations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gazetteers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Two gazetteers have been loaded so far;\n",
    "\n",
    "1) the NZ gazetteer from LINZ using just the subset of locations relating to Antartica, and\n",
    "\n",
    "2) the SCAR gazetteer with all nations' recognised names for places in Antartica.\n",
    "\n",
    "The New Zealand-related names in the SCAR list have been isolated to create a smaller subset of the larger SCAR list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:25:28.901920Z",
     "start_time": "2021-02-28T01:25:28.368007Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/myfriendscallmej/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3057: DtypeWarning: Columns (28,29,32,34) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "nzgaz = pd.read_csv(\"gaz_names.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:25:33.493892Z",
     "start_time": "2021-02-28T01:25:33.081434Z"
    }
   },
   "outputs": [],
   "source": [
    "SCARGlobal = pd.read_csv(\"SCAR_CGA_PLACE_NAMES_GLOBAL_SIMPLIFIED.csv\")\n",
    "SCARNZ = pd.read_csv(\"SCAR_CGA_PLACE_NAMES_NZ_SIMPLIFIED.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:25:34.797234Z",
     "start_time": "2021-02-28T01:25:34.752179Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FID</th>\n",
       "      <th>place_name_mapping</th>\n",
       "      <th>place_name_gazetteer</th>\n",
       "      <th>gaz_id</th>\n",
       "      <th>altitude</th>\n",
       "      <th>feature_type_name</th>\n",
       "      <th>narrative</th>\n",
       "      <th>gazetteer</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>geometry</th>\n",
       "      <th>scar_common_id</th>\n",
       "      <th>country_name</th>\n",
       "      <th>country_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Abbott Peak</td>\n",
       "      <td>Abbott Peak</td>\n",
       "      <td>112592</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Peak</td>\n",
       "      <td>Pyramidal peak on Ross Island, on the N side o...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.4333</td>\n",
       "      <td>167.0000</td>\n",
       "      <td>POINT (167 -77.433333)</td>\n",
       "      <td>8</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Abbott</td>\n",
       "      <td>Abbott, Mount</td>\n",
       "      <td>112593</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A mountain 1,020 m, which stands 3 mi NE of Ca...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-74.7000</td>\n",
       "      <td>163.8333</td>\n",
       "      <td>POINT (163.833333 -74.7)</td>\n",
       "      <td>9</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Abus Valley</td>\n",
       "      <td>Abus Valley</td>\n",
       "      <td>112594</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Valley</td>\n",
       "      <td>An ice-free valley 3 mi SE of Turnstile Ridge ...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-79.8833</td>\n",
       "      <td>155.0833</td>\n",
       "      <td>POINT (155.083333 -79.883333)</td>\n",
       "      <td>26</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Achernar</td>\n",
       "      <td>Achernar, Mount</td>\n",
       "      <td>112595</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A peak forming the NE end of MacAlpine Hills, ...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-84.2000</td>\n",
       "      <td>160.9333</td>\n",
       "      <td>POINT (160.933333 -84.2)</td>\n",
       "      <td>36</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Achilles</td>\n",
       "      <td>Achilles, Mount</td>\n",
       "      <td>112596</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A prominent pyramidal mountain (2,880 m) risin...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-71.8833</td>\n",
       "      <td>168.1333</td>\n",
       "      <td>POINT (168.133333 -71.883333)</td>\n",
       "      <td>39</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Adams Glacier</td>\n",
       "      <td>Adams Glacier</td>\n",
       "      <td>112597</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>A small glacier immediately S of Miers Glacier...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-78.1167</td>\n",
       "      <td>163.6333</td>\n",
       "      <td>POINT (163.633333 -78.116667)</td>\n",
       "      <td>55</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Adams Mountains</td>\n",
       "      <td>Adams Mountains</td>\n",
       "      <td>112598</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A small but well defined group of mountains in...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-84.5000</td>\n",
       "      <td>166.3333</td>\n",
       "      <td>POINT (166.333333 -84.5)</td>\n",
       "      <td>57</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Adams Peak</td>\n",
       "      <td>Adams Peak</td>\n",
       "      <td>112599</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Peak</td>\n",
       "      <td>Peak, 1,540 m, on the E side of Starshot Glaci...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-81.6333</td>\n",
       "      <td>160.0667</td>\n",
       "      <td>POINT (160.066667 -81.633333)</td>\n",
       "      <td>59</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Adams Ridge</td>\n",
       "      <td>Adams Ridge</td>\n",
       "      <td>112600</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ridge</td>\n",
       "      <td>A sharp-crested rock ridge, 4 mi long and risi...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-71.0000</td>\n",
       "      <td>162.3833</td>\n",
       "      <td>POINT (162.383333 -71)</td>\n",
       "      <td>60</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Adams Stream</td>\n",
       "      <td>Adams Stream</td>\n",
       "      <td>112601</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Watercourse</td>\n",
       "      <td>Small melt stream flowing east into Lake Miers...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-78.1000</td>\n",
       "      <td>163.7500</td>\n",
       "      <td>POINT (163.75 -78.1)</td>\n",
       "      <td>62</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Adam</td>\n",
       "      <td>Adam, Mount</td>\n",
       "      <td>112602</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>Mountain (4,010 m) situated 2.5 mi WNW of Moun...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-71.7833</td>\n",
       "      <td>168.6167</td>\n",
       "      <td>POINT (168.616667 -71.783333)</td>\n",
       "      <td>66</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Adare Glacier</td>\n",
       "      <td>Adare Glacier</td>\n",
       "      <td>112603</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>Refers to the glacier margin immediately above...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-71.7167</td>\n",
       "      <td>170.2000</td>\n",
       "      <td>POINT (170.2 -71.716667)</td>\n",
       "      <td>67</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Cape Adare</td>\n",
       "      <td>Adare, Cape</td>\n",
       "      <td>112604</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cape</td>\n",
       "      <td>A prominent cape of black basalt which is in v...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-71.2833</td>\n",
       "      <td>170.2333</td>\n",
       "      <td>POINT (170.233333 -71.283333)</td>\n",
       "      <td>72</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Adélie Cove</td>\n",
       "      <td>Adélie Cove</td>\n",
       "      <td>112605</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cove</td>\n",
       "      <td>A 1½km wide by 2½km deep bay in the coast of t...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-74.7667</td>\n",
       "      <td>164.0000</td>\n",
       "      <td>POINT (164 -74.766667)</td>\n",
       "      <td>78</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Adele Glacier</td>\n",
       "      <td>Adele Glacier</td>\n",
       "      <td>112606</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>Refers to the glacier margin immediately above...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.2167</td>\n",
       "      <td>166.5000</td>\n",
       "      <td>POINT (166.5 -77.216667)</td>\n",
       "      <td>76</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Admiralty Mountains</td>\n",
       "      <td>Admiralty Mountains</td>\n",
       "      <td>112607</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A large group of high mountains and individual...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-71.7500</td>\n",
       "      <td>168.5000</td>\n",
       "      <td>POINT (168.5 -71.75)</td>\n",
       "      <td>88</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Aeolus</td>\n",
       "      <td>Aeolus, Mount</td>\n",
       "      <td>112608</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>Prominent peak, over 2,000 m, between Mounts B...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.4833</td>\n",
       "      <td>161.2667</td>\n",
       "      <td>POINT (161.266667 -77.483333)</td>\n",
       "      <td>97</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Aeronaut Glacier</td>\n",
       "      <td>Aeronaut Glacier</td>\n",
       "      <td>112609</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>A glacier of low gradient, about 25 mi long, d...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-73.2667</td>\n",
       "      <td>163.6000</td>\n",
       "      <td>POINT (163.6 -73.266667)</td>\n",
       "      <td>99</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Aiken Glacier</td>\n",
       "      <td>Aiken Glacier</td>\n",
       "      <td>112610</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>A small glacier between Von Guerard Glacier an...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.6333</td>\n",
       "      <td>163.4000</td>\n",
       "      <td>POINT (163.4 -77.633333)</td>\n",
       "      <td>16638</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Ainley Peak</td>\n",
       "      <td>Ainley Peak</td>\n",
       "      <td>112611</td>\n",
       "      <td>1240.0</td>\n",
       "      <td>Peak</td>\n",
       "      <td>A prominent peak, 1240 m, located 3 mi SW of P...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.5000</td>\n",
       "      <td>169.0333</td>\n",
       "      <td>POINT (169.033333 -77.5)</td>\n",
       "      <td>16796</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Airdevronsix Icefalls</td>\n",
       "      <td>Airdevronsix Icefalls</td>\n",
       "      <td>112612</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A line of icefalls at the head of Wright Upper...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.5167</td>\n",
       "      <td>160.3667</td>\n",
       "      <td>POINT (160.366667 -77.516667)</td>\n",
       "      <td>133</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Airdrop Peak</td>\n",
       "      <td>Airdrop Peak</td>\n",
       "      <td>112613</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Peak</td>\n",
       "      <td>A twin-peaked mountain (890 m) at the N end of...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-83.7500</td>\n",
       "      <td>172.7500</td>\n",
       "      <td>POINT (172.75 -83.75)</td>\n",
       "      <td>134</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Ajax</td>\n",
       "      <td>Ajax, Mount</td>\n",
       "      <td>112614</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A mountain (3,770 m) rising 1 mi WSW of Mount ...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-71.8000</td>\n",
       "      <td>168.4500</td>\n",
       "      <td>POINT (168.45 -71.8)</td>\n",
       "      <td>141</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Alamein Range</td>\n",
       "      <td>Alamein Range</td>\n",
       "      <td>112615</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Range</td>\n",
       "      <td>A range lying W of Canham Glacier, in the Frey...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-72.0833</td>\n",
       "      <td>163.5000</td>\n",
       "      <td>POINT (163.5 -72.083333)</td>\n",
       "      <td>160</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Alatna Valley</td>\n",
       "      <td>Alatna Valley</td>\n",
       "      <td>112616</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Valley</td>\n",
       "      <td>An ice-free valley lying 4 mi N of Mount Gran ...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-76.8833</td>\n",
       "      <td>161.1667</td>\n",
       "      <td>POINT (161.166667 -76.883333)</td>\n",
       "      <td>165</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Alb Valley</td>\n",
       "      <td>Alb Valley</td>\n",
       "      <td>112617</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Valley</td>\n",
       "      <td>A name given by the BAE, 1907-09, for a featur...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-74.7833</td>\n",
       "      <td>163.7667</td>\n",
       "      <td>POINT (163.766667 -74.783333)</td>\n",
       "      <td>166</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Albanus Glacier</td>\n",
       "      <td>Albanus Glacier</td>\n",
       "      <td>112618</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>A glacier, 25 mi long, flowing W along the S s...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-85.8667</td>\n",
       "      <td>-151.0000</td>\n",
       "      <td>POINT (-151 -85.866667)</td>\n",
       "      <td>167</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Albatross Glacier</td>\n",
       "      <td>Albatross Glacier</td>\n",
       "      <td>112619</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>Falls south of Prion Glacier. New name for Qua...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.2833</td>\n",
       "      <td>166.5167</td>\n",
       "      <td>POINT (166.516667 -77.283333)</td>\n",
       "      <td>168</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Albrecht Penck Glacier</td>\n",
       "      <td>Albrecht Penck Glacier</td>\n",
       "      <td>112620</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>A glacier between the Fry Glacier and Evans Pi...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-76.6667</td>\n",
       "      <td>162.3333</td>\n",
       "      <td>POINT (162.333333 -76.666667)</td>\n",
       "      <td>177</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Alcyone Cone</td>\n",
       "      <td>Alcyone Cone</td>\n",
       "      <td>112621</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Cone</td>\n",
       "      <td>An extinct volcanic cone near the center of Th...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-72.7000</td>\n",
       "      <td>165.5500</td>\n",
       "      <td>POINT (165.55 -72.7)</td>\n",
       "      <td>180</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3427</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Moraine Ridge</td>\n",
       "      <td>Moraine Ridge</td>\n",
       "      <td>140085</td>\n",
       "      <td>1113.0</td>\n",
       "      <td>Ridge</td>\n",
       "      <td>A small ridge in the northeast part of Cartogr...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-72.2974</td>\n",
       "      <td>168.0405</td>\n",
       "      <td>POINT (168.04046 -72.29738)</td>\n",
       "      <td>9820</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3428</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Aldrich</td>\n",
       "      <td>Aldrich, Mount</td>\n",
       "      <td>140086</td>\n",
       "      <td>2996.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A massive, somewhat flat-topped mountain stand...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-80.1250</td>\n",
       "      <td>158.2344</td>\n",
       "      <td>POINT (158.23438 -80.12498)</td>\n",
       "      <td>187</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3429</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Danforth</td>\n",
       "      <td>Danforth, Mount</td>\n",
       "      <td>140087</td>\n",
       "      <td>1968.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>An ice-free, pyramidal mountain over 2000m, st...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-85.9290</td>\n",
       "      <td>-150.0233</td>\n",
       "      <td>POINT (-150.02326 -85.92897)</td>\n",
       "      <td>3309</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3430</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Durham</td>\n",
       "      <td>Durham, Mount</td>\n",
       "      <td>140088</td>\n",
       "      <td>824.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A mainly ice-free mountain, c.860m, standing a...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-85.5650</td>\n",
       "      <td>-151.1921</td>\n",
       "      <td>POINT (-151.19211 -85.56499)</td>\n",
       "      <td>3977</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3431</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Early</td>\n",
       "      <td>Early, Mount</td>\n",
       "      <td>140089</td>\n",
       "      <td>2662.0</td>\n",
       "      <td>Cone</td>\n",
       "      <td>A solitary volcanic cone c.2720m standing 24km...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-87.0703</td>\n",
       "      <td>-153.7700</td>\n",
       "      <td>POINT (-153.77003 -87.0703)</td>\n",
       "      <td>4021</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3432</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Gardiner</td>\n",
       "      <td>Gardiner, Mount</td>\n",
       "      <td>140090</td>\n",
       "      <td>2384.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A ridge-like granitic mountain, 2479m, standin...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-86.3195</td>\n",
       "      <td>-150.9651</td>\n",
       "      <td>POINT (-150.9651 -86.31953)</td>\n",
       "      <td>5123</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3433</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount McKercher</td>\n",
       "      <td>McKercher, Mount</td>\n",
       "      <td>140091</td>\n",
       "      <td>2232.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A mountain, 2230m, standing at the east side o...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-86.1556</td>\n",
       "      <td>-149.9845</td>\n",
       "      <td>POINT (-149.98454 -86.15562)</td>\n",
       "      <td>9302</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3434</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Nares</td>\n",
       "      <td>Nares, Mount</td>\n",
       "      <td>140092</td>\n",
       "      <td>3012.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A massive mountain, over 3000m, standing just ...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-81.4500</td>\n",
       "      <td>158.1667</td>\n",
       "      <td>POINT (158.16667 -81.45)</td>\n",
       "      <td>10084</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3435</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Swadener</td>\n",
       "      <td>Swadener, Mount</td>\n",
       "      <td>140093</td>\n",
       "      <td>507.0</td>\n",
       "      <td>Peak</td>\n",
       "      <td>A peak in the Sneddon Nunataks, in the north p...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.2729</td>\n",
       "      <td>-153.7381</td>\n",
       "      <td>POINT (-153.73806 -77.27294)</td>\n",
       "      <td>14313</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3436</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Youngman</td>\n",
       "      <td>Youngman, Mount</td>\n",
       "      <td>140094</td>\n",
       "      <td>604.0</td>\n",
       "      <td>Knoll</td>\n",
       "      <td>A snow-covered coastal mountain c.620m, 7km so...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.2478</td>\n",
       "      <td>-154.3237</td>\n",
       "      <td>POINT (-154.32373 -77.24776)</td>\n",
       "      <td>16386</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3437</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Organ Pipe Peaks</td>\n",
       "      <td>Organ Pipe Peaks</td>\n",
       "      <td>140095</td>\n",
       "      <td>1752.0</td>\n",
       "      <td>Range</td>\n",
       "      <td>A row of aiguille type rock peaks, 13km east t...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-86.0550</td>\n",
       "      <td>-150.1442</td>\n",
       "      <td>POINT (-150.14416 -86.054988)</td>\n",
       "      <td>10670</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3438</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Poulter Glacier</td>\n",
       "      <td>Poulter Glacier</td>\n",
       "      <td>140096</td>\n",
       "      <td>2048.0</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>A tributary glacier draining eastward along th...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-86.8333</td>\n",
       "      <td>-153.5000</td>\n",
       "      <td>POINT (-153.5 -86.8333)</td>\n",
       "      <td>11530</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3439</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Second Facet</td>\n",
       "      <td>Second Facet</td>\n",
       "      <td>140097</td>\n",
       "      <td>777.0</td>\n",
       "      <td>Cliff</td>\n",
       "      <td>A steep ice-free bluff standing just southwest...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.1852</td>\n",
       "      <td>162.2840</td>\n",
       "      <td>POINT (162.28404 -77.1852)</td>\n",
       "      <td>12967</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3440</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Starr Lake</td>\n",
       "      <td>Starr Lake</td>\n",
       "      <td>140098</td>\n",
       "      <td>133.0</td>\n",
       "      <td>Water body</td>\n",
       "      <td>A small meltwater lake which is a source of wa...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.8369</td>\n",
       "      <td>166.6805</td>\n",
       "      <td>POINT (166.680491 -77.83692)</td>\n",
       "      <td>13944</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3441</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Strider Rock</td>\n",
       "      <td>Strider Rock</td>\n",
       "      <td>140099</td>\n",
       "      <td>525.0</td>\n",
       "      <td>Rock</td>\n",
       "      <td>A rock 3.5km northwest of Mount Nilsen in the ...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-78.0333</td>\n",
       "      <td>-155.4333</td>\n",
       "      <td>POINT (-155.4333 -78.0333)</td>\n",
       "      <td>14143</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3442</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Sunny Ridge</td>\n",
       "      <td>Sunny Ridge</td>\n",
       "      <td>140100</td>\n",
       "      <td>2498.0</td>\n",
       "      <td>Ridge</td>\n",
       "      <td>A partly snow-free ridge that trends north-sou...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-87.0088</td>\n",
       "      <td>-154.4880</td>\n",
       "      <td>POINT (-154.48805 -87.00882)</td>\n",
       "      <td>14248</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3443</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Swinburne Ice Shelf</td>\n",
       "      <td>Swinburne Ice Shelf</td>\n",
       "      <td>140101</td>\n",
       "      <td>49.0</td>\n",
       "      <td>Ice shelf</td>\n",
       "      <td>An ice shelf just north of the Alexandra Mount...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.1667</td>\n",
       "      <td>-153.9167</td>\n",
       "      <td>POINT (-153.9167 -77.1667)</td>\n",
       "      <td>14332</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3444</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Vollmer Island</td>\n",
       "      <td>Vollmer Island</td>\n",
       "      <td>140102</td>\n",
       "      <td>276.0</td>\n",
       "      <td>Island</td>\n",
       "      <td>An ice-covered island approximately 20km east ...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-76.7333</td>\n",
       "      <td>-150.5000</td>\n",
       "      <td>POINT (-150.5 -76.7333)</td>\n",
       "      <td>15574</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3445</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Withrow Glacier</td>\n",
       "      <td>Withrow Glacier</td>\n",
       "      <td>140103</td>\n",
       "      <td>321.0</td>\n",
       "      <td>Glacier</td>\n",
       "      <td>A glacier on King Edward VII Land, flowing nor...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.4154</td>\n",
       "      <td>-156.5141</td>\n",
       "      <td>POINT (-156.51409 -77.41542)</td>\n",
       "      <td>16170</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3446</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Conway Range</td>\n",
       "      <td>Conway Range</td>\n",
       "      <td>140104</td>\n",
       "      <td>2156.0</td>\n",
       "      <td>Range</td>\n",
       "      <td>A north-south trending range of about 60m, in ...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-79.2667</td>\n",
       "      <td>159.5000</td>\n",
       "      <td>POINT (159.5 -79.266667)</td>\n",
       "      <td>2932</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3447</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Darwin Mountains</td>\n",
       "      <td>Darwin Mountains</td>\n",
       "      <td>140105</td>\n",
       "      <td>1651.0</td>\n",
       "      <td>Range</td>\n",
       "      <td>A group of mountains between the Darwin and Ha...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-79.8500</td>\n",
       "      <td>156.2500</td>\n",
       "      <td>POINT (156.25 -79.85)</td>\n",
       "      <td>3345</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3448</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Albert Markham</td>\n",
       "      <td>Albert Markham, Mount</td>\n",
       "      <td>140106</td>\n",
       "      <td>3203.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A striking flat-topped mountain, 3205m, standi...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-81.3902</td>\n",
       "      <td>158.2398</td>\n",
       "      <td>POINT (158.23978 -81.39019)</td>\n",
       "      <td>170</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3449</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Egerton</td>\n",
       "      <td>Egerton, Mount</td>\n",
       "      <td>140107</td>\n",
       "      <td>2793.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A mountain, c.2830m, rising 5.7km north-northw...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-80.8356</td>\n",
       "      <td>157.9279</td>\n",
       "      <td>POINT (157.92794 -80.83557)</td>\n",
       "      <td>4107</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3450</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Field</td>\n",
       "      <td>Field, Mount</td>\n",
       "      <td>140108</td>\n",
       "      <td>2982.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A mountain, c.3010m, standing 5.7km south-sout...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-80.8846</td>\n",
       "      <td>158.0255</td>\n",
       "      <td>POINT (158.02546 -80.88461)</td>\n",
       "      <td>4570</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3451</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Keltie</td>\n",
       "      <td>Keltie, Mount</td>\n",
       "      <td>140109</td>\n",
       "      <td>2642.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>Mountain, 2640m, midway between Mount Kosko to...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-79.2448</td>\n",
       "      <td>159.4937</td>\n",
       "      <td>POINT (159.49373 -79.24481)</td>\n",
       "      <td>7439</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3452</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount McClintock</td>\n",
       "      <td>McClintock, Mount</td>\n",
       "      <td>140110</td>\n",
       "      <td>3490.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>The highest mountain at 3500m in Britannia Ran...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-80.2160</td>\n",
       "      <td>157.4420</td>\n",
       "      <td>POINT (157.44201 -80.21605)</td>\n",
       "      <td>9241</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3453</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Wharton</td>\n",
       "      <td>Wharton, Mount</td>\n",
       "      <td>140111</td>\n",
       "      <td>2859.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A mountain c.2860m, standing 10km west of Turk...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-81.0521</td>\n",
       "      <td>157.8042</td>\n",
       "      <td>POINT (157.80425 -81.05209)</td>\n",
       "      <td>15945</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3454</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Talos Dome</td>\n",
       "      <td>Talos Dome</td>\n",
       "      <td>140112</td>\n",
       "      <td>2376.0</td>\n",
       "      <td>Dome</td>\n",
       "      <td>A large ice dome rising to over 2300m to the s...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-72.7955</td>\n",
       "      <td>159.0813</td>\n",
       "      <td>POINT (159.08125 -72.79546)</td>\n",
       "      <td>14405</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3455</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Taylor Dome</td>\n",
       "      <td>Taylor Dome</td>\n",
       "      <td>140113</td>\n",
       "      <td>2448.0</td>\n",
       "      <td>Dome</td>\n",
       "      <td>An elliptical ice dome up to 80km long east-so...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-77.8337</td>\n",
       "      <td>159.0470</td>\n",
       "      <td>POINT (159.04695 -77.83373)</td>\n",
       "      <td>14455</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3456</th>\n",
       "      <td>SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...</td>\n",
       "      <td>Mount Henderson</td>\n",
       "      <td>Henderson, Mount</td>\n",
       "      <td>140114</td>\n",
       "      <td>2647.0</td>\n",
       "      <td>Mountain</td>\n",
       "      <td>A prominent mountain, 2661m, standing 8km west...</td>\n",
       "      <td>NZL</td>\n",
       "      <td>-80.2006</td>\n",
       "      <td>156.2190</td>\n",
       "      <td>POINT (156.21904 -80.20061)</td>\n",
       "      <td>6250</td>\n",
       "      <td>New Zealand</td>\n",
       "      <td>554</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3457 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    FID  \\\n",
       "0     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "1     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "2     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "4     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "5     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "6     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "7     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "8     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "9     SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "10    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "11    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "12    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "13    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "14    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "15    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "16    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "17    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "18    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "19    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "20    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "21    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "22    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "23    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "24    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "25    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "26    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "27    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "28    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "29    SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "...                                                 ...   \n",
       "3427  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3428  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3429  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3430  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3431  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3432  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3433  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3434  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3435  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3436  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3437  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3438  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3439  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3440  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3441  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3442  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3443  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3444  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3445  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3446  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3447  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3448  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3449  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3450  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3451  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3452  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3453  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3454  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3455  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "3456  SCAR_CGA_PLACE_NAMES_SIMPLIFIED.fid--705bf013_...   \n",
       "\n",
       "          place_name_mapping    place_name_gazetteer  gaz_id  altitude  \\\n",
       "0                Abbott Peak             Abbott Peak  112592       NaN   \n",
       "1               Mount Abbott           Abbott, Mount  112593       NaN   \n",
       "2                Abus Valley             Abus Valley  112594       NaN   \n",
       "3             Mount Achernar         Achernar, Mount  112595       NaN   \n",
       "4             Mount Achilles         Achilles, Mount  112596       NaN   \n",
       "5              Adams Glacier           Adams Glacier  112597       NaN   \n",
       "6            Adams Mountains         Adams Mountains  112598       NaN   \n",
       "7                 Adams Peak              Adams Peak  112599       NaN   \n",
       "8                Adams Ridge             Adams Ridge  112600       NaN   \n",
       "9               Adams Stream            Adams Stream  112601       NaN   \n",
       "10                Mount Adam             Adam, Mount  112602       NaN   \n",
       "11             Adare Glacier           Adare Glacier  112603       NaN   \n",
       "12                Cape Adare             Adare, Cape  112604       NaN   \n",
       "13               Adélie Cove             Adélie Cove  112605       NaN   \n",
       "14             Adele Glacier           Adele Glacier  112606       NaN   \n",
       "15       Admiralty Mountains     Admiralty Mountains  112607       NaN   \n",
       "16              Mount Aeolus           Aeolus, Mount  112608       NaN   \n",
       "17          Aeronaut Glacier        Aeronaut Glacier  112609       NaN   \n",
       "18             Aiken Glacier           Aiken Glacier  112610       NaN   \n",
       "19               Ainley Peak             Ainley Peak  112611    1240.0   \n",
       "20     Airdevronsix Icefalls   Airdevronsix Icefalls  112612       NaN   \n",
       "21              Airdrop Peak            Airdrop Peak  112613       NaN   \n",
       "22                Mount Ajax             Ajax, Mount  112614       NaN   \n",
       "23             Alamein Range           Alamein Range  112615       NaN   \n",
       "24             Alatna Valley           Alatna Valley  112616       NaN   \n",
       "25                Alb Valley              Alb Valley  112617       NaN   \n",
       "26           Albanus Glacier         Albanus Glacier  112618       NaN   \n",
       "27         Albatross Glacier       Albatross Glacier  112619       NaN   \n",
       "28    Albrecht Penck Glacier  Albrecht Penck Glacier  112620       NaN   \n",
       "29              Alcyone Cone            Alcyone Cone  112621       NaN   \n",
       "...                      ...                     ...     ...       ...   \n",
       "3427           Moraine Ridge           Moraine Ridge  140085    1113.0   \n",
       "3428           Mount Aldrich          Aldrich, Mount  140086    2996.0   \n",
       "3429          Mount Danforth         Danforth, Mount  140087    1968.0   \n",
       "3430            Mount Durham           Durham, Mount  140088     824.0   \n",
       "3431             Mount Early            Early, Mount  140089    2662.0   \n",
       "3432          Mount Gardiner         Gardiner, Mount  140090    2384.0   \n",
       "3433         Mount McKercher        McKercher, Mount  140091    2232.0   \n",
       "3434             Mount Nares            Nares, Mount  140092    3012.0   \n",
       "3435          Mount Swadener         Swadener, Mount  140093     507.0   \n",
       "3436          Mount Youngman         Youngman, Mount  140094     604.0   \n",
       "3437        Organ Pipe Peaks        Organ Pipe Peaks  140095    1752.0   \n",
       "3438         Poulter Glacier         Poulter Glacier  140096    2048.0   \n",
       "3439            Second Facet            Second Facet  140097     777.0   \n",
       "3440              Starr Lake              Starr Lake  140098     133.0   \n",
       "3441            Strider Rock            Strider Rock  140099     525.0   \n",
       "3442             Sunny Ridge             Sunny Ridge  140100    2498.0   \n",
       "3443     Swinburne Ice Shelf     Swinburne Ice Shelf  140101      49.0   \n",
       "3444          Vollmer Island          Vollmer Island  140102     276.0   \n",
       "3445         Withrow Glacier         Withrow Glacier  140103     321.0   \n",
       "3446            Conway Range            Conway Range  140104    2156.0   \n",
       "3447        Darwin Mountains        Darwin Mountains  140105    1651.0   \n",
       "3448    Mount Albert Markham   Albert Markham, Mount  140106    3203.0   \n",
       "3449           Mount Egerton          Egerton, Mount  140107    2793.0   \n",
       "3450             Mount Field            Field, Mount  140108    2982.0   \n",
       "3451            Mount Keltie           Keltie, Mount  140109    2642.0   \n",
       "3452        Mount McClintock       McClintock, Mount  140110    3490.0   \n",
       "3453           Mount Wharton          Wharton, Mount  140111    2859.0   \n",
       "3454              Talos Dome              Talos Dome  140112    2376.0   \n",
       "3455             Taylor Dome             Taylor Dome  140113    2448.0   \n",
       "3456         Mount Henderson        Henderson, Mount  140114    2647.0   \n",
       "\n",
       "     feature_type_name                                          narrative  \\\n",
       "0                 Peak  Pyramidal peak on Ross Island, on the N side o...   \n",
       "1             Mountain  A mountain 1,020 m, which stands 3 mi NE of Ca...   \n",
       "2               Valley  An ice-free valley 3 mi SE of Turnstile Ridge ...   \n",
       "3             Mountain  A peak forming the NE end of MacAlpine Hills, ...   \n",
       "4             Mountain  A prominent pyramidal mountain (2,880 m) risin...   \n",
       "5              Glacier  A small glacier immediately S of Miers Glacier...   \n",
       "6             Mountain  A small but well defined group of mountains in...   \n",
       "7                 Peak  Peak, 1,540 m, on the E side of Starshot Glaci...   \n",
       "8                Ridge  A sharp-crested rock ridge, 4 mi long and risi...   \n",
       "9          Watercourse  Small melt stream flowing east into Lake Miers...   \n",
       "10            Mountain  Mountain (4,010 m) situated 2.5 mi WNW of Moun...   \n",
       "11             Glacier  Refers to the glacier margin immediately above...   \n",
       "12                Cape  A prominent cape of black basalt which is in v...   \n",
       "13                Cove  A 1½km wide by 2½km deep bay in the coast of t...   \n",
       "14             Glacier  Refers to the glacier margin immediately above...   \n",
       "15            Mountain  A large group of high mountains and individual...   \n",
       "16            Mountain  Prominent peak, over 2,000 m, between Mounts B...   \n",
       "17             Glacier  A glacier of low gradient, about 25 mi long, d...   \n",
       "18             Glacier  A small glacier between Von Guerard Glacier an...   \n",
       "19                Peak  A prominent peak, 1240 m, located 3 mi SW of P...   \n",
       "20                 NaN  A line of icefalls at the head of Wright Upper...   \n",
       "21                Peak  A twin-peaked mountain (890 m) at the N end of...   \n",
       "22            Mountain  A mountain (3,770 m) rising 1 mi WSW of Mount ...   \n",
       "23               Range  A range lying W of Canham Glacier, in the Frey...   \n",
       "24              Valley  An ice-free valley lying 4 mi N of Mount Gran ...   \n",
       "25              Valley  A name given by the BAE, 1907-09, for a featur...   \n",
       "26             Glacier  A glacier, 25 mi long, flowing W along the S s...   \n",
       "27             Glacier  Falls south of Prion Glacier. New name for Qua...   \n",
       "28             Glacier  A glacier between the Fry Glacier and Evans Pi...   \n",
       "29                Cone  An extinct volcanic cone near the center of Th...   \n",
       "...                ...                                                ...   \n",
       "3427             Ridge  A small ridge in the northeast part of Cartogr...   \n",
       "3428          Mountain  A massive, somewhat flat-topped mountain stand...   \n",
       "3429          Mountain  An ice-free, pyramidal mountain over 2000m, st...   \n",
       "3430          Mountain  A mainly ice-free mountain, c.860m, standing a...   \n",
       "3431              Cone  A solitary volcanic cone c.2720m standing 24km...   \n",
       "3432          Mountain  A ridge-like granitic mountain, 2479m, standin...   \n",
       "3433          Mountain  A mountain, 2230m, standing at the east side o...   \n",
       "3434          Mountain  A massive mountain, over 3000m, standing just ...   \n",
       "3435              Peak  A peak in the Sneddon Nunataks, in the north p...   \n",
       "3436             Knoll  A snow-covered coastal mountain c.620m, 7km so...   \n",
       "3437             Range  A row of aiguille type rock peaks, 13km east t...   \n",
       "3438           Glacier  A tributary glacier draining eastward along th...   \n",
       "3439             Cliff  A steep ice-free bluff standing just southwest...   \n",
       "3440        Water body  A small meltwater lake which is a source of wa...   \n",
       "3441              Rock  A rock 3.5km northwest of Mount Nilsen in the ...   \n",
       "3442             Ridge  A partly snow-free ridge that trends north-sou...   \n",
       "3443         Ice shelf  An ice shelf just north of the Alexandra Mount...   \n",
       "3444            Island  An ice-covered island approximately 20km east ...   \n",
       "3445           Glacier  A glacier on King Edward VII Land, flowing nor...   \n",
       "3446             Range  A north-south trending range of about 60m, in ...   \n",
       "3447             Range  A group of mountains between the Darwin and Ha...   \n",
       "3448          Mountain  A striking flat-topped mountain, 3205m, standi...   \n",
       "3449          Mountain  A mountain, c.2830m, rising 5.7km north-northw...   \n",
       "3450          Mountain  A mountain, c.3010m, standing 5.7km south-sout...   \n",
       "3451          Mountain  Mountain, 2640m, midway between Mount Kosko to...   \n",
       "3452          Mountain  The highest mountain at 3500m in Britannia Ran...   \n",
       "3453          Mountain  A mountain c.2860m, standing 10km west of Turk...   \n",
       "3454              Dome  A large ice dome rising to over 2300m to the s...   \n",
       "3455              Dome  An elliptical ice dome up to 80km long east-so...   \n",
       "3456          Mountain  A prominent mountain, 2661m, standing 8km west...   \n",
       "\n",
       "     gazetteer  latitude  longitude                       geometry  \\\n",
       "0          NZL  -77.4333   167.0000         POINT (167 -77.433333)   \n",
       "1          NZL  -74.7000   163.8333       POINT (163.833333 -74.7)   \n",
       "2          NZL  -79.8833   155.0833  POINT (155.083333 -79.883333)   \n",
       "3          NZL  -84.2000   160.9333       POINT (160.933333 -84.2)   \n",
       "4          NZL  -71.8833   168.1333  POINT (168.133333 -71.883333)   \n",
       "5          NZL  -78.1167   163.6333  POINT (163.633333 -78.116667)   \n",
       "6          NZL  -84.5000   166.3333       POINT (166.333333 -84.5)   \n",
       "7          NZL  -81.6333   160.0667  POINT (160.066667 -81.633333)   \n",
       "8          NZL  -71.0000   162.3833         POINT (162.383333 -71)   \n",
       "9          NZL  -78.1000   163.7500           POINT (163.75 -78.1)   \n",
       "10         NZL  -71.7833   168.6167  POINT (168.616667 -71.783333)   \n",
       "11         NZL  -71.7167   170.2000       POINT (170.2 -71.716667)   \n",
       "12         NZL  -71.2833   170.2333  POINT (170.233333 -71.283333)   \n",
       "13         NZL  -74.7667   164.0000         POINT (164 -74.766667)   \n",
       "14         NZL  -77.2167   166.5000       POINT (166.5 -77.216667)   \n",
       "15         NZL  -71.7500   168.5000           POINT (168.5 -71.75)   \n",
       "16         NZL  -77.4833   161.2667  POINT (161.266667 -77.483333)   \n",
       "17         NZL  -73.2667   163.6000       POINT (163.6 -73.266667)   \n",
       "18         NZL  -77.6333   163.4000       POINT (163.4 -77.633333)   \n",
       "19         NZL  -77.5000   169.0333       POINT (169.033333 -77.5)   \n",
       "20         NZL  -77.5167   160.3667  POINT (160.366667 -77.516667)   \n",
       "21         NZL  -83.7500   172.7500          POINT (172.75 -83.75)   \n",
       "22         NZL  -71.8000   168.4500           POINT (168.45 -71.8)   \n",
       "23         NZL  -72.0833   163.5000       POINT (163.5 -72.083333)   \n",
       "24         NZL  -76.8833   161.1667  POINT (161.166667 -76.883333)   \n",
       "25         NZL  -74.7833   163.7667  POINT (163.766667 -74.783333)   \n",
       "26         NZL  -85.8667  -151.0000        POINT (-151 -85.866667)   \n",
       "27         NZL  -77.2833   166.5167  POINT (166.516667 -77.283333)   \n",
       "28         NZL  -76.6667   162.3333  POINT (162.333333 -76.666667)   \n",
       "29         NZL  -72.7000   165.5500           POINT (165.55 -72.7)   \n",
       "...        ...       ...        ...                            ...   \n",
       "3427       NZL  -72.2974   168.0405    POINT (168.04046 -72.29738)   \n",
       "3428       NZL  -80.1250   158.2344    POINT (158.23438 -80.12498)   \n",
       "3429       NZL  -85.9290  -150.0233   POINT (-150.02326 -85.92897)   \n",
       "3430       NZL  -85.5650  -151.1921   POINT (-151.19211 -85.56499)   \n",
       "3431       NZL  -87.0703  -153.7700    POINT (-153.77003 -87.0703)   \n",
       "3432       NZL  -86.3195  -150.9651    POINT (-150.9651 -86.31953)   \n",
       "3433       NZL  -86.1556  -149.9845   POINT (-149.98454 -86.15562)   \n",
       "3434       NZL  -81.4500   158.1667       POINT (158.16667 -81.45)   \n",
       "3435       NZL  -77.2729  -153.7381   POINT (-153.73806 -77.27294)   \n",
       "3436       NZL  -77.2478  -154.3237   POINT (-154.32373 -77.24776)   \n",
       "3437       NZL  -86.0550  -150.1442  POINT (-150.14416 -86.054988)   \n",
       "3438       NZL  -86.8333  -153.5000        POINT (-153.5 -86.8333)   \n",
       "3439       NZL  -77.1852   162.2840     POINT (162.28404 -77.1852)   \n",
       "3440       NZL  -77.8369   166.6805   POINT (166.680491 -77.83692)   \n",
       "3441       NZL  -78.0333  -155.4333     POINT (-155.4333 -78.0333)   \n",
       "3442       NZL  -87.0088  -154.4880   POINT (-154.48805 -87.00882)   \n",
       "3443       NZL  -77.1667  -153.9167     POINT (-153.9167 -77.1667)   \n",
       "3444       NZL  -76.7333  -150.5000        POINT (-150.5 -76.7333)   \n",
       "3445       NZL  -77.4154  -156.5141   POINT (-156.51409 -77.41542)   \n",
       "3446       NZL  -79.2667   159.5000       POINT (159.5 -79.266667)   \n",
       "3447       NZL  -79.8500   156.2500          POINT (156.25 -79.85)   \n",
       "3448       NZL  -81.3902   158.2398    POINT (158.23978 -81.39019)   \n",
       "3449       NZL  -80.8356   157.9279    POINT (157.92794 -80.83557)   \n",
       "3450       NZL  -80.8846   158.0255    POINT (158.02546 -80.88461)   \n",
       "3451       NZL  -79.2448   159.4937    POINT (159.49373 -79.24481)   \n",
       "3452       NZL  -80.2160   157.4420    POINT (157.44201 -80.21605)   \n",
       "3453       NZL  -81.0521   157.8042    POINT (157.80425 -81.05209)   \n",
       "3454       NZL  -72.7955   159.0813    POINT (159.08125 -72.79546)   \n",
       "3455       NZL  -77.8337   159.0470    POINT (159.04695 -77.83373)   \n",
       "3456       NZL  -80.2006   156.2190    POINT (156.21904 -80.20061)   \n",
       "\n",
       "      scar_common_id country_name  country_id  \n",
       "0                  8  New Zealand         554  \n",
       "1                  9  New Zealand         554  \n",
       "2                 26  New Zealand         554  \n",
       "3                 36  New Zealand         554  \n",
       "4                 39  New Zealand         554  \n",
       "5                 55  New Zealand         554  \n",
       "6                 57  New Zealand         554  \n",
       "7                 59  New Zealand         554  \n",
       "8                 60  New Zealand         554  \n",
       "9                 62  New Zealand         554  \n",
       "10                66  New Zealand         554  \n",
       "11                67  New Zealand         554  \n",
       "12                72  New Zealand         554  \n",
       "13                78  New Zealand         554  \n",
       "14                76  New Zealand         554  \n",
       "15                88  New Zealand         554  \n",
       "16                97  New Zealand         554  \n",
       "17                99  New Zealand         554  \n",
       "18             16638  New Zealand         554  \n",
       "19             16796  New Zealand         554  \n",
       "20               133  New Zealand         554  \n",
       "21               134  New Zealand         554  \n",
       "22               141  New Zealand         554  \n",
       "23               160  New Zealand         554  \n",
       "24               165  New Zealand         554  \n",
       "25               166  New Zealand         554  \n",
       "26               167  New Zealand         554  \n",
       "27               168  New Zealand         554  \n",
       "28               177  New Zealand         554  \n",
       "29               180  New Zealand         554  \n",
       "...              ...          ...         ...  \n",
       "3427            9820  New Zealand         554  \n",
       "3428             187  New Zealand         554  \n",
       "3429            3309  New Zealand         554  \n",
       "3430            3977  New Zealand         554  \n",
       "3431            4021  New Zealand         554  \n",
       "3432            5123  New Zealand         554  \n",
       "3433            9302  New Zealand         554  \n",
       "3434           10084  New Zealand         554  \n",
       "3435           14313  New Zealand         554  \n",
       "3436           16386  New Zealand         554  \n",
       "3437           10670  New Zealand         554  \n",
       "3438           11530  New Zealand         554  \n",
       "3439           12967  New Zealand         554  \n",
       "3440           13944  New Zealand         554  \n",
       "3441           14143  New Zealand         554  \n",
       "3442           14248  New Zealand         554  \n",
       "3443           14332  New Zealand         554  \n",
       "3444           15574  New Zealand         554  \n",
       "3445           16170  New Zealand         554  \n",
       "3446            2932  New Zealand         554  \n",
       "3447            3345  New Zealand         554  \n",
       "3448             170  New Zealand         554  \n",
       "3449            4107  New Zealand         554  \n",
       "3450            4570  New Zealand         554  \n",
       "3451            7439  New Zealand         554  \n",
       "3452            9241  New Zealand         554  \n",
       "3453           15945  New Zealand         554  \n",
       "3454           14405  New Zealand         554  \n",
       "3455           14455  New Zealand         554  \n",
       "3456            6250  New Zealand         554  \n",
       "\n",
       "[3457 rows x 14 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SCARNZ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:25:40.020536Z",
     "start_time": "2021-02-28T01:25:39.988322Z"
    }
   },
   "outputs": [],
   "source": [
    "nzgazAntartica = nzgaz[nzgaz.land_district.isna()].name.unique()\n",
    "nzgaz = nzgaz.name.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:25:45.912947Z",
     "start_time": "2021-02-28T01:25:45.903372Z"
    }
   },
   "outputs": [],
   "source": [
    "SCARGlobalnames = SCARGlobal.place_name_mapping.unique()\n",
    "SCARNZnames = SCARNZ.place_name_mapping.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:25:47.563262Z",
     "start_time": "2021-02-28T01:25:47.550731Z"
    }
   },
   "outputs": [],
   "source": [
    "del SCARGlobal\n",
    "del SCARNZ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:25:48.767065Z",
     "start_time": "2021-02-28T01:25:48.763282Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3451"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SCARNZnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:25:51.624676Z",
     "start_time": "2021-02-28T01:25:51.621273Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5141"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nzgazAntartica)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:33:38.444538Z",
     "start_time": "2021-02-28T01:33:38.437258Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['15 Mile Creek', '17 Mile Creek', '1st Basin', ...,\n",
       "       'Zurbriggen Ridge', 'Zwally Glacier', 'Zykov Glacier'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.tofile(np.array(nzgaz))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:42:06.320895Z",
     "start_time": "2021-02-28T01:42:06.299858Z"
    }
   },
   "outputs": [],
   "source": [
    "np.save('nzgazetteer.npy', nzgaz)\n",
    "np.save('nzgazAntartica.npy', nzgazAntartica)\n",
    "np.save('SCARGlobalnames.npy', SCARGlobalnames)\n",
    "np.save('SCARNZnames.npy', SCARNZnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:39:51.148304Z",
     "start_time": "2021-02-28T01:39:51.128753Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['15 Mile Creek',\n",
       " '17 Mile Creek',\n",
       " '1st Basin',\n",
       " '1st Staircase',\n",
       " '2nd Basin',\n",
       " '2nd Staircase',\n",
       " '3rd Basin',\n",
       " '45th Parallel marker',\n",
       " '4th Basin',\n",
       " '5 Mile Creek',\n",
       " 'A1 Creek',\n",
       " 'Aan River',\n",
       " 'Aaron Creek',\n",
       " 'A. A. Thomas Hills',\n",
       " 'Abaft Gully',\n",
       " 'Abandon Brook',\n",
       " 'Abbassia Camp',\n",
       " 'Abbess',\n",
       " 'Abbey Caves',\n",
       " 'Abbey Pass',\n",
       " 'Abbey Rocks',\n",
       " 'Abbot',\n",
       " 'Abbots Creek',\n",
       " 'Abbots Creek Recreation Reserve',\n",
       " 'Abbotsford',\n",
       " 'Abbott',\n",
       " 'Abbott Passage',\n",
       " 'Abbott Peak',\n",
       " 'Abbotts Creek',\n",
       " 'Abbotts Hill',\n",
       " 'Abbott Spur',\n",
       " 'ABC Cave',\n",
       " 'Abel Glacier',\n",
       " 'Abel Head',\n",
       " 'Abel Janszoon Glacier',\n",
       " 'Abel Lake',\n",
       " 'Abel Tasman Monument',\n",
       " 'Abel Tasman National Park',\n",
       " 'Abel Tasman Point',\n",
       " 'Abel Tasman Roadstead',\n",
       " 'Abel Tasman Scenic Reserve',\n",
       " 'Abercrombie Crests',\n",
       " 'Abernethys Creek',\n",
       " 'Abes Gully',\n",
       " 'Able Head',\n",
       " 'Abners Head',\n",
       " 'Abrahams Bay',\n",
       " 'Abrahams Bosom',\n",
       " 'Abruzzi Glacier',\n",
       " 'Abseil Peak',\n",
       " 'Absolum Creek',\n",
       " 'Abus Valley',\n",
       " 'Abut Head',\n",
       " 'Abut Head Scenic Reserve',\n",
       " 'Acacia Bay',\n",
       " 'Acacia Bay North',\n",
       " 'Acacia Bay South',\n",
       " 'Acarospora Peak',\n",
       " 'Access Creek',\n",
       " 'Access Peak',\n",
       " 'Access Slope',\n",
       " 'Accident Creek',\n",
       " 'Accord Point',\n",
       " 'Acheron Anchorage',\n",
       " 'Acheron Basin',\n",
       " 'Acheron Flat',\n",
       " 'Acheron Lakes',\n",
       " 'Acheron Passage',\n",
       " 'Acheron Point',\n",
       " 'Acheron River',\n",
       " 'Acheron Saddle',\n",
       " 'Acheson Crossing',\n",
       " 'Acheson Stream',\n",
       " 'Achilles Peak',\n",
       " 'Achilles Point',\n",
       " 'Aciphylla Creek',\n",
       " 'Aciphylla Stream',\n",
       " 'Ackers Point',\n",
       " 'Ackers Point Scenic Reserve',\n",
       " 'Ackley Point',\n",
       " 'Ackroyd Point',\n",
       " 'Acland Falls',\n",
       " 'Acland Lagoon',\n",
       " 'Acre Creek',\n",
       " 'Acrobat Creek',\n",
       " 'Acton Stream',\n",
       " 'Acton Stream Recreation Reserve',\n",
       " 'Actual Creek',\n",
       " 'Ada Creek',\n",
       " 'Ada Flat',\n",
       " 'Adair',\n",
       " 'Adam Creek',\n",
       " 'Adam Head',\n",
       " 'Adams Bank',\n",
       " 'Adams Bluff',\n",
       " 'Adams Burn',\n",
       " 'Adams Col',\n",
       " 'Adams Creek',\n",
       " 'Adams Flat',\n",
       " 'Adams Glacier',\n",
       " 'Adams Gully',\n",
       " 'Adams Hill',\n",
       " 'Adams Island',\n",
       " 'Adams Mountains',\n",
       " 'Adamson Creek',\n",
       " 'Adams Peak',\n",
       " 'Adams Range',\n",
       " 'Adams Ridge',\n",
       " 'Adams River',\n",
       " 'Adams Rocks',\n",
       " 'Adams Stream',\n",
       " 'Adamstown Creek',\n",
       " 'Adams Wilderness Area',\n",
       " 'Ada Pass',\n",
       " 'Adare Basin',\n",
       " 'Adare Peninsula',\n",
       " 'Adare Saddle',\n",
       " 'Adare Trough',\n",
       " 'Ada River',\n",
       " 'Adderley Head',\n",
       " 'Adderley Head Scenic Reserve',\n",
       " 'Addington',\n",
       " 'Addisons Flat',\n",
       " \"A'Deanes Bush Scenic Reserve\",\n",
       " 'Adelaide Peak',\n",
       " 'Adelaide Saddle',\n",
       " 'Adelaide Tarn',\n",
       " 'Adele',\n",
       " 'Adele Glacier',\n",
       " 'Adele Island',\n",
       " 'Adélie Cove',\n",
       " 'Adélie Glacier',\n",
       " 'Adiantum Bluff',\n",
       " 'Adieu Bank',\n",
       " 'Adieu Basin',\n",
       " 'Adieu Creek',\n",
       " 'Adieu Point',\n",
       " 'Adit Stream',\n",
       " 'Adkin',\n",
       " 'Administration Bay',\n",
       " 'Admirals Bay',\n",
       " 'Admirals Pool',\n",
       " 'Admirals Scenic Reserve',\n",
       " 'Admiralty Bay',\n",
       " 'Admiralty Guyot',\n",
       " 'Admiralty Mountains',\n",
       " 'Admiralty Seamount',\n",
       " 'Adolphe Point',\n",
       " 'Advance Creek',\n",
       " 'Advance Peak',\n",
       " 'Adventure Hill',\n",
       " 'Adventure Ridge',\n",
       " 'Adverse Creek',\n",
       " 'Adverse Glacier',\n",
       " 'Aeneid Peak',\n",
       " 'Aeronaut Glacier',\n",
       " 'Aeroplane Flat',\n",
       " 'Aeroplane Peak',\n",
       " 'Afton Burn',\n",
       " 'Afton Burn East Branch',\n",
       " 'Afton Peak',\n",
       " 'Afton Saddle',\n",
       " 'Agassiz Glacier',\n",
       " 'Agassiz Range',\n",
       " 'Agate Peak',\n",
       " 'Age Glacier',\n",
       " 'Agfa Knob',\n",
       " 'Agility Creek',\n",
       " 'Agnes Glacier',\n",
       " 'Agnes Stream',\n",
       " 'Ahaaha Rocks',\n",
       " 'Ahaura',\n",
       " 'Ahaura-Kopara Scenic Reserve',\n",
       " 'Ahaura River',\n",
       " 'Ahaura River East Amenity Area',\n",
       " 'Ahaura Road Amenity Area',\n",
       " 'Ahaura Terraces Ecological Area',\n",
       " 'Ahern Creek',\n",
       " 'Ahern Glacier',\n",
       " 'Ahiapurua',\n",
       " 'Ahiaruhe Shell Bank',\n",
       " 'Ahiaruhe Stream',\n",
       " 'Ahiawa Stream',\n",
       " 'Ahiherua Stream',\n",
       " 'Ahikaeaea',\n",
       " 'Ahikaroa',\n",
       " 'Ahikiwi',\n",
       " 'Ahikopua Point',\n",
       " 'Ahikōuka',\n",
       " 'Ahikumukumu Stream',\n",
       " 'Ahimanawa',\n",
       " 'Ahimanawa Range',\n",
       " 'Ahimanawa Stream',\n",
       " 'Ahimanu',\n",
       " 'Ahimanu Bay',\n",
       " 'Ahimate Stream',\n",
       " 'Ahimia',\n",
       " 'Ahioteatua',\n",
       " 'Ahipaepae',\n",
       " 'Ahipara',\n",
       " 'Ahipara Banks',\n",
       " 'Ahipara Bay',\n",
       " 'Ahipara Gumfields Historic Reserve',\n",
       " 'Ahipara Recreation Reserve',\n",
       " 'Ahipukahu Stream',\n",
       " 'Ahirau',\n",
       " 'Ahirau Scenic Reserve',\n",
       " 'Ahirau Stream',\n",
       " 'Ahiroa Point',\n",
       " 'Ahirongahua',\n",
       " 'Ahitarakihi Bay',\n",
       " 'Ahititi',\n",
       " 'Ahititi Stream',\n",
       " 'Ahiweka',\n",
       " 'Ahmadjian Peak',\n",
       " 'Ahoroa Rock',\n",
       " 'Ahoroa Scenic Reserve',\n",
       " 'Ahuahuiti Stream',\n",
       " 'Ahuahu Stream',\n",
       " 'Ahuapunga',\n",
       " 'Ahuateatua',\n",
       " 'Ahu-a-Tūranga Stream',\n",
       " 'Ahukawakawa (sphagnum moss swamp)',\n",
       " 'Ahukawhaka',\n",
       " 'Ahukuramu Stream',\n",
       " 'Ahu-o-te-Atua',\n",
       " 'Ahurangi Stream',\n",
       " 'Ahuriri',\n",
       " 'Ahuriri Bay',\n",
       " 'Ahuriri Bush Scenic Reserve',\n",
       " 'Ahuriri Conservation Park',\n",
       " 'Ahuriri Flat',\n",
       " 'Ahuriri Pa',\n",
       " 'Ahuriri Point',\n",
       " 'Ahuriri Post Office',\n",
       " 'Ahuriri Railway Station',\n",
       " 'Ahuriri River',\n",
       " 'Ahuriri River East Branch',\n",
       " 'Ahuriri Rock',\n",
       " 'Ahuriri Stream',\n",
       " 'Ahuroa',\n",
       " 'Ahuroa River',\n",
       " 'Ahuroa Scenic Reserve',\n",
       " 'Ahuroa Stream',\n",
       " 'Ahurua',\n",
       " 'Ahuru (Eastern Reef)',\n",
       " 'Ahu Stream',\n",
       " 'Aicken Range',\n",
       " 'Aickens',\n",
       " 'Aida Glacier',\n",
       " 'Aigrette',\n",
       " 'Aiguille Rouge',\n",
       " 'Aiguilles Canyon',\n",
       " 'Aiguilles Island',\n",
       " 'Aiguilles Rouges',\n",
       " 'Aiken Creek',\n",
       " 'Aiken Glacier',\n",
       " 'Ailsa Craig',\n",
       " 'Ailsa Creek',\n",
       " 'Ailsa Mountains',\n",
       " 'Ailsa Pass',\n",
       " 'Ailsa Stream',\n",
       " 'Ainley Peak',\n",
       " 'Ainsley Creek',\n",
       " 'Airdevronsix Glacier',\n",
       " 'Airdevronsix Icefalls',\n",
       " 'Airdrop Peak',\n",
       " 'Airedale',\n",
       " 'Airedale Reef',\n",
       " 'Airstrip Stream',\n",
       " 'Aitken Nunatak',\n",
       " 'Ajax Creek',\n",
       " 'Ajax Hill',\n",
       " 'Ajax Peak',\n",
       " 'Ajax Stream',\n",
       " 'A.J. Stream',\n",
       " 'Aka Aka',\n",
       " 'Aka Aka Stream',\n",
       " 'Akakahia Stream',\n",
       " 'Akakorako Point',\n",
       " 'Akapatiki Flat',\n",
       " 'Akaroa',\n",
       " 'Akaroa Harbour',\n",
       " 'Akaroa Head',\n",
       " 'Akaroa Head Scenic Reserve',\n",
       " 'Akaroa Marine Reserve',\n",
       " 'Akaroa Sea Valleys',\n",
       " 'Akaroa Valleys',\n",
       " 'Akatarawa River',\n",
       " 'Akatarawa River West',\n",
       " 'Akatarawa Saddle',\n",
       " 'Akatārere',\n",
       " 'Akatārere Historic Reserve',\n",
       " 'Akatarere Point',\n",
       " 'Akatarewa Stream',\n",
       " 'Akatea Stream',\n",
       " 'Akatere',\n",
       " 'Akatere District',\n",
       " 'Akatere Historic Reserve',\n",
       " 'Akaterere',\n",
       " 'Akaterere District',\n",
       " 'Akaterere Road',\n",
       " 'Akatere Road',\n",
       " 'Akatore Creek',\n",
       " 'Akeake Historic Reserve',\n",
       " 'Akeake Stream',\n",
       " 'Akerama',\n",
       " 'Akersten Bay',\n",
       " 'Akina',\n",
       " 'Ākiri',\n",
       " 'Ākitio',\n",
       " 'Akitio Ridge',\n",
       " 'Ākitio River',\n",
       " 'Akitio Trough',\n",
       " 'Akopov Seamounts',\n",
       " 'Ākura',\n",
       " 'Akura Stream',\n",
       " 'Alabama Stream',\n",
       " 'Alabaster Creek',\n",
       " 'Alabaster Pass',\n",
       " 'Alamein Peak',\n",
       " 'Alamein Range',\n",
       " 'Alan Gully',\n",
       " 'Alan Stream',\n",
       " 'Alarm Creek',\n",
       " 'Alarm Mountain',\n",
       " 'Alatna Valley',\n",
       " 'Albanus Glacier',\n",
       " 'Albany',\n",
       " 'Albany Heights',\n",
       " 'Albany Scenic Reserve',\n",
       " 'Albany Village',\n",
       " 'Albatross Canyon',\n",
       " 'Albatross Glacier',\n",
       " 'Albatross Point',\n",
       " 'Albatross Tarns',\n",
       " 'Alberich Glacier',\n",
       " 'Albert Burn',\n",
       " 'Albert Burn Saddle',\n",
       " 'Albert Channel',\n",
       " 'Albert Edward Peak',\n",
       " 'Albert Glacier',\n",
       " 'Albert Hill',\n",
       " 'Albert Lake',\n",
       " 'Albert Peak',\n",
       " 'Albert Reef',\n",
       " 'Alberts Cap',\n",
       " 'Alberts Creek',\n",
       " 'Albert Shoal',\n",
       " 'Albert Town',\n",
       " 'Albert Valley',\n",
       " 'Albion Inlet',\n",
       " 'Albion Square Historic Reserve',\n",
       " 'Albrecht Penck Glacier',\n",
       " 'Albury',\n",
       " 'Albury Range',\n",
       " 'Alb Valley',\n",
       " 'Alcorta Rocks',\n",
       " 'Alcove Stream',\n",
       " 'Alcyone Cone',\n",
       " 'Alderman Trough',\n",
       " 'Aldermen Islands (Ruamaahu) Nature Reserve',\n",
       " 'Aldermen Trough',\n",
       " 'Alderson Falls',\n",
       " 'Aldinga Creek',\n",
       " 'Aldi Peak',\n",
       " 'Aldridge Peak',\n",
       " 'Alec Creek',\n",
       " 'Alert Bay',\n",
       " 'Alert Stack',\n",
       " 'Alexander',\n",
       " 'Alexander Cone',\n",
       " 'Alexander Creek',\n",
       " 'Alexander Hill',\n",
       " 'Alexander McKay Cliffs',\n",
       " 'Alexander McKay Falls',\n",
       " 'Alexander Peak',\n",
       " 'Alexander Range',\n",
       " 'Alexander River',\n",
       " 'Alexanders Creek',\n",
       " 'Alexanders Hill',\n",
       " 'Alexander Valley',\n",
       " 'Alexandra',\n",
       " 'Alexandra Courthouse Historic Reserve',\n",
       " 'Alexandra Creek',\n",
       " 'Alexandra Lakeside Recreation Reserve',\n",
       " 'Alexandra Mountains',\n",
       " 'Alexandra Peak',\n",
       " 'Alexandra Redoubt',\n",
       " 'Alexandra Stream',\n",
       " 'Alex Knob',\n",
       " 'Alf Creek',\n",
       " 'Alfern Creek',\n",
       " 'Alford Forest',\n",
       " 'Alford Range',\n",
       " 'Alford Scenic Reserve',\n",
       " 'Alfred Peak',\n",
       " 'Alfred River',\n",
       " 'Alfreds Creek',\n",
       " 'Alfred Stream',\n",
       " 'Alfredton',\n",
       " 'Alfriston',\n",
       " 'Alga Lakes',\n",
       " 'Algal Lake',\n",
       " 'Algie Glacier',\n",
       " 'Algie Knoll',\n",
       " 'Algies Bay',\n",
       " 'Alhambra Rock',\n",
       " 'Alice Burn',\n",
       " 'Alice Creek',\n",
       " 'Alice Falls',\n",
       " 'Alice Glacier',\n",
       " 'Alice Glen',\n",
       " 'Alice Peak',\n",
       " 'Alice Peaks',\n",
       " 'Alicetown',\n",
       " 'Alick Glacier',\n",
       " 'Alioth Brook',\n",
       " 'Alison Creek',\n",
       " 'Allaire Peak',\n",
       " 'Allandale',\n",
       " 'Allan Hills',\n",
       " 'Allan Knob',\n",
       " 'Allans Beach',\n",
       " 'Allans Beach Recreation Reserve',\n",
       " 'Allans Beach Wildlife Management Reserve',\n",
       " 'Allans Point',\n",
       " 'Allanton',\n",
       " 'Allanton Wildlife Management Reserve',\n",
       " 'Allan Water',\n",
       " 'All Black Peak',\n",
       " 'All-Blacks Nunataks',\n",
       " 'All Day Bay',\n",
       " 'Allegro Valley',\n",
       " 'Allemand Peak',\n",
       " 'Allen Creek',\n",
       " 'Allen Range',\n",
       " 'Allen River',\n",
       " 'Allen River Left Branch',\n",
       " 'Allen River Right Branch',\n",
       " 'Allen Rocks',\n",
       " 'Allens Hill',\n",
       " 'Allen Strait (Guards Pass)',\n",
       " 'Allen Swamp',\n",
       " 'Allenton',\n",
       " 'Alley Glacier',\n",
       " 'Alligator Eyes',\n",
       " 'Alligator Head',\n",
       " 'Alligator Peak',\n",
       " 'Alligator Ridge',\n",
       " 'Allison Glacier',\n",
       " 'Allison Mountain',\n",
       " 'Allman Bay',\n",
       " 'All Nations Creek',\n",
       " 'Allom Bay',\n",
       " 'Allowitz Peak',\n",
       " 'Allports Island',\n",
       " 'Allports Island Scenic Reserve',\n",
       " 'All Round Peak',\n",
       " 'Allsops Bay',\n",
       " 'Allsops Bay Wildlife Reserve',\n",
       " 'All-Up Creek',\n",
       " 'Allup Ravine',\n",
       " 'Ally Trickle',\n",
       " 'Alma',\n",
       " 'Alma Bend',\n",
       " 'Alma Col',\n",
       " 'Almadale',\n",
       " 'Almadale Scenic Reserve',\n",
       " 'Alma Heights',\n",
       " 'Alma Hill',\n",
       " 'Alma Peak',\n",
       " 'Alma River',\n",
       " 'Alma Saddle',\n",
       " 'Alma Spur',\n",
       " 'Alma Stream',\n",
       " 'Almer Glacier',\n",
       " 'Almond',\n",
       " 'Alpha',\n",
       " 'Alpha Bluff',\n",
       " 'Alpha Burn',\n",
       " 'Alpha Creek',\n",
       " 'Alpha Mountain',\n",
       " 'Alph Caves',\n",
       " 'Alpheus Creek',\n",
       " 'Alph Lake',\n",
       " 'Alph River',\n",
       " 'Alph Stream',\n",
       " 'Alpine Creek',\n",
       " 'Alpine Flat',\n",
       " 'Alpine Lake',\n",
       " 'Alpine Lake/Ata Puai',\n",
       " 'Also Gully',\n",
       " 'Alsops Bay',\n",
       " 'Altar Mountain',\n",
       " 'Alt Glacier',\n",
       " 'Altimarloch',\n",
       " 'Alton',\n",
       " 'Alton Burn',\n",
       " 'Alum Gully',\n",
       " 'Alum Lake',\n",
       " 'Alva Creek',\n",
       " 'Alvarez Glacier',\n",
       " 'Amazon Basin',\n",
       " 'Amazon Cove',\n",
       " 'Amazon Peak',\n",
       " 'Amazons Breasts',\n",
       " 'Ambalada Peak',\n",
       " 'Amber Col',\n",
       " 'Amberley',\n",
       " 'Amberley Beach',\n",
       " 'Amberley Beach Lagoon',\n",
       " 'Ambrose Creek',\n",
       " 'Ambrose Saddle',\n",
       " 'Ambury Bluff',\n",
       " 'Ambury Park',\n",
       " 'Ameku',\n",
       " 'Amerikiwhati Island',\n",
       " 'Amethyst Ravine',\n",
       " 'Amherst Glacier',\n",
       " 'Amherst Rock',\n",
       " 'Amia Stream',\n",
       " 'Amisfield',\n",
       " 'Amisfield Burn',\n",
       " 'Amodeo Bay',\n",
       " 'Amodeo Rocks',\n",
       " 'Amohia Peak',\n",
       " 'Amokura Railway Station',\n",
       " 'Amorphous Glacier',\n",
       " 'Amos Glacier',\n",
       " 'Amphibole Peak',\n",
       " 'Amphion Peak',\n",
       " 'Amphitheatre',\n",
       " 'Amphitheatre Creek',\n",
       " 'Amphitheatre Glacier',\n",
       " 'Amphlett Stream',\n",
       " 'Amundsen Arm',\n",
       " 'Amundsen Coast',\n",
       " 'Amundsen Glacier',\n",
       " 'Amundsen Icefall',\n",
       " 'Amuri Pass',\n",
       " 'Amuri Plain',\n",
       " 'Amuri Range',\n",
       " 'Anaconda',\n",
       " 'Ana-kai-nehu',\n",
       " 'Anakakata Bay',\n",
       " 'Anakawau Bay',\n",
       " 'Anakiwa',\n",
       " 'Anakoha Bay',\n",
       " 'Anama',\n",
       " 'Anamata Stream',\n",
       " 'Ananui Caves',\n",
       " 'Ananui Creek',\n",
       " 'Ananui Falls',\n",
       " 'Ananui Stream',\n",
       " 'Anaomaki Point',\n",
       " 'Anaotamaraukura',\n",
       " 'Anaotehuruhuru Stream',\n",
       " 'Anapai Bay',\n",
       " 'Anapuhipuhi Blowhole',\n",
       " 'Anaputa Point',\n",
       " 'Anaputa Rock',\n",
       " 'Anarake Point',\n",
       " 'Anare Mountains',\n",
       " 'Anare Pass',\n",
       " 'Anarewa Point',\n",
       " 'Anarewa Stream',\n",
       " 'Anaru Stream',\n",
       " 'Anashkin Seamount',\n",
       " 'Anatakapau Bay',\n",
       " 'Anatakupu Island',\n",
       " 'Anataonga Cave',\n",
       " 'Anatimo',\n",
       " 'Anatohia Bay',\n",
       " 'Anatoki Bend',\n",
       " 'Anatoki Peak',\n",
       " 'Anatoki Range',\n",
       " 'Anatoki River',\n",
       " 'Anatori',\n",
       " 'Anatori River',\n",
       " 'Anatori River North Branch',\n",
       " 'Anatori River South Branch',\n",
       " 'Ana Tupapaku',\n",
       " 'Anaura Bay',\n",
       " 'Anaura Bay Recreation Reserve',\n",
       " 'Anaura Bay Scenic Reserve',\n",
       " 'Anauraiti Stream',\n",
       " 'Anaura Marae',\n",
       " 'Anaura Road',\n",
       " 'Anaura Stream',\n",
       " 'Anaweka River',\n",
       " 'Anawera Point',\n",
       " 'Anawera Stream',\n",
       " 'Anawhata',\n",
       " 'Anawhata Stream',\n",
       " 'Anawhenua Stream',\n",
       " 'Anchorage Arm',\n",
       " 'Anchorage Bay',\n",
       " 'Anchorage Cove',\n",
       " 'Anchorage Island',\n",
       " 'Anchor Bay',\n",
       " 'Anchor Island',\n",
       " 'Anchor Island Harbour',\n",
       " 'Anchorite Rock',\n",
       " 'Anchor Peak',\n",
       " 'Anchor Point',\n",
       " 'Anchor Ridge',\n",
       " 'Andermatten Glacier',\n",
       " 'Anderson Creek',\n",
       " 'Anderson Glen',\n",
       " 'Anderson Heights',\n",
       " 'Anderson Icefalls',\n",
       " 'Anderson Knoll',\n",
       " 'Anderson Peak',\n",
       " 'Anderson Peninsula',\n",
       " 'Anderson Pyramid',\n",
       " 'Anderson Ridge',\n",
       " 'Anderson Rock',\n",
       " 'Andersons Bay',\n",
       " 'Andersons Bay Inlet',\n",
       " 'Anderson Scarp',\n",
       " 'Andersons Cove',\n",
       " 'Andersons Flat',\n",
       " 'Andersons Floodway',\n",
       " 'Andersons Stream',\n",
       " 'Anderson Stream',\n",
       " 'Andersson Ridge',\n",
       " 'Anderton Glacier',\n",
       " 'Andesite Point',\n",
       " 'Andesite Stream',\n",
       " 'Andes Knolls',\n",
       " 'Andes Seamount Complex',\n",
       " 'Andrew Burn',\n",
       " 'Andrew Creek',\n",
       " 'Andrews Creek',\n",
       " 'Andrews Kauri Scenic Reserve',\n",
       " 'Andrews Peak',\n",
       " 'Andrews Pond Scientific Reserve',\n",
       " 'Andrews Ridge',\n",
       " 'Andrews Stream',\n",
       " 'Andrewville',\n",
       " 'Andrus Point',\n",
       " 'Andy Flat',\n",
       " 'Andy Glacier',\n",
       " 'Anehu Stream',\n",
       " 'Aneroa Stream',\n",
       " 'Angakakahi Stream',\n",
       " 'Angapaua',\n",
       " 'Angel Col',\n",
       " 'Angel Spur',\n",
       " 'Angelus Peak',\n",
       " 'Angelus Ridge',\n",
       " 'Angino Buttress',\n",
       " 'Angle Knob',\n",
       " 'Angle Knob Creek',\n",
       " 'Angle Peak',\n",
       " 'Angley Gully',\n",
       " 'Anguilla Burn',\n",
       " 'Angus Burn',\n",
       " 'Angus Flat',\n",
       " 'Anini Clearing',\n",
       " 'Anini Stream',\n",
       " 'Aniseed Hill',\n",
       " 'Aniseed Valley',\n",
       " 'Aniseed Valley Recreation Reserve',\n",
       " 'Aniseed Valley Scenic Reserve',\n",
       " 'Anise Hill',\n",
       " 'Anise Spur',\n",
       " 'Anita Bay',\n",
       " 'Anita Peak',\n",
       " 'Aniwaniwa',\n",
       " 'Aniwaniwa Falls',\n",
       " 'Āniwaniwa Falls',\n",
       " 'Aniwaniwa Stream',\n",
       " 'Aniwhaniwha Falls',\n",
       " 'Aniwhenua Falls',\n",
       " 'Anjou Point',\n",
       " 'Anna Glacier',\n",
       " 'Annan Gully',\n",
       " 'Anna Stream',\n",
       " 'Annat',\n",
       " 'Annear Creek',\n",
       " 'Annedale Stream',\n",
       " 'Anne Hill',\n",
       " 'Anne River',\n",
       " 'Anne Saddle',\n",
       " 'Annesbrook',\n",
       " 'Annetta Mountain',\n",
       " 'Annette Plateau',\n",
       " 'Annetts Creek',\n",
       " 'Annexation Point',\n",
       " 'Annick Peak',\n",
       " 'Annie Bay',\n",
       " 'Annie Peak',\n",
       " 'Anniversary Bluff',\n",
       " 'Anniversary Point',\n",
       " 'Annoyance Bluff',\n",
       " 'Ann Point',\n",
       " 'Anns Creek',\n",
       " 'Anopungapunga Stream',\n",
       " 'Anoura Road',\n",
       " 'Anslow Creek',\n",
       " 'Anson Stream',\n",
       " 'Anstey Stream',\n",
       " 'Ant',\n",
       " 'Antarctic Bay',\n",
       " 'Antarctic Ocean',\n",
       " 'Ant Hill',\n",
       " 'Ant Hill Glacier',\n",
       " 'Anthony Bluff',\n",
       " 'Anticline Creek',\n",
       " 'Anti Crow River',\n",
       " 'Antimony Crossing',\n",
       " 'Antimony Saddle',\n",
       " 'Antipodes Fracture Zone',\n",
       " 'Antipodes Island',\n",
       " 'Antipodes Island Group',\n",
       " 'Antipodes Islands Nature Reserve',\n",
       " 'Antipodes Scarp',\n",
       " 'Antoinette Mountain',\n",
       " 'Antomany Creek',\n",
       " 'Antonios Creek',\n",
       " 'Antonios Flat',\n",
       " 'Antony Creek',\n",
       " 'Ant Stream',\n",
       " 'Anu Whakatoro Glacier',\n",
       " 'Anvil Islet',\n",
       " 'Anvil Pond',\n",
       " 'Anxiety Point',\n",
       " 'Anybody Creek',\n",
       " 'Anzac Bay',\n",
       " 'Anzac Peaks',\n",
       " 'Ao-ata-te-pō',\n",
       " 'Aohanga',\n",
       " 'Aohanga River',\n",
       " 'Aohanga Road',\n",
       " 'Aokaparangi',\n",
       " 'Aokaparangi Creek',\n",
       " 'Aokautere',\n",
       " 'Aokautere Forest',\n",
       " 'Aongatete',\n",
       " 'Aongatete Recreation Reserve',\n",
       " 'Aongatete River',\n",
       " 'Aoparauri',\n",
       " 'Aoraki/Mount Cook',\n",
       " 'Aoraki/Mount Cook National Park',\n",
       " 'Aorangaia Island',\n",
       " 'Aorangi',\n",
       " 'Aorangi Creek',\n",
       " 'Aorangi Forest Park',\n",
       " 'Aorangi Island',\n",
       " 'Aorangi Lagoon',\n",
       " 'Aorangi No. 1 Drain',\n",
       " 'Aorangi Range',\n",
       " 'Aorangi Ridge',\n",
       " 'Aorangi Scenic Reserve',\n",
       " 'Aorangi Stream',\n",
       " 'Aorangi Trough',\n",
       " 'Aorangiwai River',\n",
       " 'Aorangiwai Scenic Reserve',\n",
       " 'Aorere',\n",
       " 'Aorere Caves',\n",
       " 'Aorere Caves Recreation Reserve',\n",
       " 'Aorere Peak',\n",
       " 'Aorere River',\n",
       " 'Aorere Saddle',\n",
       " 'Aorere Scenic Reserve',\n",
       " 'Aorere Stream',\n",
       " 'Aorta Ridge',\n",
       " 'Aotea',\n",
       " 'Aotea Cliffs',\n",
       " 'Aotea Conservation Park',\n",
       " 'Aotea Harbour',\n",
       " 'Aotea Heads Scientific Reserve',\n",
       " 'Aotea Reef',\n",
       " 'Aotearoa Marae',\n",
       " 'Aotearoa Pa',\n",
       " 'Aotearoa Point',\n",
       " 'Aotea Seamount',\n",
       " 'Aotea / Shelly Beach',\n",
       " 'Aotuhia',\n",
       " 'Aotutao Stream',\n",
       " 'Apahai Creek',\n",
       " 'Apakura Stream',\n",
       " 'Apanui',\n",
       " 'Aparima',\n",
       " 'Aparima Peaks',\n",
       " 'Aparima River',\n",
       " 'Aparima River South Branch',\n",
       " 'Apata',\n",
       " 'Apias Creek',\n",
       " 'Apirana Peak',\n",
       " 'Apiti',\n",
       " 'Āpiti',\n",
       " 'Apiti Bay',\n",
       " 'Āpiti Saddle',\n",
       " 'Apiti Scenic Reserve',\n",
       " 'Apiti Stream',\n",
       " 'Apley',\n",
       " 'Apocalypse Peaks',\n",
       " 'Apogee Creek',\n",
       " 'Apollo Creek',\n",
       " 'Apollo Peak',\n",
       " 'Aponga',\n",
       " 'Aponga Settlement Scenic Reserve',\n",
       " 'Aponga Stream',\n",
       " 'Apostles Rock',\n",
       " 'Apostrophe Island',\n",
       " 'Apotu',\n",
       " 'Apotu Stream',\n",
       " 'Apparition Spur',\n",
       " 'Appleby',\n",
       " 'Apple Tree Bay',\n",
       " 'Appletree Creek',\n",
       " 'Apple Tree Flat',\n",
       " 'Appleyard Stream',\n",
       " 'Appos Creek',\n",
       " 'Appos Flat',\n",
       " 'Apricot Flat',\n",
       " 'Apteryx Spur',\n",
       " 'Apuapu Bay',\n",
       " 'Apuau Channel',\n",
       " 'Apungi Pa',\n",
       " 'Aputa Stream',\n",
       " 'Aputerewa',\n",
       " 'Aputerewa Scenic Reserve',\n",
       " 'Ara',\n",
       " 'Araara Island',\n",
       " 'Araara Seamount',\n",
       " 'Arabella Rock',\n",
       " 'Aragorn Peak',\n",
       " 'Arahaki Lagoon',\n",
       " 'Araheke Stream',\n",
       " 'Arahiwi',\n",
       " 'Arahiwi Railway Scenic Reserve',\n",
       " 'Arahiwi Scenic Reserve',\n",
       " 'Arahura',\n",
       " 'Arahura River',\n",
       " 'Arai Terraces',\n",
       " 'Ārai Terraces',\n",
       " 'Araiteuru Pa',\n",
       " 'Arai-Te-Uru Recreation Reserve',\n",
       " 'Araiwhenua Stream',\n",
       " 'Arakaninihi Island',\n",
       " 'Arakihi',\n",
       " 'Arakitahi Stream',\n",
       " 'Arakoko Stream',\n",
       " 'Arakura',\n",
       " 'Aramahoe Stream',\n",
       " 'Aramaire',\n",
       " 'Aramaire Stream',\n",
       " 'Aramiro',\n",
       " 'Aramoana',\n",
       " 'Aramoana Domain Recreation Reserve',\n",
       " 'Aramoana Ecological Area',\n",
       " 'Aramoana Recreation Reserve',\n",
       " 'Aramoho',\n",
       " 'Aran Brook',\n",
       " 'Aranga',\n",
       " 'Aranga Beach',\n",
       " 'Aranui',\n",
       " 'Aranui Cave',\n",
       " 'Aranui Park',\n",
       " 'Arapae',\n",
       " 'Arapaoa',\n",
       " 'Arapaoa Island',\n",
       " 'Arapaoanoi Block',\n",
       " 'Arapaoanoi River',\n",
       " 'Arapaoanoi Road',\n",
       " 'Arapaoa River',\n",
       " 'Arapaoa River Scenic Reserve',\n",
       " 'Arapara Stream',\n",
       " 'Araparera',\n",
       " 'Araparera River',\n",
       " 'Arapatiki Bay',\n",
       " 'Arapawa',\n",
       " 'Arapawa Bay Post Office',\n",
       " 'Arapawa Island',\n",
       " 'Arapawa Island Scenic Reserve',\n",
       " 'Arapawaiti',\n",
       " 'Arapawaiti Point',\n",
       " 'Arapawanui Pa',\n",
       " 'Arapeti Stream',\n",
       " 'Arapito',\n",
       " 'Arapito Creek',\n",
       " 'Arapohue',\n",
       " 'Arapoto Stream',\n",
       " 'Arapuni',\n",
       " 'Arapuni Scenic Reserve',\n",
       " 'Ararata',\n",
       " 'Ararata Stream',\n",
       " 'Ararawa Stream',\n",
       " 'Ararimu',\n",
       " 'Ararimu Stream',\n",
       " 'Ararua',\n",
       " 'Arataha Stream',\n",
       " 'Arataki',\n",
       " 'Arataki Stream',\n",
       " 'Aratapu',\n",
       " 'Aratapu Creek',\n",
       " 'Aratataramoa',\n",
       " 'Arataua',\n",
       " 'Arataura',\n",
       " 'Arataura Point',\n",
       " 'Aratawa',\n",
       " 'Aratawa Bay',\n",
       " 'Aratawa Stream',\n",
       " 'Arateka Stream',\n",
       " 'Aratiatia',\n",
       " 'Aratiatia Rapids Recreation Reserve',\n",
       " 'Aratiatia Rapids Scenic Reserve',\n",
       " 'Aratika',\n",
       " 'Aratira (The Drop Scene)',\n",
       " 'Aratoro',\n",
       " 'Aratoro Scenic Reserve',\n",
       " 'Aratoro Stream',\n",
       " 'Aratoto Stream',\n",
       " 'Araukuku',\n",
       " 'Araukuku Pa',\n",
       " 'Arawaru',\n",
       " 'Arawata Caves Scenic Reserve',\n",
       " 'Arawata Forest',\n",
       " 'Arawata Glacier',\n",
       " 'Arawata River',\n",
       " 'Arawata Rock Bivouac',\n",
       " 'Arawata Saddle',\n",
       " 'Arawata Stream',\n",
       " 'Arawhata',\n",
       " 'Arawhata Canyon',\n",
       " 'Arawhata Forest',\n",
       " 'Arawhata Glacier',\n",
       " 'Arawhatapara Stream',\n",
       " 'Arawhata Point',\n",
       " 'Arawhata River',\n",
       " 'Arawhata Rock Bivouac',\n",
       " 'Arawhata Saddle',\n",
       " 'Arawhata Stream',\n",
       " 'Arawhatawhata Stream',\n",
       " 'Arawhatawhito',\n",
       " 'Arawiri',\n",
       " 'Arbor Rift',\n",
       " 'Arcade Creek',\n",
       " 'Arcade Glacier',\n",
       " 'Arcade Saddle',\n",
       " 'Archambault Ridge',\n",
       " 'Arch Creek',\n",
       " 'Archer Gully',\n",
       " 'Archer Peak',\n",
       " 'Archer Point',\n",
       " 'Archer Rock',\n",
       " 'Archers Stream',\n",
       " 'Archer Stream',\n",
       " 'Arch Falls',\n",
       " 'Arch Hill',\n",
       " 'Archies Creek',\n",
       " 'Archies Flat',\n",
       " 'Archies Spur',\n",
       " 'Architect Creek',\n",
       " 'Arch Point',\n",
       " 'Arch Reef',\n",
       " 'Archway',\n",
       " 'Archway Island',\n",
       " 'Archway Islands',\n",
       " 'Ardeer Burn',\n",
       " 'Ardeer Peak',\n",
       " 'Ardgour',\n",
       " 'Ardgowan',\n",
       " 'Ardlussa',\n",
       " 'Ardmore',\n",
       " 'Ardmore College Post Office',\n",
       " 'Ardmore Post Office',\n",
       " 'Ardneil Peak',\n",
       " 'Arduous Gully',\n",
       " 'Arena Creek',\n",
       " 'Arena Saddle',\n",
       " 'Arena Valley',\n",
       " 'Arero',\n",
       " 'Arerotepa Stream',\n",
       " 'Arete',\n",
       " 'Arete Forks',\n",
       " 'Arete Stream',\n",
       " 'Arethusa Icefall',\n",
       " 'Arethusa Pool',\n",
       " 'Argentum',\n",
       " 'Argo Glacier',\n",
       " 'Argo Gully',\n",
       " 'Argonaut Glacier',\n",
       " 'Argosy Glacier',\n",
       " 'Argosy Stream',\n",
       " 'Argument Creek',\n",
       " 'Argyle Beach',\n",
       " 'Argyle Burn',\n",
       " 'Argyle Creek',\n",
       " 'Argyle Hill',\n",
       " 'Argyle Pond',\n",
       " 'Argyle Rock',\n",
       " 'Argyle Substation',\n",
       " 'Argyll East',\n",
       " 'Argyll Water Race',\n",
       " 'Aria',\n",
       " ...]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(np.load('nzgazetteer.npy', allow_pickle=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:43:52.662693Z",
     "start_time": "2021-02-28T01:43:31.048139Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/myfriendscallmej/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3057: DtypeWarning: Columns (6) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n",
      "/Users/myfriendscallmej/anaconda3/lib/python3.7/site-packages/numpy/lib/arraysetops.py:569: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  mask |= (ar1 == a)\n"
     ]
    }
   ],
   "source": [
    "GeoNames = pd.read_csv('GeoNames.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:43:52.715102Z",
     "start_time": "2021-02-28T01:43:52.704256Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>geonameid</th>\n",
       "      <th>name</th>\n",
       "      <th>asciiname</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>country code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2986043</td>\n",
       "      <td>Pic de Font Blanca</td>\n",
       "      <td>Pic de Font Blanca</td>\n",
       "      <td>42.64991</td>\n",
       "      <td>1.53335</td>\n",
       "      <td>AD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2994701</td>\n",
       "      <td>Roc Mélé</td>\n",
       "      <td>Roc Mele</td>\n",
       "      <td>42.58765</td>\n",
       "      <td>1.74028</td>\n",
       "      <td>AD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3007683</td>\n",
       "      <td>Pic des Langounelles</td>\n",
       "      <td>Pic des Langounelles</td>\n",
       "      <td>42.61203</td>\n",
       "      <td>1.47364</td>\n",
       "      <td>AD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3017832</td>\n",
       "      <td>Pic de les Abelletes</td>\n",
       "      <td>Pic de les Abelletes</td>\n",
       "      <td>42.52535</td>\n",
       "      <td>1.73343</td>\n",
       "      <td>AD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3017833</td>\n",
       "      <td>Estany de les Abelletes</td>\n",
       "      <td>Estany de les Abelletes</td>\n",
       "      <td>42.52915</td>\n",
       "      <td>1.73362</td>\n",
       "      <td>AD</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   geonameid                     name                asciiname  latitude  \\\n",
       "0    2986043       Pic de Font Blanca       Pic de Font Blanca  42.64991   \n",
       "1    2994701                 Roc Mélé                 Roc Mele  42.58765   \n",
       "2    3007683     Pic des Langounelles     Pic des Langounelles  42.61203   \n",
       "3    3017832     Pic de les Abelletes     Pic de les Abelletes  42.52535   \n",
       "4    3017833  Estany de les Abelletes  Estany de les Abelletes  42.52915   \n",
       "\n",
       "   longitude country code  \n",
       "0    1.53335           AD  \n",
       "1    1.74028           AD  \n",
       "2    1.47364           AD  \n",
       "3    1.73343           AD  \n",
       "4    1.73362           AD  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GeoNames.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T01:44:02.820279Z",
     "start_time": "2021-02-28T01:43:52.753852Z"
    }
   },
   "outputs": [],
   "source": [
    "GeoNamesUnique = GeoNames.name.unique()\n",
    "GeoNamesAnt = GeoNames[GeoNames['country code']=='AQ'].name.unique()\n",
    "GeoNamesNZ = GeoNames[GeoNames['country code']=='NZ'].name.unique()\n",
    "dropWords = ['Inner', 'Harbour', 'Lake', 'Fig', 'Valleys', 'Soil', \n",
    "             'Lakes', 'South', 'North', 'Inner', 'Mount', 'Frozen', 'Oceans',\n",
    "            'Upper', 'Contour']\n",
    "GeoNamesFiltered = [place for place in GeoNamesUnique if place not in dropWords]\n",
    "\n",
    "del GeoNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-28T02:16:22.013292Z",
     "start_time": "2021-02-28T02:16:02.842747Z"
    }
   },
   "outputs": [],
   "source": [
    "np.save('GeoNamesAnt.npy', GeoNamesAnt)\n",
    "np.save('GeoNamesNZ.npy', GeoNamesNZ)\n",
    "np.save('GeoNamesFiltered.npy', GeoNamesFiltered)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:10:34.076271Z",
     "start_time": "2021-02-08T03:10:34.072963Z"
    }
   },
   "outputs": [],
   "source": [
    "import difflib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T08:20:08.588486Z",
     "start_time": "2021-02-26T08:20:08.525590Z"
    }
   },
   "outputs": [],
   "source": [
    "def find_locations(doc):\n",
    "    places = []\n",
    "    for x in doc.ents:\n",
    "        if x.label_ is 'LOC':\n",
    "            if doc[x.start].text == 'the' or doc[x.start].text == 'The':\n",
    "                places.append([x.text[4:], x.text, x.start+1, x.end, doc[x.start+1].idx, doc[x.end-1].idx + len(doc[x.end-1]), x.sent])\n",
    "            elif doc[x.start].text== 'Mt':\n",
    "                places.append(['Mount' + x.text[2:], x.text, x.start+1, x.end, doc[x.start].idx, doc[x.end-1].idx + len(doc[x.end-1]), x.sent])\n",
    "            elif doc[x.start].text== 'Mt.':\n",
    "                places.append(['Mount' + x.text[3:], x.text, x.start+1, x.end, doc[x.start].idx, doc[x.end-1].idx + len(doc[x.end-1]), x.sent])              \n",
    "            else:\n",
    "                places.append([x.text, x. text, x.start, x.end, doc[x.start].idx, doc[x.end-1].idx + len(doc[x.end-1]), x.sent])\n",
    "    \n",
    "    df = pd.DataFrame(places,columns=['Location', 'Original Tokenised Text','Start_Token', 'End_Token','Start_Index','End_Index','Sentence']\n",
    "             )\n",
    "#     Check possible locations in gazetteers for exact matches\n",
    "    df['NZGazAnt']=[True if X in nzgazAntartica else False for X in df.Location ]\n",
    "    df['NZGaz']=[True if (df['NZGazAnt'] is True or X in nzgaz) else False for X in df.Location ]\n",
    "    df['ScarNZ']=[True if X in SCARNZnames else False for X in df.Location ]\n",
    "    df['ScarGlobal']=[True if X in SCARGlobalnames else False for X in df.Location]\n",
    "    df['GeoNamesNZ']=[True if X in GeoNamesNZ else False for X in df.Location]\n",
    "    df['GeoNamesAnt']=[True if X in GeoNamesAnt else False for X in df.Location]\n",
    "    df['GeoNames']=[True if (df['GeoNamesNZ'] is True or df['GeoNamesAnt'] is True or X in GeoNamesFiltered) else False for X in df.Location ]\n",
    "    df['inNZ'] = df.NZGaz | df.NZGazAnt | df.GeoNamesNZ\n",
    "    df['inAntarctica'] = df.ScarGlobal | df.NZGazAnt | df.GeoNamesAnt\n",
    "    df['exactMatch'] = df.NZGaz | df.ScarGlobal | df.GeoNames\n",
    "    \n",
    "    \n",
    "#     filter locations by those not found in Antarctica or New Zealand gazeteers \n",
    "    df2=df[(df['inAntarctica']==False) & (df['inNZ']==False)].copy()\n",
    "    \n",
    "\n",
    "#     look through these remaining locations (including those found only in GeoNames) for close matches \n",
    "#     eg. McMurdo Dry Valley v McMurdo Dry Valley or for partial matches eg. Ross Sea Region == Ross Sea \n",
    "   \n",
    "    df2['Close_Match_NZGazAnt']=  [difflib.get_close_matches(X, nzgazAntartica, cutoff=0.9)[0]\n",
    "                                                  if len(difflib.get_close_matches(X, nzgazAntartica, cutoff=0.9))>0\n",
    "                                                  else np.nan \n",
    "                                                  for X in df2['Location']] \n",
    "    df2['Close_Match_NZGaz']=  [difflib.get_close_matches(X, nzgaz, cutoff=0.9)[0]\n",
    "                                                      if len(difflib.get_close_matches(X, nzgaz, cutoff=0.9))>0\n",
    "                                                      else np.nan \n",
    "                                                      for X in df2['Location']] \n",
    "    df2['Close_Match_ScarNZ']=  [difflib.get_close_matches(X, SCARNZnames, cutoff=0.9)[0] \n",
    "                                                      if len(difflib.get_close_matches(X, SCARNZnames, cutoff=0.9))>0\n",
    "                                                      else np.nan \n",
    "                                                      for X in df2['Location']] \n",
    "    df2['Close_Match_ScarGlobal']=  [difflib.get_close_matches(X, SCARGlobalnames, cutoff=0.9)[0]\n",
    "                                                      if (X !='Antarctic' and len(difflib.get_close_matches(X, SCARGlobalnames, cutoff=0.9))>0)\n",
    "                                                      else np.nan \n",
    "                                                      for X in df2['Location']] \n",
    "    df2['Close_Match_GeoNamesNZ'] = [difflib.get_close_matches(X, GeoNamesNZ, cutoff=0.9)[0] \n",
    "                                                      if len(difflib.get_close_matches(X, GeoNamesNZ, cutoff=0.9))>0\n",
    "                                                      else np.nan \n",
    "                                                      for X in df2['Location']] \n",
    "    df2['Close_Match_GeoNamesAnt']=  [difflib.get_close_matches(X, GeoNamesAnt, cutoff=0.9)[0]\n",
    "                                                      if (X !='Antarctic' and len(difflib.get_close_matches(X, GeoNamesAnt, cutoff=0.9))>0)\n",
    "                                                      else np.nan \n",
    "                                                      for X in df2['Location']]\n",
    "    df2['Close_Match']= (df2.Close_Match_NZGazAnt.notna() | df2.Close_Match_NZGaz.notna() | \n",
    "                         df2.Close_Match_ScarNZ.notna() | df2.Close_Match_ScarGlobal.notna() | \n",
    "                         df2.Close_Match_GeoNamesAnt.notna())\n",
    "    \n",
    "    df2['PartialMatch_NZGazAnt']=getBiggestSubStringMatch(doc, df2, nzgazAntartica)\n",
    "    df2['PartialMatch_NZGaz']=getBiggestSubStringMatch(doc, df2, nzgaz)\n",
    "    df2['PartialMatch_ScarNZ']=getBiggestSubStringMatch(doc, df2, SCARNZnames)\n",
    "    df2['PartialMatch_ScarGlobal']=getBiggestSubStringMatch(doc, df2, SCARGlobalnames)\n",
    "    df2['PartialMatch_GeoNamesNZ']=getBiggestSubStringMatch(doc, df2, GeoNamesNZ)\n",
    "    df2['PartialMatch_GeoNamesAnt']=getBiggestSubStringMatch(doc, df2, GeoNamesAnt)\n",
    "    df2['PartialMatch'] = (df2['PartialMatch_NZGazAnt'].notna() | df2['PartialMatch_NZGaz'].notna() | \n",
    "                           df2['PartialMatch_ScarNZ'].notna() | df2['PartialMatch_ScarGlobal'].notna() | \n",
    "                           df2['PartialMatch_GeoNamesAnt'].notna())\n",
    "#     df2['PartialMatch'] = [X if X==True else np.nan for X in df2['PartialMatch']]\n",
    "    \n",
    "#     merge the filtered dataframe (now with close and partial matches) with the unfiltered dataframe\n",
    "    df_unified = pd.merge(df, df2, how='left')\n",
    "    \n",
    "#     redo the inNZ,inAntarctica and Found columns to include close and partial matches\n",
    "    df_unified.drop(columns=['inNZ', 'inAntarctica'], inplace=True)\n",
    "    df_unified['inNZ'] = (df_unified.NZGaz | df_unified.NZGazAnt | df_unified.GeoNamesNZ | \n",
    "                          df_unified.Close_Match_NZGaz | df_unified.Close_Match_NZGazAnt | \n",
    "                          df_unified.Close_Match_GeoNamesNZ | df_unified.PartialMatch_NZGaz | \n",
    "                          df_unified.PartialMatch_NZGazAnt | df_unified.PartialMatch_GeoNamesNZ)\n",
    "    df_unified['inAntarctica'] = (df_unified.ScarGlobal | df_unified.NZGazAnt | df_unified.GeoNamesAnt |\n",
    "                                  df_unified.Close_Match_ScarGlobal | df_unified.Close_Match_NZGazAnt | \n",
    "                                  df_unified.Close_Match_GeoNamesAnt | df_unified.PartialMatch_ScarGlobal | \n",
    "                                  df_unified.PartialMatch_NZGazAnt | df_unified.PartialMatch_GeoNamesAnt)\n",
    "    df_unified['Found'] = df_unified.exactMatch | df_unified.Close_Match | df_unified.PartialMatch\n",
    "    return df_unified[df_unified.Found==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T04:29:48.477852Z",
     "start_time": "2021-02-26T04:29:48.427765Z"
    }
   },
   "outputs": [],
   "source": [
    "def getBiggestSubStringMatch(document, df, gazetteer):\n",
    "    matches=[]\n",
    "    for row in df.index:\n",
    "#         print('working on row', row)\n",
    "        fullStart = df.loc[row,'Start_Token']\n",
    "        fullEnd = df.loc[row,'End_Token']\n",
    "        subLength = fullEnd-fullStart-1\n",
    "#         print(fullEnd, fullStart, subLength)\n",
    "        Found = False\n",
    "        while subLength >1:\n",
    "            subStart = fullStart\n",
    "            subEnd = fullStart+subLength\n",
    "            while subEnd<=fullEnd:\n",
    "                subString = document[subStart:subEnd].text\n",
    "#                 print('looking for', subString)\n",
    "                if subString in gazetteer:\n",
    "                    matches.append(subString)\n",
    "                    Found=True,\n",
    "#                     print('found it!')\n",
    "                    break\n",
    "                subStart+=1\n",
    "                subEnd+=1\n",
    "            if Found:\n",
    "                    break\n",
    "            else:\n",
    "                subLength-=1\n",
    "        if Found==False:\n",
    "            matches.append(np.nan)\n",
    "    return matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T04:20:32.015388Z",
     "start_time": "2021-02-26T04:20:32.011185Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "101"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document[9].idx + len(document[9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T04:19:24.442017Z",
     "start_time": "2021-02-26T04:19:24.438370Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Barrientos Island'"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc[84:101]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using Spacy's Small Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T08:20:26.131689Z",
     "start_time": "2021-02-26T08:20:16.662111Z"
    }
   },
   "outputs": [],
   "source": [
    "locations = find_locations(document)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T08:20:27.733808Z",
     "start_time": "2021-02-26T08:20:27.703192Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Location</th>\n",
       "      <th>Original Tokenised Text</th>\n",
       "      <th>Start_Token</th>\n",
       "      <th>End_Token</th>\n",
       "      <th>Start_Index</th>\n",
       "      <th>End_Index</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>NZGazAnt</th>\n",
       "      <th>NZGaz</th>\n",
       "      <th>ScarNZ</th>\n",
       "      <th>...</th>\n",
       "      <th>PartialMatch_NZGazAnt</th>\n",
       "      <th>PartialMatch_NZGaz</th>\n",
       "      <th>PartialMatch_ScarNZ</th>\n",
       "      <th>PartialMatch_ScarGlobal</th>\n",
       "      <th>PartialMatch_GeoNamesNZ</th>\n",
       "      <th>PartialMatch_GeoNamesAnt</th>\n",
       "      <th>PartialMatch</th>\n",
       "      <th>inNZ</th>\n",
       "      <th>inAntarctica</th>\n",
       "      <th>Found</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>103</td>\n",
       "      <td>113</td>\n",
       "      <td>(https://doi.org/10.1007/s00300-017-2230-0, OR...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>214</td>\n",
       "      <td>215</td>\n",
       "      <td>1299</td>\n",
       "      <td>1309</td>\n",
       "      <td>(A, total, of, ten, species, were, recorded, ,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>422</td>\n",
       "      <td>423</td>\n",
       "      <td>2573</td>\n",
       "      <td>2583</td>\n",
       "      <td>(The, online, version, of, this, pelago, ,, An...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pacific Ocean</td>\n",
       "      <td>Pacific Ocean</td>\n",
       "      <td>606</td>\n",
       "      <td>608</td>\n",
       "      <td>3584</td>\n",
       "      <td>3597</td>\n",
       "      <td>(3077000, Pacific, Ocean, Ui, ctose@, ee, ))</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>678</td>\n",
       "      <td>679</td>\n",
       "      <td>3883</td>\n",
       "      <td>3893</td>\n",
       "      <td>((, Sites, 13, -, 01, to, 13, -, 39, ), most, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Location Original Tokenised Text  Start_Token  End_Token  Start_Index  \\\n",
       "0     Antarctica              Antarctica           11         12          103   \n",
       "1     Antarctica              Antarctica          214        215         1299   \n",
       "2     Antarctica              Antarctica          422        423         2573   \n",
       "3  Pacific Ocean           Pacific Ocean          606        608         3584   \n",
       "4     Antarctica              Antarctica          678        679         3883   \n",
       "\n",
       "   End_Index                                           Sentence  NZGazAnt  \\\n",
       "0        113  (https://doi.org/10.1007/s00300-017-2230-0, OR...     False   \n",
       "1       1309  (A, total, of, ten, species, were, recorded, ,...     False   \n",
       "2       2583  (The, online, version, of, this, pelago, ,, An...     False   \n",
       "3       3597       (3077000, Pacific, Ocean, Ui, ctose@, ee, ))     False   \n",
       "4       3893  ((, Sites, 13, -, 01, to, 13, -, 39, ), most, ...     False   \n",
       "\n",
       "   NZGaz  ScarNZ  ...  PartialMatch_NZGazAnt  PartialMatch_NZGaz  \\\n",
       "0  False   False  ...                    NaN                 NaN   \n",
       "1  False   False  ...                    NaN                 NaN   \n",
       "2  False   False  ...                    NaN                 NaN   \n",
       "3  False   False  ...                    NaN                 NaN   \n",
       "4  False   False  ...                    NaN                 NaN   \n",
       "\n",
       "   PartialMatch_ScarNZ  PartialMatch_ScarGlobal  PartialMatch_GeoNamesNZ  \\\n",
       "0                  NaN                      NaN                      NaN   \n",
       "1                  NaN                      NaN                      NaN   \n",
       "2                  NaN                      NaN                      NaN   \n",
       "3                  NaN                      NaN                      NaN   \n",
       "4                  NaN                      NaN                      NaN   \n",
       "\n",
       "  PartialMatch_GeoNamesAnt PartialMatch   inNZ inAntarctica Found  \n",
       "0                      NaN          NaN  False         True  True  \n",
       "1                      NaN          NaN  False         True  True  \n",
       "2                      NaN          NaN  False         True  True  \n",
       "3                      NaN        False  False        False  True  \n",
       "4                      NaN          NaN  False         True  True  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "locations.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T08:20:45.487096Z",
     "start_time": "2021-02-26T08:20:45.482686Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Antarctica'"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc[1299:1309]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:22.975180Z",
     "start_time": "2021-01-31T01:42:22.890482Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Location</th>\n",
       "      <th>Original Tokenised Text</th>\n",
       "      <th>Start</th>\n",
       "      <th>End</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>NZGazAnt</th>\n",
       "      <th>NZGaz</th>\n",
       "      <th>ScarNZ</th>\n",
       "      <th>ScarGlobal</th>\n",
       "      <th>GeoNamesNZ</th>\n",
       "      <th>...</th>\n",
       "      <th>PartialMatch_NZGazAnt</th>\n",
       "      <th>PartialMatch_NZGaz</th>\n",
       "      <th>PartialMatch_ScarNZ</th>\n",
       "      <th>PartialMatch_ScarGlobal</th>\n",
       "      <th>PartialMatch_GeoNamesNZ</th>\n",
       "      <th>PartialMatch_GeoNamesAnt</th>\n",
       "      <th>PartialMatch</th>\n",
       "      <th>inNZ</th>\n",
       "      <th>inAntarctica</th>\n",
       "      <th>Found</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>(CrossMark, \\n, Collembola, of, Barrientos, Is...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>221</td>\n",
       "      <td>222</td>\n",
       "      <td>(A, total, of, ten, species, were, recorded, ,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>432</td>\n",
       "      <td>433</td>\n",
       "      <td>(The, online, version, of, this, pelago, ,, An...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pacific Ocean</td>\n",
       "      <td>Pacific Ocean</td>\n",
       "      <td>647</td>\n",
       "      <td>649</td>\n",
       "      <td>(BY, 4, of, een, le, \\n, 3077500, \\n, 3077000,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>724</td>\n",
       "      <td>725</td>\n",
       "      <td>((, Sites, 13, -, 01, to, 13, -, 39, ), \\n, mo...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>888</td>\n",
       "      <td>889</td>\n",
       "      <td>(Collembola, in, Antarctica, are, relatively, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>the Antarctic Peninsula</td>\n",
       "      <td>946</td>\n",
       "      <td>948</td>\n",
       "      <td>(Particularly, ,, Collembola, from, maritime, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>997</td>\n",
       "      <td>999</td>\n",
       "      <td>(However, ,, the, edaphic, fauna, of, Barrient...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>1334</td>\n",
       "      <td>1336</td>\n",
       "      <td>(The, aim, of, this, study, is, to, identify, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>1364</td>\n",
       "      <td>1365</td>\n",
       "      <td>(The, aim, of, this, study, is, to, identify, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>2940</td>\n",
       "      <td>2941</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>3025</td>\n",
       "      <td>3026</td>\n",
       "      <td>(All, species, have, already, been, recorded, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>4099</td>\n",
       "      <td>4101</td>\n",
       "      <td>(The, least, favorable, substrates, for, C., a...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>5177</td>\n",
       "      <td>5179</td>\n",
       "      <td>(This, is, the, first, census, of, Collembola,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>5218</td>\n",
       "      <td>5219</td>\n",
       "      <td>((, 2014, ), for, the, maritime, Antarctica, r...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>5345</td>\n",
       "      <td>5346</td>\n",
       "      <td>(Accordingly, ,, recent, monitoring, activitie...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>5349</td>\n",
       "      <td>5351</td>\n",
       "      <td>(Accordingly, ,, recent, monitoring, activitie...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>5422</td>\n",
       "      <td>5423</td>\n",
       "      <td>(Hypogastrura, viatica, and, Mesaphorura, macr...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>South Georgia</td>\n",
       "      <td>South Georgia</td>\n",
       "      <td>5548</td>\n",
       "      <td>5550</td>\n",
       "      <td>(Its, presence, could, affect, the, native, Co...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>the Antarctic Peninsula</td>\n",
       "      <td>5768</td>\n",
       "      <td>5770</td>\n",
       "      <td>(Collembola, is, the, dominant, non, -, indige...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Deception Island</td>\n",
       "      <td>Deception Island</td>\n",
       "      <td>5864</td>\n",
       "      <td>5866</td>\n",
       "      <td>(The, first, non, -, indigenous, invertebrate,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>5939</td>\n",
       "      <td>5941</td>\n",
       "      <td>(These, findings, suggest, that, the, introduc...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>5942</td>\n",
       "      <td>5945</td>\n",
       "      <td>(These, findings, suggest, that, the, introduc...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Macquarie Island</td>\n",
       "      <td>Macquarie Island</td>\n",
       "      <td>5961</td>\n",
       "      <td>5963</td>\n",
       "      <td>(The, species, M., macrochaeta, was, reported,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>6011</td>\n",
       "      <td>6012</td>\n",
       "      <td>(All, locations, colonized, by, non, -, indige...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>6201</td>\n",
       "      <td>6203</td>\n",
       "      <td>(In, this, context, ,, there, is, a, real, pos...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>6847</td>\n",
       "      <td>6848</td>\n",
       "      <td>(This, species, plays, an, essential, role, in...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>7041</td>\n",
       "      <td>7043</td>\n",
       "      <td>(On, Barrientos, Island, ,, the, species, with...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>7204</td>\n",
       "      <td>7206</td>\n",
       "      <td>(This, indicates, that, F., grisea, was, widel...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>7414</td>\n",
       "      <td>7416</td>\n",
       "      <td>(The, reason, is, that, a, stratified, samplin...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Heard Island</td>\n",
       "      <td>Heard Island</td>\n",
       "      <td>7580</td>\n",
       "      <td>7582</td>\n",
       "      <td>(Archisotoma, brucei, is, usually, classified,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>7783</td>\n",
       "      <td>7785</td>\n",
       "      <td>(The, present, study, increases, the, knowledg...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>7807</td>\n",
       "      <td>7810</td>\n",
       "      <td>(The, present, study, increases, the, knowledg...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>7820</td>\n",
       "      <td>7821</td>\n",
       "      <td>(Although, all, recorded, species, had, alread...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>Barrientos Island</td>\n",
       "      <td>8312</td>\n",
       "      <td>8314</td>\n",
       "      <td>(Barrientos, Island, .)</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Mediterranean</td>\n",
       "      <td>Mediterranean</td>\n",
       "      <td>8367</td>\n",
       "      <td>8368</td>\n",
       "      <td>(Accessed, 10, Feb, 2015, \\n, Baena, CW, ,, An...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Signy Island</td>\n",
       "      <td>Signy Island</td>\n",
       "      <td>8541</td>\n",
       "      <td>8543</td>\n",
       "      <td>(1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>South Orkney Islands</td>\n",
       "      <td>the South Orkney Islands</td>\n",
       "      <td>8616</td>\n",
       "      <td>8619</td>\n",
       "      <td>(Collembola, from, the, South, Orkney, Islands...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>South Sandwich Islands</td>\n",
       "      <td>South Sandwich Islands</td>\n",
       "      <td>8764</td>\n",
       "      <td>8767</td>\n",
       "      <td>(The, terrestrial, microarthropod, fauna, of, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>8802</td>\n",
       "      <td>8803</td>\n",
       "      <td>((, Collembola, :, Hypogastruridae, ), at, Cap...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>8917</td>\n",
       "      <td>8918</td>\n",
       "      <td>(Fur, seals, ,, whales, and, tourists, :, a, c...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>9067</td>\n",
       "      <td>9069</td>\n",
       "      <td>(1995, ), Collembola, from, the, Scotia, Arc, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>9117</td>\n",
       "      <td>9120</td>\n",
       "      <td>(Collembola, fauna, of, the, South, Shetland, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Macquarie Island</td>\n",
       "      <td>Macquarie Island</td>\n",
       "      <td>9180</td>\n",
       "      <td>9182</td>\n",
       "      <td>(Marine, littoral, and, coastal, Collembola, o...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Deception Island</td>\n",
       "      <td>Deception Island</td>\n",
       "      <td>9231</td>\n",
       "      <td>9233</td>\n",
       "      <td>(Global, Collembola, on, Deception, Island, .)</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>the Antarctic Peninsula</td>\n",
       "      <td>9290</td>\n",
       "      <td>9292</td>\n",
       "      <td>(Notes, on, arthropod, populations, in, the, A...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>South Orkney Islands</td>\n",
       "      <td>South Orkney Islands</td>\n",
       "      <td>9295</td>\n",
       "      <td>9298</td>\n",
       "      <td>(South, Orkney, Islands, area, .)</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>9521</td>\n",
       "      <td>9522</td>\n",
       "      <td>(Oxford, University, Press, ,, Oxford, \\n, Hug...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Arctic</td>\n",
       "      <td>Arctic</td>\n",
       "      <td>9763</td>\n",
       "      <td>9764</td>\n",
       "      <td>(Birds, spread, soil, microarthropods, to, Arc...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Mediterranean</td>\n",
       "      <td>Mediterranean</td>\n",
       "      <td>9839</td>\n",
       "      <td>9840</td>\n",
       "      <td>(Influence, of, forest, cover, and, herbaceous...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Marion Island</td>\n",
       "      <td>Marion Island</td>\n",
       "      <td>9904</td>\n",
       "      <td>9906</td>\n",
       "      <td>(The, influence, of, elephant, seals, on, the,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>9987</td>\n",
       "      <td>9988</td>\n",
       "      <td>(Antarctophorus, —, a, new, genus, of, Isotomi...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>10078</td>\n",
       "      <td>10080</td>\n",
       "      <td>(Native, terrestrial, invertebrate, fauna, fro...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>10152</td>\n",
       "      <td>10153</td>\n",
       "      <td>(A, new, species, and, redescriptions, of, Col...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>10229</td>\n",
       "      <td>10230</td>\n",
       "      <td>(Determinants, of, terrestrial, arthropod, com...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>10470</td>\n",
       "      <td>10473</td>\n",
       "      <td>((, Collembola, ,, Neanuridae, ), from, King, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>10474</td>\n",
       "      <td>10475</td>\n",
       "      <td>((, Collembola, ,, Neanuridae, ), from, King, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>10518</td>\n",
       "      <td>10519</td>\n",
       "      <td>(New, records, of, Collembola, and, Acarina, i...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>Heard Island</td>\n",
       "      <td>Heard Island</td>\n",
       "      <td>10563</td>\n",
       "      <td>10565</td>\n",
       "      <td>(Wise, KAJ, (, 1970b, ), Collembola, of, Heard...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>59 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Location   Original Tokenised Text  Start    End  \\\n",
       "0               Antarctica                Antarctica     14     15   \n",
       "2               Antarctica                Antarctica    221    222   \n",
       "3               Antarctica                Antarctica    432    433   \n",
       "4            Pacific Ocean             Pacific Ocean    647    649   \n",
       "5               Antarctica                Antarctica    724    725   \n",
       "6               Antarctica                Antarctica    888    889   \n",
       "7      Antarctic Peninsula   the Antarctic Peninsula    946    948   \n",
       "8        Barrientos Island         Barrientos Island    997    999   \n",
       "9        Barrientos Island         Barrientos Island   1334   1336   \n",
       "10              Antarctica                Antarctica   1364   1365   \n",
       "11              Antarctica                Antarctica   2940   2941   \n",
       "12              Antarctica                Antarctica   3025   3026   \n",
       "13       Barrientos Island         Barrientos Island   4099   4101   \n",
       "14       Barrientos Island         Barrientos Island   5177   5179   \n",
       "15              Antarctica                Antarctica   5218   5219   \n",
       "16              Antarctica                Antarctica   5345   5346   \n",
       "17     Antarctic Peninsula       Antarctic Peninsula   5349   5351   \n",
       "18              Antarctica                Antarctica   5422   5423   \n",
       "19           South Georgia             South Georgia   5548   5550   \n",
       "20     Antarctic Peninsula   the Antarctic Peninsula   5768   5770   \n",
       "21        Deception Island          Deception Island   5864   5866   \n",
       "23     Antarctic Peninsula       Antarctic Peninsula   5939   5941   \n",
       "24  South Shetland Islands    South Shetland Islands   5942   5945   \n",
       "25        Macquarie Island          Macquarie Island   5961   5963   \n",
       "27              Antarctica                Antarctica   6011   6012   \n",
       "28       Barrientos Island         Barrientos Island   6201   6203   \n",
       "30              Antarctica                Antarctica   6847   6848   \n",
       "31       Barrientos Island         Barrientos Island   7041   7043   \n",
       "32       Barrientos Island         Barrientos Island   7204   7206   \n",
       "33       Barrientos Island         Barrientos Island   7414   7416   \n",
       "34            Heard Island              Heard Island   7580   7582   \n",
       "35       Barrientos Island         Barrientos Island   7783   7785   \n",
       "36  South Shetland Islands    South Shetland Islands   7807   7810   \n",
       "37              Antarctica                Antarctica   7820   7821   \n",
       "38       Barrientos Island         Barrientos Island   8312   8314   \n",
       "39           Mediterranean             Mediterranean   8367   8368   \n",
       "41            Signy Island              Signy Island   8541   8543   \n",
       "42    South Orkney Islands  the South Orkney Islands   8616   8619   \n",
       "43  South Sandwich Islands    South Sandwich Islands   8764   8767   \n",
       "44              Antarctica                Antarctica   8802   8803   \n",
       "45              Antarctica                Antarctica   8917   8918   \n",
       "46     Antarctic Peninsula       Antarctic Peninsula   9067   9069   \n",
       "47  South Shetland Islands    South Shetland Islands   9117   9120   \n",
       "48        Macquarie Island          Macquarie Island   9180   9182   \n",
       "49        Deception Island          Deception Island   9231   9233   \n",
       "50     Antarctic Peninsula   the Antarctic Peninsula   9290   9292   \n",
       "51    South Orkney Islands      South Orkney Islands   9295   9298   \n",
       "52              Antarctica                Antarctica   9521   9522   \n",
       "53                  Arctic                    Arctic   9763   9764   \n",
       "54           Mediterranean             Mediterranean   9839   9840   \n",
       "55           Marion Island             Marion Island   9904   9906   \n",
       "56              Antarctica                Antarctica   9987   9988   \n",
       "57     Antarctic Peninsula       Antarctic Peninsula  10078  10080   \n",
       "58              Antarctica                Antarctica  10152  10153   \n",
       "59              Antarctica                Antarctica  10229  10230   \n",
       "60  South Shetland Islands    South Shetland Islands  10470  10473   \n",
       "61              Antarctica                Antarctica  10474  10475   \n",
       "62              Antarctica                Antarctica  10518  10519   \n",
       "63            Heard Island              Heard Island  10563  10565   \n",
       "\n",
       "                                             Sentence  NZGazAnt  NZGaz  \\\n",
       "0   (CrossMark, \\n, Collembola, of, Barrientos, Is...     False  False   \n",
       "2   (A, total, of, ten, species, were, recorded, ,...     False  False   \n",
       "3   (The, online, version, of, this, pelago, ,, An...     False  False   \n",
       "4   (BY, 4, of, een, le, \\n, 3077500, \\n, 3077000,...     False  False   \n",
       "5   ((, Sites, 13, -, 01, to, 13, -, 39, ), \\n, mo...     False  False   \n",
       "6   (Collembola, in, Antarctica, are, relatively, ...     False  False   \n",
       "7   (Particularly, ,, Collembola, from, maritime, ...     False  False   \n",
       "8   (However, ,, the, edaphic, fauna, of, Barrient...     False  False   \n",
       "9   (The, aim, of, this, study, is, to, identify, ...     False  False   \n",
       "10  (The, aim, of, this, study, is, to, identify, ...     False  False   \n",
       "11  (Eight, species, are, native, in, Antarctica, ...     False  False   \n",
       "12  (All, species, have, already, been, recorded, ...     False  False   \n",
       "13  (The, least, favorable, substrates, for, C., a...     False  False   \n",
       "14  (This, is, the, first, census, of, Collembola,...     False  False   \n",
       "15  ((, 2014, ), for, the, maritime, Antarctica, r...     False  False   \n",
       "16  (Accordingly, ,, recent, monitoring, activitie...     False  False   \n",
       "17  (Accordingly, ,, recent, monitoring, activitie...     False  False   \n",
       "18  (Hypogastrura, viatica, and, Mesaphorura, macr...     False  False   \n",
       "19  (Its, presence, could, affect, the, native, Co...     False  False   \n",
       "20  (Collembola, is, the, dominant, non, -, indige...     False  False   \n",
       "21  (The, first, non, -, indigenous, invertebrate,...     False  False   \n",
       "23  (These, findings, suggest, that, the, introduc...     False  False   \n",
       "24  (These, findings, suggest, that, the, introduc...     False  False   \n",
       "25  (The, species, M., macrochaeta, was, reported,...     False  False   \n",
       "27  (All, locations, colonized, by, non, -, indige...     False  False   \n",
       "28  (In, this, context, ,, there, is, a, real, pos...     False  False   \n",
       "30  (This, species, plays, an, essential, role, in...     False  False   \n",
       "31  (On, Barrientos, Island, ,, the, species, with...     False  False   \n",
       "32  (This, indicates, that, F., grisea, was, widel...     False  False   \n",
       "33  (The, reason, is, that, a, stratified, samplin...     False  False   \n",
       "34  (Archisotoma, brucei, is, usually, classified,...     False  False   \n",
       "35  (The, present, study, increases, the, knowledg...     False  False   \n",
       "36  (The, present, study, increases, the, knowledg...     False  False   \n",
       "37  (Although, all, recorded, species, had, alread...     False  False   \n",
       "38                            (Barrientos, Island, .)     False  False   \n",
       "39  (Accessed, 10, Feb, 2015, \\n, Baena, CW, ,, An...     False  False   \n",
       "41  (1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...     False  False   \n",
       "42  (Collembola, from, the, South, Orkney, Islands...     False  False   \n",
       "43  (The, terrestrial, microarthropod, fauna, of, ...     False  False   \n",
       "44  ((, Collembola, :, Hypogastruridae, ), at, Cap...     False  False   \n",
       "45  (Fur, seals, ,, whales, and, tourists, :, a, c...     False  False   \n",
       "46  (1995, ), Collembola, from, the, Scotia, Arc, ...     False  False   \n",
       "47  (Collembola, fauna, of, the, South, Shetland, ...     False  False   \n",
       "48  (Marine, littoral, and, coastal, Collembola, o...     False  False   \n",
       "49     (Global, Collembola, on, Deception, Island, .)     False  False   \n",
       "50  (Notes, on, arthropod, populations, in, the, A...     False  False   \n",
       "51                  (South, Orkney, Islands, area, .)     False  False   \n",
       "52  (Oxford, University, Press, ,, Oxford, \\n, Hug...     False  False   \n",
       "53  (Birds, spread, soil, microarthropods, to, Arc...     False  False   \n",
       "54  (Influence, of, forest, cover, and, herbaceous...     False  False   \n",
       "55  (The, influence, of, elephant, seals, on, the,...     False  False   \n",
       "56  (Antarctophorus, —, a, new, genus, of, Isotomi...     False  False   \n",
       "57  (Native, terrestrial, invertebrate, fauna, fro...     False  False   \n",
       "58  (A, new, species, and, redescriptions, of, Col...     False  False   \n",
       "59  (Determinants, of, terrestrial, arthropod, com...     False  False   \n",
       "60  ((, Collembola, ,, Neanuridae, ), from, King, ...     False  False   \n",
       "61  ((, Collembola, ,, Neanuridae, ), from, King, ...     False  False   \n",
       "62  (New, records, of, Collembola, and, Acarina, i...     False  False   \n",
       "63  (Wise, KAJ, (, 1970b, ), Collembola, of, Heard...     False  False   \n",
       "\n",
       "    ScarNZ  ScarGlobal  GeoNamesNZ  ...  PartialMatch_NZGazAnt  \\\n",
       "0    False        True       False  ...                    NaN   \n",
       "2    False        True       False  ...                    NaN   \n",
       "3    False        True       False  ...                    NaN   \n",
       "4    False       False       False  ...                    NaN   \n",
       "5    False        True       False  ...                    NaN   \n",
       "6    False        True       False  ...                    NaN   \n",
       "7    False        True       False  ...                    NaN   \n",
       "8    False        True       False  ...                    NaN   \n",
       "9    False        True       False  ...                    NaN   \n",
       "10   False        True       False  ...                    NaN   \n",
       "11   False        True       False  ...                    NaN   \n",
       "12   False        True       False  ...                    NaN   \n",
       "13   False        True       False  ...                    NaN   \n",
       "14   False        True       False  ...                    NaN   \n",
       "15   False        True       False  ...                    NaN   \n",
       "16   False        True       False  ...                    NaN   \n",
       "17   False        True       False  ...                    NaN   \n",
       "18   False        True       False  ...                    NaN   \n",
       "19   False       False       False  ...                    NaN   \n",
       "20   False        True       False  ...                    NaN   \n",
       "21   False        True       False  ...                    NaN   \n",
       "23   False        True       False  ...                    NaN   \n",
       "24   False        True       False  ...                    NaN   \n",
       "25   False       False       False  ...                    NaN   \n",
       "27   False        True       False  ...                    NaN   \n",
       "28   False        True       False  ...                    NaN   \n",
       "30   False        True       False  ...                    NaN   \n",
       "31   False        True       False  ...                    NaN   \n",
       "32   False        True       False  ...                    NaN   \n",
       "33   False        True       False  ...                    NaN   \n",
       "34   False       False       False  ...                    NaN   \n",
       "35   False        True       False  ...                    NaN   \n",
       "36   False        True       False  ...                    NaN   \n",
       "37   False        True       False  ...                    NaN   \n",
       "38   False        True       False  ...                    NaN   \n",
       "39   False       False       False  ...                    NaN   \n",
       "41   False        True       False  ...                    NaN   \n",
       "42   False        True       False  ...                    NaN   \n",
       "43   False       False       False  ...                    NaN   \n",
       "44   False        True       False  ...                    NaN   \n",
       "45   False        True       False  ...                    NaN   \n",
       "46   False        True       False  ...                    NaN   \n",
       "47   False        True       False  ...                    NaN   \n",
       "48   False       False       False  ...                    NaN   \n",
       "49   False        True       False  ...                    NaN   \n",
       "50   False        True       False  ...                    NaN   \n",
       "51   False        True       False  ...                    NaN   \n",
       "52   False        True       False  ...                    NaN   \n",
       "53   False       False       False  ...                    NaN   \n",
       "54   False       False       False  ...                    NaN   \n",
       "55   False       False       False  ...                    NaN   \n",
       "56   False        True       False  ...                    NaN   \n",
       "57   False        True       False  ...                    NaN   \n",
       "58   False        True       False  ...                    NaN   \n",
       "59   False        True       False  ...                    NaN   \n",
       "60   False        True       False  ...                    NaN   \n",
       "61   False        True       False  ...                    NaN   \n",
       "62   False        True       False  ...                    NaN   \n",
       "63   False       False       False  ...                    NaN   \n",
       "\n",
       "    PartialMatch_NZGaz  PartialMatch_ScarNZ PartialMatch_ScarGlobal  \\\n",
       "0                  NaN                  NaN                     NaN   \n",
       "2                  NaN                  NaN                     NaN   \n",
       "3                  NaN                  NaN                     NaN   \n",
       "4                  NaN                  NaN                     NaN   \n",
       "5                  NaN                  NaN                     NaN   \n",
       "6                  NaN                  NaN                     NaN   \n",
       "7                  NaN                  NaN                     NaN   \n",
       "8                  NaN                  NaN                     NaN   \n",
       "9                  NaN                  NaN                     NaN   \n",
       "10                 NaN                  NaN                     NaN   \n",
       "11                 NaN                  NaN                     NaN   \n",
       "12                 NaN                  NaN                     NaN   \n",
       "13                 NaN                  NaN                     NaN   \n",
       "14                 NaN                  NaN                     NaN   \n",
       "15                 NaN                  NaN                     NaN   \n",
       "16                 NaN                  NaN                     NaN   \n",
       "17                 NaN                  NaN                     NaN   \n",
       "18                 NaN                  NaN                     NaN   \n",
       "19                 NaN                  NaN                     NaN   \n",
       "20                 NaN                  NaN                     NaN   \n",
       "21                 NaN                  NaN                     NaN   \n",
       "23                 NaN                  NaN                     NaN   \n",
       "24                 NaN                  NaN                     NaN   \n",
       "25                 NaN                  NaN                     NaN   \n",
       "27                 NaN                  NaN                     NaN   \n",
       "28                 NaN                  NaN                     NaN   \n",
       "30                 NaN                  NaN                     NaN   \n",
       "31                 NaN                  NaN                     NaN   \n",
       "32                 NaN                  NaN                     NaN   \n",
       "33                 NaN                  NaN                     NaN   \n",
       "34                 NaN                  NaN                     NaN   \n",
       "35                 NaN                  NaN                     NaN   \n",
       "36                 NaN                  NaN                     NaN   \n",
       "37                 NaN                  NaN                     NaN   \n",
       "38                 NaN                  NaN                     NaN   \n",
       "39                 NaN                  NaN                     NaN   \n",
       "41                 NaN                  NaN                     NaN   \n",
       "42                 NaN                  NaN                     NaN   \n",
       "43                 NaN                  NaN                     NaN   \n",
       "44                 NaN                  NaN                     NaN   \n",
       "45                 NaN                  NaN                     NaN   \n",
       "46                 NaN                  NaN                     NaN   \n",
       "47                 NaN                  NaN                     NaN   \n",
       "48                 NaN                  NaN                     NaN   \n",
       "49                 NaN                  NaN                     NaN   \n",
       "50                 NaN                  NaN                     NaN   \n",
       "51                 NaN                  NaN                     NaN   \n",
       "52                 NaN                  NaN                     NaN   \n",
       "53                 NaN                  NaN                     NaN   \n",
       "54                 NaN                  NaN                     NaN   \n",
       "55                 NaN                  NaN                     NaN   \n",
       "56                 NaN                  NaN                     NaN   \n",
       "57                 NaN                  NaN                     NaN   \n",
       "58                 NaN                  NaN                     NaN   \n",
       "59                 NaN                  NaN                     NaN   \n",
       "60                 NaN                  NaN                     NaN   \n",
       "61                 NaN                  NaN                     NaN   \n",
       "62                 NaN                  NaN                     NaN   \n",
       "63                 NaN                  NaN                     NaN   \n",
       "\n",
       "   PartialMatch_GeoNamesNZ PartialMatch_GeoNamesAnt PartialMatch   inNZ  \\\n",
       "0                      NaN                      NaN          NaN  False   \n",
       "2                      NaN                      NaN          NaN  False   \n",
       "3                      NaN                      NaN          NaN  False   \n",
       "4                      NaN                      NaN        False  False   \n",
       "5                      NaN                      NaN          NaN  False   \n",
       "6                      NaN                      NaN          NaN  False   \n",
       "7                      NaN                      NaN          NaN  False   \n",
       "8                      NaN                      NaN          NaN  False   \n",
       "9                      NaN                      NaN          NaN  False   \n",
       "10                     NaN                      NaN          NaN  False   \n",
       "11                     NaN                      NaN          NaN  False   \n",
       "12                     NaN                      NaN          NaN  False   \n",
       "13                     NaN                      NaN          NaN  False   \n",
       "14                     NaN                      NaN          NaN  False   \n",
       "15                     NaN                      NaN          NaN  False   \n",
       "16                     NaN                      NaN          NaN  False   \n",
       "17                     NaN                      NaN          NaN  False   \n",
       "18                     NaN                      NaN          NaN  False   \n",
       "19                     NaN                      NaN        False  False   \n",
       "20                     NaN                      NaN          NaN  False   \n",
       "21                     NaN                      NaN          NaN  False   \n",
       "23                     NaN                      NaN          NaN  False   \n",
       "24                     NaN                      NaN          NaN  False   \n",
       "25                     NaN                      NaN        False  False   \n",
       "27                     NaN                      NaN          NaN  False   \n",
       "28                     NaN                      NaN          NaN  False   \n",
       "30                     NaN                      NaN          NaN  False   \n",
       "31                     NaN                      NaN          NaN  False   \n",
       "32                     NaN                      NaN          NaN  False   \n",
       "33                     NaN                      NaN          NaN  False   \n",
       "34                     NaN                      NaN        False   True   \n",
       "35                     NaN                      NaN          NaN  False   \n",
       "36                     NaN                      NaN          NaN  False   \n",
       "37                     NaN                      NaN          NaN  False   \n",
       "38                     NaN                      NaN          NaN  False   \n",
       "39                     NaN                      NaN        False  False   \n",
       "41                     NaN                      NaN          NaN  False   \n",
       "42                     NaN                      NaN          NaN  False   \n",
       "43                     NaN                      NaN        False  False   \n",
       "44                     NaN                      NaN          NaN  False   \n",
       "45                     NaN                      NaN          NaN  False   \n",
       "46                     NaN                      NaN          NaN  False   \n",
       "47                     NaN                      NaN          NaN  False   \n",
       "48                     NaN                      NaN        False  False   \n",
       "49                     NaN                      NaN          NaN  False   \n",
       "50                     NaN                      NaN          NaN  False   \n",
       "51                     NaN                      NaN          NaN  False   \n",
       "52                     NaN                      NaN          NaN  False   \n",
       "53                     NaN                      NaN        False  False   \n",
       "54                     NaN                      NaN        False  False   \n",
       "55                     NaN                      NaN        False   True   \n",
       "56                     NaN                      NaN          NaN  False   \n",
       "57                     NaN                      NaN          NaN  False   \n",
       "58                     NaN                      NaN          NaN  False   \n",
       "59                     NaN                      NaN          NaN  False   \n",
       "60                     NaN                      NaN          NaN  False   \n",
       "61                     NaN                      NaN          NaN  False   \n",
       "62                     NaN                      NaN          NaN  False   \n",
       "63                     NaN                      NaN        False   True   \n",
       "\n",
       "   inAntarctica Found  \n",
       "0          True  True  \n",
       "2          True  True  \n",
       "3          True  True  \n",
       "4         False  True  \n",
       "5          True  True  \n",
       "6          True  True  \n",
       "7          True  True  \n",
       "8          True  True  \n",
       "9          True  True  \n",
       "10         True  True  \n",
       "11         True  True  \n",
       "12         True  True  \n",
       "13         True  True  \n",
       "14         True  True  \n",
       "15         True  True  \n",
       "16         True  True  \n",
       "17         True  True  \n",
       "18         True  True  \n",
       "19        False  True  \n",
       "20         True  True  \n",
       "21         True  True  \n",
       "23         True  True  \n",
       "24         True  True  \n",
       "25        False  True  \n",
       "27         True  True  \n",
       "28         True  True  \n",
       "30         True  True  \n",
       "31         True  True  \n",
       "32         True  True  \n",
       "33         True  True  \n",
       "34         True  True  \n",
       "35         True  True  \n",
       "36         True  True  \n",
       "37         True  True  \n",
       "38         True  True  \n",
       "39        False  True  \n",
       "41         True  True  \n",
       "42         True  True  \n",
       "43        False  True  \n",
       "44         True  True  \n",
       "45         True  True  \n",
       "46         True  True  \n",
       "47         True  True  \n",
       "48        False  True  \n",
       "49         True  True  \n",
       "50         True  True  \n",
       "51         True  True  \n",
       "52         True  True  \n",
       "53        False  True  \n",
       "54        False  True  \n",
       "55         True  True  \n",
       "56         True  True  \n",
       "57         True  True  \n",
       "58         True  True  \n",
       "59         True  True  \n",
       "60         True  True  \n",
       "61         True  True  \n",
       "62         True  True  \n",
       "63         True  True  \n",
       "\n",
       "[59 rows x 30 columns]"
      ]
     },
     "execution_count": 446,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#locations confirmed by exact match in at least one gazetter\n",
    "locations[locations.exactMatch==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 447,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:25.675175Z",
     "start_time": "2021-01-31T01:42:25.670134Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All locations extracted by small Spacy model\n",
      "--------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['Antarctica', 'South Shetland', 'Pacific Ocean',\n",
       "       'Antarctic Peninsula', 'Barrientos Island', 'South Georgia',\n",
       "       'Deception Island', 'Antarctic Macquarie Island',\n",
       "       'South Shetland Islands', 'Macquarie Island',\n",
       "       'Southern Hemisphere', 'Antarctic Treaty', 'Heard Island',\n",
       "       'Mediterranean', 'Cuenca Mountain', 'Signy Island',\n",
       "       'South Orkney Islands', 'South Sandwich Islands', 'Arctic',\n",
       "       'Marion Island', 'Osmotic'], dtype=object)"
      ]
     },
     "execution_count": 447,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"All locations extracted by small Spacy model\")\n",
    "print(\"--------------------------------------------\")\n",
    "locations.Location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 448,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:27.519348Z",
     "start_time": "2021-01-31T01:42:27.511547Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not found in any of the referenced Antartica gazetteers\n",
      "---------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['South Shetland', 'Pacific Ocean', 'South Georgia',\n",
       "       'Antarctic Macquarie Island', 'Macquarie Island',\n",
       "       'Southern Hemisphere', 'Antarctic Treaty', 'Heard Island',\n",
       "       'Mediterranean', 'Cuenca Mountain', 'South Sandwich Islands',\n",
       "       'Arctic', 'Marion Island', 'Osmotic'], dtype=object)"
      ]
     },
     "execution_count": 448,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not found in any of the referenced Antartica gazetteers\")\n",
    "print(\"---------------------------------------------------------------------------\")\n",
    "locations[(locations.NZGazAnt==False) & (locations.ScarNZ==False) & (locations.ScarGlobal==False)].Location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:29.376589Z",
     "start_time": "2021-01-31T01:42:29.370955Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not found in any gazetteer as an exact, close or partial match\n",
      "----------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['South Shetland', 'Antarctic Macquarie Island',\n",
       "       'Southern Hemisphere', 'Antarctic Treaty', 'Cuenca Mountain',\n",
       "       'Osmotic'], dtype=object)"
      ]
     },
     "execution_count": 449,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not found in any gazetteer as an exact, close or partial match\")\n",
    "print(\"----------------------------------------------------------------------------------\")\n",
    "locations[locations.Found==False].Location.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using Spacy's Medium Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:37.277345Z",
     "start_time": "2021-01-31T01:42:36.826011Z"
    }
   },
   "outputs": [],
   "source": [
    "locations_med = find_locations(document_med)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:39.223406Z",
     "start_time": "2021-01-31T01:42:39.216074Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not found in any of the referenced Antartica gazetteers\n",
      "---------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['Adams', 'Birds'], dtype=object)"
      ]
     },
     "execution_count": 451,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not found in any of the referenced Antartica gazetteers\")\n",
    "print(\"---------------------------------------------------------------------------\")\n",
    "locations_med[(locations_med.NZGazAnt==False) & (locations_med.ScarNZ==False) & (locations_med.ScarGlobal==False)].Location.unique()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:41.061860Z",
     "start_time": "2021-01-31T01:42:41.042383Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Location</th>\n",
       "      <th>Original Tokenised Text</th>\n",
       "      <th>Start</th>\n",
       "      <th>End</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>NZGazAnt</th>\n",
       "      <th>NZGaz</th>\n",
       "      <th>ScarNZ</th>\n",
       "      <th>ScarGlobal</th>\n",
       "      <th>GeoNamesNZ</th>\n",
       "      <th>...</th>\n",
       "      <th>PartialMatch_NZGazAnt</th>\n",
       "      <th>PartialMatch_NZGaz</th>\n",
       "      <th>PartialMatch_ScarNZ</th>\n",
       "      <th>PartialMatch_ScarGlobal</th>\n",
       "      <th>PartialMatch_GeoNamesNZ</th>\n",
       "      <th>PartialMatch_GeoNamesAnt</th>\n",
       "      <th>PartialMatch</th>\n",
       "      <th>inNZ</th>\n",
       "      <th>inAntarctica</th>\n",
       "      <th>Found</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adams</td>\n",
       "      <td>Adams</td>\n",
       "      <td>6346</td>\n",
       "      <td>6347</td>\n",
       "      <td>(We, already, commented, that, Collembola, pre...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Birds</td>\n",
       "      <td>Birds</td>\n",
       "      <td>9758</td>\n",
       "      <td>9759</td>\n",
       "      <td>((, 2003, ), Birds, spread, soil, microarthrop...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Location Original Tokenised Text  Start   End  \\\n",
       "0    Adams                   Adams   6346  6347   \n",
       "1    Birds                   Birds   9758  9759   \n",
       "\n",
       "                                            Sentence  NZGazAnt  NZGaz  ScarNZ  \\\n",
       "0  (We, already, commented, that, Collembola, pre...     False  False   False   \n",
       "1  ((, 2003, ), Birds, spread, soil, microarthrop...     False  False   False   \n",
       "\n",
       "   ScarGlobal  GeoNamesNZ  ...  PartialMatch_NZGazAnt  PartialMatch_NZGaz  \\\n",
       "0       False       False  ...                    NaN                 NaN   \n",
       "1       False       False  ...                    NaN                 NaN   \n",
       "\n",
       "   PartialMatch_ScarNZ  PartialMatch_ScarGlobal  PartialMatch_GeoNamesNZ  \\\n",
       "0                  NaN                      NaN                      NaN   \n",
       "1                  NaN                      NaN                      NaN   \n",
       "\n",
       "   PartialMatch_GeoNamesAnt  PartialMatch   inNZ  inAntarctica  Found  \n",
       "0                       NaN         False  False         False   True  \n",
       "1                       NaN         False  False         False   True  \n",
       "\n",
       "[2 rows x 30 columns]"
      ]
     },
     "execution_count": 452,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#locations confirmed by presence in at least one gazetter\n",
    "locations_med[locations_med.exactMatch==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:43.740837Z",
     "start_time": "2021-01-31T01:42:43.735346Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All locations extracted by medium Spacy model\n",
      "---------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['Adams', 'Birds'], dtype=object)"
      ]
     },
     "execution_count": 453,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"All locations extracted by medium Spacy model\")\n",
    "print(\"---------------------------------------------\")\n",
    "locations_med.Location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:53.335721Z",
     "start_time": "2021-01-31T01:42:53.328544Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not found in any of the referenced gazetteers\n",
      "-----------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Adams', 'Birds']"
      ]
     },
     "execution_count": 454,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not found in any of the referenced gazetteers\")\n",
    "print(\"-----------------------------------------------------------------\")\n",
    "list(locations_med[(locations_med.NZGazAnt==False) & (locations_med.ScarNZ==False) & (locations_med.ScarGlobal==False)].Location.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:42:55.268699Z",
     "start_time": "2021-01-31T01:42:55.263293Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not found in any gazetteer as an exact, close or partial match\n",
      "----------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([], dtype=object)"
      ]
     },
     "execution_count": 455,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not found in any gazetteer as an exact, close or partial match\")\n",
    "print(\"----------------------------------------------------------------------------------\")\n",
    "locations_med[locations_med.Found==False].Location.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using Spacy's Large Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 456,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:43:29.239879Z",
     "start_time": "2021-01-31T01:43:14.254130Z"
    }
   },
   "outputs": [],
   "source": [
    "locations_lg = find_locations(document_lg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:43:31.313432Z",
     "start_time": "2021-01-31T01:43:31.290408Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Location</th>\n",
       "      <th>Original Tokenised Text</th>\n",
       "      <th>Start</th>\n",
       "      <th>End</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>NZGazAnt</th>\n",
       "      <th>NZGaz</th>\n",
       "      <th>ScarNZ</th>\n",
       "      <th>ScarGlobal</th>\n",
       "      <th>GeoNamesNZ</th>\n",
       "      <th>...</th>\n",
       "      <th>PartialMatch_NZGazAnt</th>\n",
       "      <th>PartialMatch_NZGaz</th>\n",
       "      <th>PartialMatch_ScarNZ</th>\n",
       "      <th>PartialMatch_ScarGlobal</th>\n",
       "      <th>PartialMatch_GeoNamesNZ</th>\n",
       "      <th>PartialMatch_GeoNamesAnt</th>\n",
       "      <th>PartialMatch</th>\n",
       "      <th>inNZ</th>\n",
       "      <th>inAntarctica</th>\n",
       "      <th>Found</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Antarctic</td>\n",
       "      <td>Antarctic</td>\n",
       "      <td>94</td>\n",
       "      <td>95</td>\n",
       "      <td>(Springer, -, Verlag, GmbH, Germany, ,, part, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>221</td>\n",
       "      <td>222</td>\n",
       "      <td>(A, total, of, ten, species, were, recorded, ,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>°24’S</td>\n",
       "      <td>°24’S</td>\n",
       "      <td>401</td>\n",
       "      <td>404</td>\n",
       "      <td>(set, -, Inter, -, annual, resampling, \\n, Int...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>432</td>\n",
       "      <td>433</td>\n",
       "      <td>(The, online, version, of, this, pelago, ,, An...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>724</td>\n",
       "      <td>725</td>\n",
       "      <td>(most, visited, terrestrial, sites, in, Antarc...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Location Original Tokenised Text  Start  End  \\\n",
       "0   Antarctic               Antarctic     94   95   \n",
       "1  Antarctica              Antarctica    221  222   \n",
       "2       °24’S                   °24’S    401  404   \n",
       "3  Antarctica              Antarctica    432  433   \n",
       "4  Antarctica              Antarctica    724  725   \n",
       "\n",
       "                                            Sentence  NZGazAnt  NZGaz  ScarNZ  \\\n",
       "0  (Springer, -, Verlag, GmbH, Germany, ,, part, ...     False  False   False   \n",
       "1  (A, total, of, ten, species, were, recorded, ,...     False  False   False   \n",
       "2  (set, -, Inter, -, annual, resampling, \\n, Int...     False  False   False   \n",
       "3  (The, online, version, of, this, pelago, ,, An...     False  False   False   \n",
       "4  (most, visited, terrestrial, sites, in, Antarc...     False  False   False   \n",
       "\n",
       "   ScarGlobal  GeoNamesNZ  ...  PartialMatch_NZGazAnt  PartialMatch_NZGaz  \\\n",
       "0       False       False  ...                    NaN                 NaN   \n",
       "1        True       False  ...                    NaN                 NaN   \n",
       "2       False       False  ...                    NaN                 NaN   \n",
       "3        True       False  ...                    NaN                 NaN   \n",
       "4        True       False  ...                    NaN                 NaN   \n",
       "\n",
       "   PartialMatch_ScarNZ PartialMatch_ScarGlobal PartialMatch_GeoNamesNZ  \\\n",
       "0                  NaN                     NaN                     NaN   \n",
       "1                  NaN                     NaN                     NaN   \n",
       "2                  NaN                     NaN                     NaN   \n",
       "3                  NaN                     NaN                     NaN   \n",
       "4                  NaN                     NaN                     NaN   \n",
       "\n",
       "  PartialMatch_GeoNamesAnt PartialMatch   inNZ inAntarctica  Found  \n",
       "0                      NaN        False  False        False  False  \n",
       "1                      NaN          NaN  False         True   True  \n",
       "2                      NaN        False  False        False  False  \n",
       "3                      NaN          NaN  False         True   True  \n",
       "4                      NaN          NaN  False         True   True  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 457,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "locations_lg.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 458,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:43:47.539867Z",
     "start_time": "2021-01-31T01:43:47.479368Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Location</th>\n",
       "      <th>Original Tokenised Text</th>\n",
       "      <th>Start</th>\n",
       "      <th>End</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>NZGazAnt</th>\n",
       "      <th>NZGaz</th>\n",
       "      <th>ScarNZ</th>\n",
       "      <th>ScarGlobal</th>\n",
       "      <th>GeoNamesNZ</th>\n",
       "      <th>...</th>\n",
       "      <th>PartialMatch_NZGazAnt</th>\n",
       "      <th>PartialMatch_NZGaz</th>\n",
       "      <th>PartialMatch_ScarNZ</th>\n",
       "      <th>PartialMatch_ScarGlobal</th>\n",
       "      <th>PartialMatch_GeoNamesNZ</th>\n",
       "      <th>PartialMatch_GeoNamesAnt</th>\n",
       "      <th>PartialMatch</th>\n",
       "      <th>inNZ</th>\n",
       "      <th>inAntarctica</th>\n",
       "      <th>Found</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>221</td>\n",
       "      <td>222</td>\n",
       "      <td>(A, total, of, ten, species, were, recorded, ,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>432</td>\n",
       "      <td>433</td>\n",
       "      <td>(The, online, version, of, this, pelago, ,, An...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>724</td>\n",
       "      <td>725</td>\n",
       "      <td>(most, visited, terrestrial, sites, in, Antarc...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>888</td>\n",
       "      <td>889</td>\n",
       "      <td>(Collembola, in, Antarctica, are, relatively, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>936</td>\n",
       "      <td>937</td>\n",
       "      <td>(Particularly, ,, Collembola, from, maritime, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>the Antarctic Peninsula</td>\n",
       "      <td>946</td>\n",
       "      <td>948</td>\n",
       "      <td>(Particularly, ,, Collembola, from, maritime, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>1364</td>\n",
       "      <td>1365</td>\n",
       "      <td>(The, aim, of, this, study, is, to, identify, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>2940</td>\n",
       "      <td>2941</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>3005</td>\n",
       "      <td>3006</td>\n",
       "      <td>(Two, species, are, non, -, indigenous, in, An...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>3025</td>\n",
       "      <td>3026</td>\n",
       "      <td>(All, species, have, already, been, recorded, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>5218</td>\n",
       "      <td>5219</td>\n",
       "      <td>((, 2014, ), for, the, maritime, Antarctica, r...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>5345</td>\n",
       "      <td>5346</td>\n",
       "      <td>(Accordingly, ,, recent, monitoring, activitie...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>5349</td>\n",
       "      <td>5351</td>\n",
       "      <td>(Accordingly, ,, recent, monitoring, activitie...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>5422</td>\n",
       "      <td>5423</td>\n",
       "      <td>(were, found, during, two, different, field, c...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>South Georgia</td>\n",
       "      <td>South Georgia</td>\n",
       "      <td>5548</td>\n",
       "      <td>5550</td>\n",
       "      <td>(Its, presence, could, affect, the, native, Co...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>the Antarctic Peninsula</td>\n",
       "      <td>5768</td>\n",
       "      <td>5770</td>\n",
       "      <td>(Collembola, is, the, dominant, non, -, indige...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>the South Shetland Islands</td>\n",
       "      <td>5852</td>\n",
       "      <td>5855</td>\n",
       "      <td>(The, first, non, -, indigenous, invertebrate,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>5939</td>\n",
       "      <td>5941</td>\n",
       "      <td>(These, findings, suggest, that, the, introduc...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>5942</td>\n",
       "      <td>5945</td>\n",
       "      <td>(These, findings, suggest, that, the, introduc...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Macquarie Island</td>\n",
       "      <td>Macquarie Island</td>\n",
       "      <td>5961</td>\n",
       "      <td>5963</td>\n",
       "      <td>(The, species, M., macrochaeta, was, reported,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>6011</td>\n",
       "      <td>6012</td>\n",
       "      <td>(All, locations, colonized, by, non, -, indige...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>6847</td>\n",
       "      <td>6848</td>\n",
       "      <td>(This, species, plays, an, essential, role, in...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Heard Island</td>\n",
       "      <td>Heard Island</td>\n",
       "      <td>7580</td>\n",
       "      <td>7582</td>\n",
       "      <td>(Archisotoma, brucei, is, usually, classified,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>the South Shetland Islands</td>\n",
       "      <td>7807</td>\n",
       "      <td>7810</td>\n",
       "      <td>(The, present, study, increases, the, knowledg...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>7820</td>\n",
       "      <td>7821</td>\n",
       "      <td>(Although, all, recorded, species, had, alread...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Mediterranean</td>\n",
       "      <td>Mediterranean</td>\n",
       "      <td>8367</td>\n",
       "      <td>8368</td>\n",
       "      <td>(Thinning, and, recovery, effects, on, soil, p...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>South Orkney Islands</td>\n",
       "      <td>the South Orkney Islands</td>\n",
       "      <td>8616</td>\n",
       "      <td>8619</td>\n",
       "      <td>(Collembola, from, the, South, Orkney, Islands...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>South Sandwich Islands</td>\n",
       "      <td>the South Sandwich Islands</td>\n",
       "      <td>8764</td>\n",
       "      <td>8767</td>\n",
       "      <td>(The, terrestrial, microarthropod, fauna, of, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>8802</td>\n",
       "      <td>8803</td>\n",
       "      <td>(Analysis, of, gut, contents, of, Gomphiocepha...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Deception Island</td>\n",
       "      <td>Deception Island</td>\n",
       "      <td>8914</td>\n",
       "      <td>8916</td>\n",
       "      <td>(Fur, seals, ,, whales, and, tourists, :, a, c...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>8917</td>\n",
       "      <td>8918</td>\n",
       "      <td>(Fur, seals, ,, whales, and, tourists, :, a, c...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>9067</td>\n",
       "      <td>9069</td>\n",
       "      <td>(Collembola, from, the, Scotia, Arc, and, Anta...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>South Shetland Islands</td>\n",
       "      <td>the South Shetland Islands</td>\n",
       "      <td>9117</td>\n",
       "      <td>9120</td>\n",
       "      <td>(Collembola, fauna, of, the, South, Shetland, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Macquarie Island</td>\n",
       "      <td>Macquarie Island</td>\n",
       "      <td>9180</td>\n",
       "      <td>9182</td>\n",
       "      <td>(Marine, littoral, and, coastal, Collembola, o...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>9290</td>\n",
       "      <td>9292</td>\n",
       "      <td>(Notes, on, arthropod, populations, in, the, A...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>South Orkney Islands</td>\n",
       "      <td>South Orkney Islands</td>\n",
       "      <td>9295</td>\n",
       "      <td>9298</td>\n",
       "      <td>(South, Orkney, Islands, area, .)</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>9521</td>\n",
       "      <td>9522</td>\n",
       "      <td>(Biological, invasions, in, terrestrial, Antar...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>Arctic</td>\n",
       "      <td>Arctic</td>\n",
       "      <td>9763</td>\n",
       "      <td>9764</td>\n",
       "      <td>(Birds, spread, soil, microarthropods, to, Arc...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>Mediterranean</td>\n",
       "      <td>Mediterranean</td>\n",
       "      <td>9839</td>\n",
       "      <td>9840</td>\n",
       "      <td>(Influence, of, forest, cover, and, herbaceous...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>Marion Island</td>\n",
       "      <td>Marion Island</td>\n",
       "      <td>9904</td>\n",
       "      <td>9906</td>\n",
       "      <td>(The, influence, of, elephant, seals, on, the,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>9987</td>\n",
       "      <td>9988</td>\n",
       "      <td>(Potapov, M, (, 1991, ), Antarctophorus, —, a,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>Antarctic Peninsula</td>\n",
       "      <td>10078</td>\n",
       "      <td>10080</td>\n",
       "      <td>(Native, terrestrial, invertebrate, fauna, fro...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>10152</td>\n",
       "      <td>10153</td>\n",
       "      <td>(A, new, species, and, redescriptions, of, Col...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>10229</td>\n",
       "      <td>10230</td>\n",
       "      <td>(Determinants, of, terrestrial, arthropod, com...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>King George Island</td>\n",
       "      <td>King George Island</td>\n",
       "      <td>10466</td>\n",
       "      <td>10469</td>\n",
       "      <td>((, Collembola, ,, Neanuridae, ), from, King, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>10474</td>\n",
       "      <td>10475</td>\n",
       "      <td>((, Collembola, ,, Neanuridae, ), from, King, ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>Antarctica</td>\n",
       "      <td>Antarctica</td>\n",
       "      <td>10518</td>\n",
       "      <td>10519</td>\n",
       "      <td>(New, records, of, Collembola, and, Acarina, i...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>47 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Location     Original Tokenised Text  Start    End  \\\n",
       "1               Antarctica                  Antarctica    221    222   \n",
       "3               Antarctica                  Antarctica    432    433   \n",
       "4               Antarctica                  Antarctica    724    725   \n",
       "5               Antarctica                  Antarctica    888    889   \n",
       "6               Antarctica                  Antarctica    936    937   \n",
       "7      Antarctic Peninsula     the Antarctic Peninsula    946    948   \n",
       "9               Antarctica                  Antarctica   1364   1365   \n",
       "13              Antarctica                  Antarctica   2940   2941   \n",
       "14              Antarctica                  Antarctica   3005   3006   \n",
       "15              Antarctica                  Antarctica   3025   3026   \n",
       "17              Antarctica                  Antarctica   5218   5219   \n",
       "18              Antarctica                  Antarctica   5345   5346   \n",
       "19     Antarctic Peninsula         Antarctic Peninsula   5349   5351   \n",
       "20              Antarctica                  Antarctica   5422   5423   \n",
       "22           South Georgia               South Georgia   5548   5550   \n",
       "23     Antarctic Peninsula     the Antarctic Peninsula   5768   5770   \n",
       "24  South Shetland Islands  the South Shetland Islands   5852   5855   \n",
       "26     Antarctic Peninsula         Antarctic Peninsula   5939   5941   \n",
       "27  South Shetland Islands      South Shetland Islands   5942   5945   \n",
       "28        Macquarie Island            Macquarie Island   5961   5963   \n",
       "30              Antarctica                  Antarctica   6011   6012   \n",
       "35              Antarctica                  Antarctica   6847   6848   \n",
       "36            Heard Island                Heard Island   7580   7582   \n",
       "37  South Shetland Islands  the South Shetland Islands   7807   7810   \n",
       "38              Antarctica                  Antarctica   7820   7821   \n",
       "41           Mediterranean               Mediterranean   8367   8368   \n",
       "43    South Orkney Islands    the South Orkney Islands   8616   8619   \n",
       "48  South Sandwich Islands  the South Sandwich Islands   8764   8767   \n",
       "49              Antarctica                  Antarctica   8802   8803   \n",
       "51        Deception Island            Deception Island   8914   8916   \n",
       "52              Antarctica                  Antarctica   8917   8918   \n",
       "55     Antarctic Peninsula         Antarctic Peninsula   9067   9069   \n",
       "56  South Shetland Islands  the South Shetland Islands   9117   9120   \n",
       "57        Macquarie Island            Macquarie Island   9180   9182   \n",
       "59     Antarctic Peninsula         Antarctic Peninsula   9290   9292   \n",
       "60    South Orkney Islands        South Orkney Islands   9295   9298   \n",
       "64              Antarctica                  Antarctica   9521   9522   \n",
       "65                  Arctic                      Arctic   9763   9764   \n",
       "66           Mediterranean               Mediterranean   9839   9840   \n",
       "67           Marion Island               Marion Island   9904   9906   \n",
       "69              Antarctica                  Antarctica   9987   9988   \n",
       "71     Antarctic Peninsula         Antarctic Peninsula  10078  10080   \n",
       "72              Antarctica                  Antarctica  10152  10153   \n",
       "73              Antarctica                  Antarctica  10229  10230   \n",
       "77      King George Island          King George Island  10466  10469   \n",
       "78              Antarctica                  Antarctica  10474  10475   \n",
       "79              Antarctica                  Antarctica  10518  10519   \n",
       "\n",
       "                                             Sentence  NZGazAnt  NZGaz  \\\n",
       "1   (A, total, of, ten, species, were, recorded, ,...     False  False   \n",
       "3   (The, online, version, of, this, pelago, ,, An...     False  False   \n",
       "4   (most, visited, terrestrial, sites, in, Antarc...     False  False   \n",
       "5   (Collembola, in, Antarctica, are, relatively, ...     False  False   \n",
       "6   (Particularly, ,, Collembola, from, maritime, ...     False  False   \n",
       "7   (Particularly, ,, Collembola, from, maritime, ...     False  False   \n",
       "9   (The, aim, of, this, study, is, to, identify, ...     False  False   \n",
       "13  (Eight, species, are, native, in, Antarctica, ...     False  False   \n",
       "14  (Two, species, are, non, -, indigenous, in, An...     False  False   \n",
       "15  (All, species, have, already, been, recorded, ...     False  False   \n",
       "17  ((, 2014, ), for, the, maritime, Antarctica, r...     False  False   \n",
       "18  (Accordingly, ,, recent, monitoring, activitie...     False  False   \n",
       "19  (Accordingly, ,, recent, monitoring, activitie...     False  False   \n",
       "20  (were, found, during, two, different, field, c...     False  False   \n",
       "22  (Its, presence, could, affect, the, native, Co...     False  False   \n",
       "23  (Collembola, is, the, dominant, non, -, indige...     False  False   \n",
       "24  (The, first, non, -, indigenous, invertebrate,...     False  False   \n",
       "26  (These, findings, suggest, that, the, introduc...     False  False   \n",
       "27  (These, findings, suggest, that, the, introduc...     False  False   \n",
       "28  (The, species, M., macrochaeta, was, reported,...     False  False   \n",
       "30  (All, locations, colonized, by, non, -, indige...     False  False   \n",
       "35  (This, species, plays, an, essential, role, in...     False  False   \n",
       "36  (Archisotoma, brucei, is, usually, classified,...     False  False   \n",
       "37  (The, present, study, increases, the, knowledg...     False  False   \n",
       "38  (Although, all, recorded, species, had, alread...     False  False   \n",
       "41  (Thinning, and, recovery, effects, on, soil, p...     False  False   \n",
       "43  (Collembola, from, the, South, Orkney, Islands...     False  False   \n",
       "48  (The, terrestrial, microarthropod, fauna, of, ...     False  False   \n",
       "49  (Analysis, of, gut, contents, of, Gomphiocepha...     False  False   \n",
       "51  (Fur, seals, ,, whales, and, tourists, :, a, c...     False  False   \n",
       "52  (Fur, seals, ,, whales, and, tourists, :, a, c...     False  False   \n",
       "55  (Collembola, from, the, Scotia, Arc, and, Anta...     False  False   \n",
       "56  (Collembola, fauna, of, the, South, Shetland, ...     False  False   \n",
       "57  (Marine, littoral, and, coastal, Collembola, o...     False  False   \n",
       "59  (Notes, on, arthropod, populations, in, the, A...     False  False   \n",
       "60                  (South, Orkney, Islands, area, .)     False  False   \n",
       "64  (Biological, invasions, in, terrestrial, Antar...     False  False   \n",
       "65  (Birds, spread, soil, microarthropods, to, Arc...     False  False   \n",
       "66  (Influence, of, forest, cover, and, herbaceous...     False  False   \n",
       "67  (The, influence, of, elephant, seals, on, the,...     False  False   \n",
       "69  (Potapov, M, (, 1991, ), Antarctophorus, —, a,...     False  False   \n",
       "71  (Native, terrestrial, invertebrate, fauna, fro...     False  False   \n",
       "72  (A, new, species, and, redescriptions, of, Col...     False  False   \n",
       "73  (Determinants, of, terrestrial, arthropod, com...     False  False   \n",
       "77  ((, Collembola, ,, Neanuridae, ), from, King, ...     False  False   \n",
       "78  ((, Collembola, ,, Neanuridae, ), from, King, ...     False  False   \n",
       "79  (New, records, of, Collembola, and, Acarina, i...     False  False   \n",
       "\n",
       "    ScarNZ  ScarGlobal  GeoNamesNZ  ...  PartialMatch_NZGazAnt  \\\n",
       "1    False        True       False  ...                    NaN   \n",
       "3    False        True       False  ...                    NaN   \n",
       "4    False        True       False  ...                    NaN   \n",
       "5    False        True       False  ...                    NaN   \n",
       "6    False        True       False  ...                    NaN   \n",
       "7    False        True       False  ...                    NaN   \n",
       "9    False        True       False  ...                    NaN   \n",
       "13   False        True       False  ...                    NaN   \n",
       "14   False        True       False  ...                    NaN   \n",
       "15   False        True       False  ...                    NaN   \n",
       "17   False        True       False  ...                    NaN   \n",
       "18   False        True       False  ...                    NaN   \n",
       "19   False        True       False  ...                    NaN   \n",
       "20   False        True       False  ...                    NaN   \n",
       "22   False       False       False  ...                    NaN   \n",
       "23   False        True       False  ...                    NaN   \n",
       "24   False        True       False  ...                    NaN   \n",
       "26   False        True       False  ...                    NaN   \n",
       "27   False        True       False  ...                    NaN   \n",
       "28   False       False       False  ...                    NaN   \n",
       "30   False        True       False  ...                    NaN   \n",
       "35   False        True       False  ...                    NaN   \n",
       "36   False       False       False  ...                    NaN   \n",
       "37   False        True       False  ...                    NaN   \n",
       "38   False        True       False  ...                    NaN   \n",
       "41   False       False       False  ...                    NaN   \n",
       "43   False        True       False  ...                    NaN   \n",
       "48   False       False       False  ...                    NaN   \n",
       "49   False        True       False  ...                    NaN   \n",
       "51   False        True       False  ...                    NaN   \n",
       "52   False        True       False  ...                    NaN   \n",
       "55   False        True       False  ...                    NaN   \n",
       "56   False        True       False  ...                    NaN   \n",
       "57   False       False       False  ...                    NaN   \n",
       "59   False        True       False  ...                    NaN   \n",
       "60   False        True       False  ...                    NaN   \n",
       "64   False        True       False  ...                    NaN   \n",
       "65   False       False       False  ...                    NaN   \n",
       "66   False       False       False  ...                    NaN   \n",
       "67   False       False       False  ...                    NaN   \n",
       "69   False        True       False  ...                    NaN   \n",
       "71   False        True       False  ...                    NaN   \n",
       "72   False        True       False  ...                    NaN   \n",
       "73   False        True       False  ...                    NaN   \n",
       "77   False        True       False  ...                    NaN   \n",
       "78   False        True       False  ...                    NaN   \n",
       "79   False        True       False  ...                    NaN   \n",
       "\n",
       "    PartialMatch_NZGaz  PartialMatch_ScarNZ PartialMatch_ScarGlobal  \\\n",
       "1                  NaN                  NaN                     NaN   \n",
       "3                  NaN                  NaN                     NaN   \n",
       "4                  NaN                  NaN                     NaN   \n",
       "5                  NaN                  NaN                     NaN   \n",
       "6                  NaN                  NaN                     NaN   \n",
       "7                  NaN                  NaN                     NaN   \n",
       "9                  NaN                  NaN                     NaN   \n",
       "13                 NaN                  NaN                     NaN   \n",
       "14                 NaN                  NaN                     NaN   \n",
       "15                 NaN                  NaN                     NaN   \n",
       "17                 NaN                  NaN                     NaN   \n",
       "18                 NaN                  NaN                     NaN   \n",
       "19                 NaN                  NaN                     NaN   \n",
       "20                 NaN                  NaN                     NaN   \n",
       "22                 NaN                  NaN                     NaN   \n",
       "23                 NaN                  NaN                     NaN   \n",
       "24                 NaN                  NaN                     NaN   \n",
       "26                 NaN                  NaN                     NaN   \n",
       "27                 NaN                  NaN                     NaN   \n",
       "28                 NaN                  NaN                     NaN   \n",
       "30                 NaN                  NaN                     NaN   \n",
       "35                 NaN                  NaN                     NaN   \n",
       "36                 NaN                  NaN                     NaN   \n",
       "37                 NaN                  NaN                     NaN   \n",
       "38                 NaN                  NaN                     NaN   \n",
       "41                 NaN                  NaN                     NaN   \n",
       "43                 NaN                  NaN                     NaN   \n",
       "48                 NaN                  NaN                     NaN   \n",
       "49                 NaN                  NaN                     NaN   \n",
       "51                 NaN                  NaN                     NaN   \n",
       "52                 NaN                  NaN                     NaN   \n",
       "55                 NaN                  NaN                     NaN   \n",
       "56                 NaN                  NaN                     NaN   \n",
       "57                 NaN                  NaN                     NaN   \n",
       "59                 NaN                  NaN                     NaN   \n",
       "60                 NaN                  NaN                     NaN   \n",
       "64                 NaN                  NaN                     NaN   \n",
       "65                 NaN                  NaN                     NaN   \n",
       "66                 NaN                  NaN                     NaN   \n",
       "67                 NaN                  NaN                     NaN   \n",
       "69                 NaN                  NaN                     NaN   \n",
       "71                 NaN                  NaN                     NaN   \n",
       "72                 NaN                  NaN                     NaN   \n",
       "73                 NaN                  NaN                     NaN   \n",
       "77                 NaN                  NaN                     NaN   \n",
       "78                 NaN                  NaN                     NaN   \n",
       "79                 NaN                  NaN                     NaN   \n",
       "\n",
       "   PartialMatch_GeoNamesNZ PartialMatch_GeoNamesAnt PartialMatch   inNZ  \\\n",
       "1                      NaN                      NaN          NaN  False   \n",
       "3                      NaN                      NaN          NaN  False   \n",
       "4                      NaN                      NaN          NaN  False   \n",
       "5                      NaN                      NaN          NaN  False   \n",
       "6                      NaN                      NaN          NaN  False   \n",
       "7                      NaN                      NaN          NaN  False   \n",
       "9                      NaN                      NaN          NaN  False   \n",
       "13                     NaN                      NaN          NaN  False   \n",
       "14                     NaN                      NaN          NaN  False   \n",
       "15                     NaN                      NaN          NaN  False   \n",
       "17                     NaN                      NaN          NaN  False   \n",
       "18                     NaN                      NaN          NaN  False   \n",
       "19                     NaN                      NaN          NaN  False   \n",
       "20                     NaN                      NaN          NaN  False   \n",
       "22                     NaN                      NaN        False  False   \n",
       "23                     NaN                      NaN          NaN  False   \n",
       "24                     NaN                      NaN          NaN  False   \n",
       "26                     NaN                      NaN          NaN  False   \n",
       "27                     NaN                      NaN          NaN  False   \n",
       "28                     NaN                      NaN        False  False   \n",
       "30                     NaN                      NaN          NaN  False   \n",
       "35                     NaN                      NaN          NaN  False   \n",
       "36                     NaN                      NaN        False   True   \n",
       "37                     NaN                      NaN          NaN  False   \n",
       "38                     NaN                      NaN          NaN  False   \n",
       "41                     NaN                      NaN        False  False   \n",
       "43                     NaN                      NaN          NaN  False   \n",
       "48                     NaN                      NaN        False  False   \n",
       "49                     NaN                      NaN          NaN  False   \n",
       "51                     NaN                      NaN          NaN  False   \n",
       "52                     NaN                      NaN          NaN  False   \n",
       "55                     NaN                      NaN          NaN  False   \n",
       "56                     NaN                      NaN          NaN  False   \n",
       "57                     NaN                      NaN        False  False   \n",
       "59                     NaN                      NaN          NaN  False   \n",
       "60                     NaN                      NaN          NaN  False   \n",
       "64                     NaN                      NaN          NaN  False   \n",
       "65                     NaN                      NaN        False  False   \n",
       "66                     NaN                      NaN        False  False   \n",
       "67                     NaN                      NaN        False   True   \n",
       "69                     NaN                      NaN          NaN  False   \n",
       "71                     NaN                      NaN          NaN  False   \n",
       "72                     NaN                      NaN          NaN  False   \n",
       "73                     NaN                      NaN          NaN  False   \n",
       "77                     NaN                      NaN          NaN  False   \n",
       "78                     NaN                      NaN          NaN  False   \n",
       "79                     NaN                      NaN          NaN  False   \n",
       "\n",
       "   inAntarctica Found  \n",
       "1          True  True  \n",
       "3          True  True  \n",
       "4          True  True  \n",
       "5          True  True  \n",
       "6          True  True  \n",
       "7          True  True  \n",
       "9          True  True  \n",
       "13         True  True  \n",
       "14         True  True  \n",
       "15         True  True  \n",
       "17         True  True  \n",
       "18         True  True  \n",
       "19         True  True  \n",
       "20         True  True  \n",
       "22        False  True  \n",
       "23         True  True  \n",
       "24         True  True  \n",
       "26         True  True  \n",
       "27         True  True  \n",
       "28        False  True  \n",
       "30         True  True  \n",
       "35         True  True  \n",
       "36         True  True  \n",
       "37         True  True  \n",
       "38         True  True  \n",
       "41        False  True  \n",
       "43         True  True  \n",
       "48        False  True  \n",
       "49         True  True  \n",
       "51         True  True  \n",
       "52         True  True  \n",
       "55         True  True  \n",
       "56         True  True  \n",
       "57        False  True  \n",
       "59         True  True  \n",
       "60         True  True  \n",
       "64         True  True  \n",
       "65        False  True  \n",
       "66        False  True  \n",
       "67         True  True  \n",
       "69         True  True  \n",
       "71         True  True  \n",
       "72         True  True  \n",
       "73         True  True  \n",
       "77         True  True  \n",
       "78         True  True  \n",
       "79         True  True  \n",
       "\n",
       "[47 rows x 30 columns]"
      ]
     },
     "execution_count": 458,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#locations confirmed by presence in at least one gazetter\n",
    "locations_lg[locations_lg.exactMatch==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 459,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:43:53.838904Z",
     "start_time": "2021-01-31T01:43:53.832934Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All locations extracted by large Spacy model\n",
      "--------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['Antarctic', 'Antarctica', '°24’S', 'Antarctic Peninsula',\n",
       "       'Navi Horiba', 'Fig', 'South Georgia', 'South Shetland Islands',\n",
       "       'Macquarie Island', 'Southern Hemisphere', 'Heard Island',\n",
       "       'Mediterranean', 'South Orkney Islands', 'Antarctic South',\n",
       "       'South Sandwich Islands', 'Deception Island', 'Pinus',\n",
       "       'Solid Earth', 'Arctic', 'Marion Island', 'Potapov M',\n",
       "       'King George Island'], dtype=object)"
      ]
     },
     "execution_count": 459,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"All locations extracted by large Spacy model\")\n",
    "print(\"--------------------------------------------\")\n",
    "locations_lg.Location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 460,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:43:56.020518Z",
     "start_time": "2021-01-31T01:43:56.013486Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not found in any of the referenced Antartica gazetteers\n",
      "---------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Antarctic',\n",
       " '°24’S',\n",
       " 'Navi Horiba',\n",
       " 'Fig',\n",
       " 'South Georgia',\n",
       " 'Macquarie Island',\n",
       " 'Southern Hemisphere',\n",
       " 'Heard Island',\n",
       " 'Mediterranean',\n",
       " 'Antarctic South',\n",
       " 'South Sandwich Islands',\n",
       " 'Pinus',\n",
       " 'Solid Earth',\n",
       " 'Arctic',\n",
       " 'Marion Island',\n",
       " 'Potapov M']"
      ]
     },
     "execution_count": 460,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not found in any of the referenced Antartica gazetteers\")\n",
    "print(\"---------------------------------------------------------------------------\")\n",
    "list(locations_lg[(locations_lg.NZGazAnt==False) & (locations_lg.ScarNZ==False) & (locations_lg.ScarGlobal==False)].Location.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 461,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:43:57.967158Z",
     "start_time": "2021-01-31T01:43:57.961141Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not found in any gazetteer as an exact, close or partial match\n",
      "----------------------------------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['Antarctic', '°24’S', 'Navi Horiba', 'Fig', 'Southern Hemisphere',\n",
       "       'Antarctic South', 'Pinus', 'Solid Earth', 'Potapov M'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 461,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not found in any gazetteer as an exact, close or partial match\")\n",
    "print(\"----------------------------------------------------------------------------------\")\n",
    "locations_lg[locations_lg.Found==False].Location.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extracting and Saving Locations for each Document in a Folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T00:59:18.206211Z",
     "start_time": "2021-02-11T00:59:18.199873Z"
    }
   },
   "outputs": [],
   "source": [
    "def extractLocationsfromFolderofDocuments(folder, NLPmodel='large'):\n",
    "    filelist = [x for x in os.listdir(folder) if x.endswith('txt')]\n",
    "    for filename in filelist:\n",
    "        file = open(folder+'/'+ filename)\n",
    "        doc = file.read()\n",
    "        file.close()\n",
    "        if NLPmodel == 'small':\n",
    "            document = nlp(doc)\n",
    "        elif NLPmodel == 'medium':\n",
    "            document = nlp_md(doc)\n",
    "        elif NLPmodel == 'large':\n",
    "            document = nlp_lg(doc)\n",
    "            \n",
    "        df = find_locations(document)\n",
    "        newfilename = re.sub('.txt', '-locations.csv', filename)\n",
    "        df.to_csv(folder+'/'+ newfilename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T01:29:24.464252Z",
     "start_time": "2021-02-11T01:28:52.563394Z"
    }
   },
   "outputs": [],
   "source": [
    "extractLocationsfromFolderofDocuments('SmallPDFs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T01:36:08.512496Z",
     "start_time": "2021-02-11T01:29:25.931420Z"
    }
   },
   "outputs": [],
   "source": [
    "extractLocationsfromFolderofDocuments('PDFs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T01:37:40.823060Z",
     "start_time": "2021-02-11T01:36:09.815026Z"
    }
   },
   "outputs": [],
   "source": [
    "extractLocationsfromFolderofDocuments('FraserSet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T08:23:03.258408Z",
     "start_time": "2021-02-26T08:21:40.249575Z"
    }
   },
   "outputs": [],
   "source": [
    "extractLocationsfromFolderofDocuments('FraserSet/BodyOnly/Clean')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding Locations in Collection of Documents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running through the folder of converted text documents we have so far to see what sort of extracted locations *are not* confirmed by either of the two gazetteers to see what refinement needs to be added. More gazetteers, and some refienment where a name begins with 'the' are most likely next steps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-30T04:46:51.345739Z",
     "start_time": "2021-01-30T04:46:51.331910Z"
    }
   },
   "outputs": [],
   "source": [
    "def findLocationsInAllDocs(folder, NLPmodel='large'):\n",
    "    \n",
    "        \n",
    "    filelist = [x for x in os.listdir(folder) if x.endswith('txt')]\n",
    "    df = pd.DataFrame()\n",
    "    for filename in filelist:\n",
    "        file = open(folder+'/'+ filename)\n",
    "        doc = file.read()\n",
    "        file.close()\n",
    "        if NLPmodel == 'small':\n",
    "            document = nlp(doc)\n",
    "        elif NLPmodel == 'medium':\n",
    "            document = nlp_md(doc)\n",
    "        elif NLPmodel == 'large':\n",
    "            document = nlp_lg(doc)\n",
    "    \n",
    "        df2 = find_locations(document)\n",
    "        df2['Source'] = filename\n",
    "        df = pd.concat([df,df2])\n",
    "        df = df.reset_index(drop=True)\n",
    "        \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T01:49:40.089402Z",
     "start_time": "2021-01-31T01:45:57.378329Z"
    }
   },
   "outputs": [],
   "source": [
    "locations_in_inital_set = findLocationsInAllDocs(\"PDFs\", \"small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 473,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T02:53:25.145529Z",
     "start_time": "2021-01-31T02:52:35.090951Z"
    }
   },
   "outputs": [],
   "source": [
    "locations_in_inital_set_med = findLocationsInAllDocs(\"PDFs\", \"medium\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 478,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T03:03:39.998928Z",
     "start_time": "2021-01-31T02:56:14.498567Z"
    }
   },
   "outputs": [],
   "source": [
    "locations_in_inital_set_lg = findLocationsInAllDocs(\"PDFs\", 'large')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 480,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T03:07:21.049484Z",
     "start_time": "2021-01-31T03:07:20.983862Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using Small Spacy Model\n",
      "-----------------------\n",
      "1527 locations (239 unique) extracted from 9 documents\n",
      "1402locations with exact, close or partial match in gazeteers (170 unique)\n",
      "1089 confirmed by exact match in any gazetteer (118 unique)\n",
      "1270 locations with exact, close or partial match in an Antarctic gazeteer (118 unique)\n",
      "960 confirmed by an exact match in Antarctic gazetteer (68 unique)\n",
      "\n",
      "Using Medium Spacy Model\n",
      "------------------------\n",
      "71 locations (41 unique) extracted from 9 documents\n",
      "49 with exact, close or partial match in gazeteers (23 unique)\n",
      "47 confirmed by exact match in any gazetteer (22 unique)\n",
      "17 locations with exact, close or partial match in an Antarctic gazeteer (6 unique)\n",
      "15 confirmed by an exact match in Antarctic gazetteer (5 unique)\n",
      "\n",
      "Using Large Spacy Model\n",
      "-----------------------\n",
      "2336 locations (378 unique) extracted from 9 documents\n",
      "1744 with exact, close or partial match in gazeteers (222 unique)\n",
      "1323 confirmed by exact match in any gazetteer (152 unique)\n",
      "1521 locations with exact, close or partial match in an Antarctic gazeteer (148 unique)\n",
      "1113 confirmed by an exact match in Antarctic gazetteer (89 unique)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "length =  len([x for x in os.listdir('PDFs') if x.endswith('txt')])\n",
    "\n",
    "confirmed_total_small = len(locations_in_inital_set[locations_in_inital_set.exactMatch==True])\n",
    "confirmed_unique_small = locations_in_inital_set[locations_in_inital_set.exactMatch==True].Location.nunique()\n",
    "confirmed_ant_total_small = len(locations_in_inital_set[(locations_in_inital_set.exactMatch==True) & \n",
    "                                                        (locations_in_inital_set.inAntarctica==True)])\n",
    "confirmed_ant_unique_small = locations_in_inital_set[(locations_in_inital_set.exactMatch==True) & \n",
    "                                                        (locations_in_inital_set.inAntarctica==True)].Location.nunique()\n",
    "possible_total_small = len(locations_in_inital_set[locations_in_inital_set.Found==True])\n",
    "possible_unique_small = locations_in_inital_set[locations_in_inital_set.Found==True].Location.nunique()\n",
    "possible_ant_total_small = len(locations_in_inital_set[(locations_in_inital_set.Found==True) & \n",
    "                                                        (locations_in_inital_set.inAntarctica==True)])\n",
    "possible_ant_unique_small = locations_in_inital_set[(locations_in_inital_set.Found==True) & \n",
    "                                                    (locations_in_inital_set.inAntarctica==True)].Location.nunique()\n",
    "\n",
    "confirmed_total_med = len(locations_in_inital_set_med[locations_in_inital_set_med.exactMatch==True])\n",
    "confirmed_unique_med = locations_in_inital_set_med[locations_in_inital_set_med.exactMatch==True].Location.nunique()\n",
    "confirmed_ant_total_med = len(locations_in_inital_set_med[(locations_in_inital_set_med.exactMatch==True) & \n",
    "                                                        (locations_in_inital_set_med.inAntarctica==True)])\n",
    "confirmed_ant_unique_med = locations_in_inital_set_med[(locations_in_inital_set_med.exactMatch==True) & \n",
    "                                                        (locations_in_inital_set_med.inAntarctica==True)].Location.nunique()\n",
    "possible_total_med = len(locations_in_inital_set_med[locations_in_inital_set_med.Found==True])\n",
    "possible_unique_med = locations_in_inital_set_med[locations_in_inital_set_med.Found==True].Location.nunique()\n",
    "possible_ant_total_med = len(locations_in_inital_set_med[(locations_in_inital_set_med.Found==True) & \n",
    "                                                        (locations_in_inital_set_med.inAntarctica==True)])\n",
    "possible_ant_unique_med = locations_in_inital_set_med[(locations_in_inital_set_med.Found==True) & \n",
    "                                                      (locations_in_inital_set_med.inAntarctica==True)].Location.nunique()\n",
    "    \n",
    "confirmed_total_lg = len(locations_in_inital_set_lg[locations_in_inital_set_lg.exactMatch==True])\n",
    "confirmed_unique_lg = locations_in_inital_set_lg[locations_in_inital_set_lg.exactMatch==True].Location.nunique()\n",
    "confirmed_ant_total_lg = len(locations_in_inital_set_lg[(locations_in_inital_set_lg.exactMatch==True) & \n",
    "                                                        (locations_in_inital_set_lg.inAntarctica==True)])\n",
    "confirmed_ant_unique_lg = locations_in_inital_set_lg[(locations_in_inital_set_lg.exactMatch==True) & \n",
    "                                                        (locations_in_inital_set_lg.inAntarctica==True)].Location.nunique()\n",
    "possible_total_lg = len(locations_in_inital_set_lg[locations_in_inital_set_lg.Found==True])\n",
    "possible_unique_lg = locations_in_inital_set_lg[locations_in_inital_set_lg.Found==True].Location.nunique()\n",
    "possible_ant_total_lg = len(locations_in_inital_set_lg[(locations_in_inital_set_lg.Found==True) & \n",
    "                                                        (locations_in_inital_set_lg.inAntarctica==True)])\n",
    "possible_ant_unique_lg = locations_in_inital_set_lg[(locations_in_inital_set_lg.Found==True) & \n",
    "                                                      (locations_in_inital_set_lg.inAntarctica==True)].Location.nunique() \n",
    "\n",
    "print ('Using Small Spacy Model') \n",
    "print ('-----------------------')\n",
    "print (len(locations_in_inital_set), \" locations (\", locations_in_inital_set.Location.nunique(), \n",
    "       \" unique) extracted from \", length, \" documents\", sep ='')\n",
    "print (possible_total_small, 'locations with exact, close or partial match in gazeteers (', possible_unique_small,\n",
    "       ' unique)', sep='')\n",
    "print (confirmed_total_small, ' confirmed by exact match in any gazetteer (', confirmed_unique_small, \n",
    "       ' unique)', sep='')\n",
    "print (possible_ant_total_small, ' locations with exact, close or partial match in an Antarctic gazeteer (', \n",
    "       possible_ant_unique_small,\n",
    "       ' unique)', sep='')\n",
    "print (confirmed_ant_total_small, ' confirmed by an exact match in Antarctic gazetteer (', \n",
    "       confirmed_ant_unique_small, ' unique)\\n', sep='')\n",
    "\n",
    "print ('Using Medium Spacy Model') \n",
    "print ('------------------------')\n",
    "print (len(locations_in_inital_set_med), \" locations (\", locations_in_inital_set_med.Location.nunique(), \n",
    "       \" unique) extracted from \", length, \" documents\", sep ='')\n",
    "print (possible_total_med, ' with exact, close or partial match in gazeteers (', possible_unique_med,\n",
    "       ' unique)', sep='')\n",
    "print (confirmed_total_med, ' confirmed by exact match in any gazetteer (', confirmed_unique_med, \n",
    "       ' unique)', sep='')\n",
    "print (possible_ant_total_med, ' locations with exact, close or partial match in an Antarctic gazeteer (', \n",
    "       possible_ant_unique_med,\n",
    "       ' unique)', sep='')\n",
    "print (confirmed_ant_total_med, ' confirmed by an exact match in Antarctic gazetteer (', \n",
    "       confirmed_ant_unique_med, ' unique)\\n', sep='')\n",
    "\n",
    "print ('Using Large Spacy Model') \n",
    "print ('-----------------------')\n",
    "print (len(locations_in_inital_set_lg), \" locations (\", locations_in_inital_set_lg.Location.nunique(), \n",
    "       \" unique) extracted from \", length, \" documents\", sep ='')\n",
    "print (possible_total_lg, ' with exact, close or partial match in gazeteers (', possible_unique_lg,\n",
    "       ' unique)', sep='')\n",
    "print (confirmed_total_lg, ' confirmed by exact match in any gazetteer (', confirmed_unique_lg, \n",
    "       ' unique)', sep='')\n",
    "print (possible_ant_total_lg, ' locations with exact, close or partial match in an Antarctic gazeteer (', \n",
    "       possible_ant_unique_lg,\n",
    "       ' unique)', sep='')\n",
    "print (confirmed_ant_total_lg, ' confirmed by an exact match in Antarctic gazetteer (', \n",
    "       confirmed_ant_unique_lg, ' unique)\\n', sep='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T03:07:58.587220Z",
     "start_time": "2021-01-31T03:07:58.576324Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All locations extracted by large Spacy model\n",
      "--------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['(Fanciulli et al.', '(South America', '015° Granite Harbour',\n",
       "       '4a-c', 'AAZ6358', 'ANTARCTIC COASTAL LANDSCAPES', 'ANTARCTICA',\n",
       "       'Africa', 'Alexander Island', 'Alpine', 'Antarctic',\n",
       "       'Antarctic B.', 'Antarctic Bryum', 'Antarctic Ceratodon',\n",
       "       'Antarctic Circle', 'Antarctic Coastal Landscapes',\n",
       "       'Antarctic Collembola', 'Antarctic Continent',\n",
       "       'Antarctic Continent\\n', 'Antarctic Desert Ecosystems',\n",
       "       'Antarctic Dry Valley', 'Antarctic Ecology',\n",
       "       'Antarctic Hennidiella', 'Antarctic Ice-Free Coastal Landscapes',\n",
       "       'Antarctic Ice-Free Coastal Landscapes, Ecological Studies Analysis and Synthesis',\n",
       "       'Antarctic Ice-Free Landscapes', 'Antarctic Lake',\n",
       "       'Antarctic Map Folio Series - Folio 5', 'Antarctic Marion Island',\n",
       "       'Antarctic Oasis', 'Antarctic Peninsula', 'Antarctic Penisula',\n",
       "       'Antarctic Soil', 'Antarctic South', 'Antarctic Terrestrial',\n",
       "       'Antarctic region', 'Antarctica', 'Antarctica Cape Bird',\n",
       "       'Antarctica Granite Harbour', 'Antarctica Miers Valley',\n",
       "       'Antarctica Peninsula', 'Antarctica Taylor Valley', 'Arctic',\n",
       "       'Asia', 'Australasian Antarctic Expedition 1912',\n",
       "       'Avicennia marina', 'Barwick Valley', 'Bay of Fundy',\n",
       "       'Bay of Plenty', 'Beaufort', 'Beaufort Island', 'Beet et al.',\n",
       "       'Black IV', 'Bluff', 'Botany Bay', 'Botany Bay Physcia',\n",
       "       'Botany bay', 'Bratina Island', 'Brighton River', 'Brisbane River',\n",
       "       'Bunger Hills', 'Bunger Oasis', 'Byers Peninsula',\n",
       "       'CONTINENT GRANITE HARBOUR Cape Geology', 'Campbell Island',\n",
       "       'Canada Glacier', 'Canary Islands', 'Cape 7', 'Cape Bird',\n",
       "       'Cape Geology', 'Cape Royds', 'Caribbean', 'Catlins Lake',\n",
       "       'Cenozoic Islands', 'Chathams Islands', 'Chignecto Bay',\n",
       "       'Clare Range', 'Cluster VI', 'Commonwealth Glacier',\n",
       "       'Continental Antarctica', 'Contour', 'Cook Strait',\n",
       "       'Cook Strait (Fig', 'Corbicula', 'Corophium', 'Crater Cirque',\n",
       "       'Cumberland Basin', 'D. East Antarctica', 'Deception Island',\n",
       "       'Deep Lake Tarn', 'Discovery Bluff', 'Dnieper River',\n",
       "       'Driest area', 'Dry Valley', 'Dry Valley Streams',\n",
       "       'Dry Valley Terrestrial Plant Communities', 'Dry Valleys',\n",
       "       'Dry Valleys’', 'Dry | Valleys', 'Drygalski Ice Tongue', 'Earth',\n",
       "       'East', 'East Antarctic', 'East Antarctic ice', 'East Antarctica',\n",
       "       'East Cape', 'East Ongul Island', 'Eastern Antarctic Peninsula',\n",
       "       'Eastern Antarctica', 'Ecosystem',\n",
       "       'Enderby Land eastern Antarctica', 'Europe', 'Fig', 'Fold Island',\n",
       "       'Franklin Island', 'GHOD', 'Glacier Mount', 'Glaciers',\n",
       "       'Glaciology Dry Valleys', 'Gneiss Point', 'Gomphiocephalus',\n",
       "       'Granite Harbor', 'Granite Harbour',\n",
       "       'Granite Harbour\\nGranite Harbour', 'Granite Harbour B.',\n",
       "       'Great Barrier Island', 'Great Lakes',\n",
       "       'Great Lakes - St. Lawrence River', 'Harbour',\n",
       "       'Harbour | Ross Sea', 'Heard Island', 'Hervey Bay',\n",
       "       'High Latitude Limnology', 'Hut Point Peninsula',\n",
       "       'I. Pacific Insects', 'IV', 'Inner', 'Island Cape', 'Island INI',\n",
       "       'Isthmus', 'J. Great Lakes Res', 'Janetschek', 'Jn Antarctic',\n",
       "       'Jurassic', 'Kandpal et al.', 'Kar Plateau Antarctica',\n",
       "       'Kar plateau', 'Kay Island', 'Keller Peninsula', 'Kilometers',\n",
       "       'King George Island', 'L.', 'LTER', 'La 97', 'Lake',\n",
       "       'Lake Brownworth', 'Lake Chad', 'Lake Fryxell',\n",
       "       'Lake Fryxell Basin', 'Lake Hoare', 'Lake North | Cape',\n",
       "       'Lake Penny', 'Lake Rotorua', 'Lakes', 'Lakes Rotorua',\n",
       "       'Latitude /Longitude', 'Laurentian Great Lakes', 'Lower Rhine',\n",
       "       'Lower Taylor Valley', 'Mackay Glacier', 'Mackay Glacier\\n',\n",
       "       'Mackay Glacier region', 'Macquarie Island', 'Macquarie Islands',\n",
       "       'Manukau Harbour', 'Marble Point', 'Marion Island',\n",
       "       'Marr Glacier east', 'Mawson Glacier', 'Mawson Rock', 'McMurdo',\n",
       "       'McMurdo Dry Valley', 'McMurdo Dry Valleys', 'McMurdo Sound',\n",
       "       'McMurdo Sound Region', 'McMurdo Upland (Dry Valleys',\n",
       "       'Mcmurdo Dry Valleys', 'Mediterranean', 'Mediterranean Sea',\n",
       "       'Megaleptoperla', 'Mi Mt', 'Miers Valley',\n",
       "       'Modelling West Antarctic', 'Molodyezhnaya', 'Moreton Bay',\n",
       "       'Mossel Lake', 'Mount', 'Mount Cerberus', 'Mount England',\n",
       "       'Mount Erebus', 'Mount George Murray', 'Mount Gran',\n",
       "       'Mount Melbourne', 'Mount Murray', 'Mount Seuss', 'Muelleria',\n",
       "       'NEW ZEALAND ESTUARY', 'Nanorchestidae', 'Navi Horiba',\n",
       "       'New Glacier', 'New Zealand', 'New Zealand Adrift',\n",
       "       'North America', 'North American Great Lakes', 'North Base',\n",
       "       'North Island', 'North Island North Island', 'North Island P.',\n",
       "       'North Islands', 'North Victoria Land', 'North, South',\n",
       "       'Northern Dry Valleys', 'Northern Victoria',\n",
       "       'Northern Victoria Land', 'Ny 7 North Island', 'O11 Taylor Valley',\n",
       "       'Oceans', 'Ongul Island', 'Ongul Islands', 'Onyx River',\n",
       "       'Onyx river', 'Opalifski 1972', 'Orkney Islands', 'P.', 'Pacific',\n",
       "       'Pacific Ocean', 'Papanui Inlet', 'Paracorophium',\n",
       "       'Pegtop Mountain', 'Peninsula', 'Physcia', 'Pinus', 'Pleistocene',\n",
       "       'Pleistocene Speciation', 'Pleistocene glaciation',\n",
       "       'Pleistocene glaciations', 'Pleistocene vicariance', 'Pliocene',\n",
       "       'Podocerus', 'Pole', 'Polytrichaceae', 'Polytrichum',\n",
       "       'Porangahau River', 'Potapov M', 'Prince Charles Mountains',\n",
       "       'Protoranker', 'Queen Maud Mountains', 'Rangauna Harbour |',\n",
       "       'Rhine', 'River', 'River 36°13’,174°28', 'Ross\\nIsland',\n",
       "       'Ross Desert', 'Ross Ice Shelf', 'Ross Ice Shelf Polar Biology',\n",
       "       'Ross Island', 'Ross Island Granite Harbour', 'Ross Sea',\n",
       "       'Ross Sea Region of Antarctica', 'Ross Sea Regions 2',\n",
       "       'Ross Sea Sector of Antarctica', 'Ross-Sea', 'S.',\n",
       "       'SOUTHERN VICTORIA', 'SOUTHERN VICTORIA LAND', 'Schlétterer',\n",
       "       'Scotia Sea', 'Scottnema', 'Seppelt', 'Serpentine Creek',\n",
       "       'Shag River', 'Shangri La', 'Signy Island',\n",
       "       'Signy Island South Orkney Islands', 'Soil', 'Solid Earth',\n",
       "       'South', 'South America', 'South Atlantic', 'South Georgia',\n",
       "       'South Island', 'South Islands', 'South Orkney',\n",
       "       'South Orkney Islands', 'South Polar Region',\n",
       "       'South Sandwich Islands', 'South Shetland',\n",
       "       'South Shetland Island', 'South Shetland Islands',\n",
       "       'South Shetlands', 'South Victoria Land', 'Southern Alps',\n",
       "       'Southern Hemisphere', 'Southern Ocean', 'Southern Ocean Islands',\n",
       "       'Southern Victoria Land', 'Southland', 'Sperm Bluff', 'Sphagnum',\n",
       "       'Springtail', 'Springtail Point', 'Springtail Pt',\n",
       "       'St. Lawrence River', 'Stewart Island', 'Strandtmann',\n",
       "       'Sub-Antarctic', 'Sub-Antarctic islands', 'Syowa', 'Syowa Station',\n",
       "       'S§ Covarrubias', 'Taq', 'Tasman Sea', 'Tauranga Harbour',\n",
       "       'Taylor Valley', 'Tereshkovoi Oasis', 'Terra Nova Bay',\n",
       "       'Terra Nova Bay/Jang Bogo Station', 'Terrestrial Antarctic',\n",
       "       'Terrestrial Fauna', 'Tiger Island', 'Torricelli et al.',\n",
       "       'Towle Glacier', 'Transantarctic Mountain',\n",
       "       'Transantarctic Mountains', 'Tripp Island', 'Tris',\n",
       "       'Tucker Glacier', 'Tukey', 'Upper', 'Usnea antarctica', 'Valley',\n",
       "       'Valleys', 'Vestfold Hills', 'Victoria', 'Victoria Land',\n",
       "       'Victoria Land of COI', 'Vill', 'Waikawa Harbour |',\n",
       "       'Waikouaiti River', 'Waimapu Estuary', 'West', 'West Antarctic',\n",
       "       'West Antarctic Ice Sheet', 'West Antarctica',\n",
       "       'Western Antarctic Ice Sheet', 'Western Antarctic Peninsula',\n",
       "       'Western Antarctica', 'Western Australia', 'Whanganui River',\n",
       "       'Windmill Islands', 'Womersley', 'Wright Valley', 'Yule Bay',\n",
       "       '_ the Dry Valleys', 'a Maritime Antarctic', 'a Tauranga Harbour',\n",
       "       'aurait', 'earth', 'g Tris-HCl Dissolve', 'gm~°',\n",
       "       'pre-Pleistocene', 'voy Ba ea', '°', '° 00', '°24’S', '°40’S',\n",
       "       '°S.', '’s Island'], dtype=object)"
      ]
     },
     "execution_count": 481,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"All locations extracted by large Spacy model\")\n",
    "print(\"--------------------------------------------\")\n",
    "locations_in_inital_set_lg.sort_values('Location').Location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 484,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T03:14:04.097637Z",
     "start_time": "2021-01-31T03:14:04.081251Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not found in any of the referenced Antarctica gazetteers\n",
      "---------------------------------------------------------------------------\n",
      "1227\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['(Fanciulli et al.', '(South America', '015° Granite Harbour',\n",
       "       '4a-c', 'AAZ6358', 'ANTARCTIC COASTAL LANDSCAPES', 'ANTARCTICA',\n",
       "       'Africa', 'Alpine', 'Antarctic', 'Antarctic B.', 'Antarctic Bryum',\n",
       "       'Antarctic Ceratodon', 'Antarctic Circle',\n",
       "       'Antarctic Coastal Landscapes', 'Antarctic Collembola',\n",
       "       'Antarctic Continent\\n', 'Antarctic Desert Ecosystems',\n",
       "       'Antarctic Dry Valley', 'Antarctic Ecology',\n",
       "       'Antarctic Hennidiella', 'Antarctic Ice-Free Coastal Landscapes',\n",
       "       'Antarctic Ice-Free Coastal Landscapes, Ecological Studies Analysis and Synthesis',\n",
       "       'Antarctic Ice-Free Landscapes', 'Antarctic Lake',\n",
       "       'Antarctic Map Folio Series - Folio 5', 'Antarctic Marion Island',\n",
       "       'Antarctic Oasis', 'Antarctic Penisula', 'Antarctic Soil',\n",
       "       'Antarctic South', 'Antarctic Terrestrial', 'Antarctic region',\n",
       "       'Antarctica Cape Bird', 'Antarctica Granite Harbour',\n",
       "       'Antarctica Miers Valley', 'Antarctica Peninsula',\n",
       "       'Antarctica Taylor Valley', 'Arctic', 'Asia',\n",
       "       'Australasian Antarctic Expedition 1912', 'Avicennia marina',\n",
       "       'Bay of Fundy', 'Bay of Plenty', 'Beaufort', 'Beet et al.',\n",
       "       'Black IV', 'Bluff', 'Botany Bay Physcia', 'Botany bay',\n",
       "       'Brighton River', 'Brisbane River', 'Bunger Oasis',\n",
       "       'CONTINENT GRANITE HARBOUR Cape Geology', 'Campbell Island',\n",
       "       'Canary Islands', 'Cape 7', 'Caribbean', 'Catlins Lake',\n",
       "       'Cenozoic Islands', 'Chathams Islands', 'Chignecto Bay',\n",
       "       'Cluster VI', 'Continental Antarctica', 'Contour', 'Cook Strait',\n",
       "       'Cook Strait (Fig', 'Corbicula', 'Corophium', 'Cumberland Basin',\n",
       "       'D. East Antarctica', 'Dnieper River', 'Driest area',\n",
       "       'Dry Valley Streams', 'Dry Valley Terrestrial Plant Communities',\n",
       "       'Dry Valleys', 'Dry Valleys’', 'Dry | Valleys', 'Earth', 'East',\n",
       "       'East Antarctic', 'East Antarctic ice',\n",
       "       'Eastern Antarctic Peninsula', 'Eastern Antarctica', 'Ecosystem',\n",
       "       'Enderby Land eastern Antarctica', 'Europe', 'Fig', 'GHOD',\n",
       "       'Glacier Mount', 'Glaciers', 'Glaciology Dry Valleys',\n",
       "       'Gomphiocephalus', 'Granite Harbour\\nGranite Harbour',\n",
       "       'Granite Harbour B.', 'Great Barrier Island', 'Great Lakes',\n",
       "       'Great Lakes - St. Lawrence River', 'Harbour',\n",
       "       'Harbour | Ross Sea', 'Heard Island', 'Hervey Bay',\n",
       "       'High Latitude Limnology', 'I. Pacific Insects', 'IV', 'Inner',\n",
       "       'Island Cape', 'Island INI', 'Isthmus', 'J. Great Lakes Res',\n",
       "       'Janetschek', 'Jn Antarctic', 'Jurassic', 'Kandpal et al.',\n",
       "       'Kar Plateau Antarctica', 'Kar plateau', 'Kilometers', 'L.',\n",
       "       'LTER', 'La 97', 'Lake', 'Lake Fryxell Basin', 'Lake North | Cape',\n",
       "       'Lake Rotorua', 'Lakes', 'Lakes Rotorua', 'Latitude /Longitude',\n",
       "       'Laurentian Great Lakes', 'Lower Rhine', 'Lower Taylor Valley',\n",
       "       'Mackay Glacier\\n', 'Mackay Glacier region', 'Macquarie Island',\n",
       "       'Macquarie Islands', 'Manukau Harbour', 'Marion Island',\n",
       "       'Marr Glacier east', 'Mawson Rock', 'McMurdo',\n",
       "       'McMurdo Dry Valley', 'McMurdo Sound Region',\n",
       "       'McMurdo Upland (Dry Valleys', 'Mcmurdo Dry Valleys',\n",
       "       'Mediterranean', 'Mediterranean Sea', 'Megaleptoperla', 'Mi Mt',\n",
       "       'Modelling West Antarctic', 'Molodyezhnaya', 'Moreton Bay',\n",
       "       'Mount', 'Mount Seuss', 'Muelleria', 'NEW ZEALAND ESTUARY',\n",
       "       'Nanorchestidae', 'Navi Horiba', 'New Zealand',\n",
       "       'New Zealand Adrift', 'North America',\n",
       "       'North American Great Lakes', 'North Base', 'North Island',\n",
       "       'North Island North Island', 'North Island P.', 'North Islands',\n",
       "       'North Victoria Land', 'North, South', 'Northern Dry Valleys',\n",
       "       'Northern Victoria', 'Northern Victoria Land', 'Ny 7 North Island',\n",
       "       'O11 Taylor Valley', 'Oceans', 'Ongul Islands', 'Onyx river',\n",
       "       'Opalifski 1972', 'Orkney Islands', 'P.', 'Pacific',\n",
       "       'Pacific Ocean', 'Papanui Inlet', 'Paracorophium', 'Peninsula',\n",
       "       'Physcia', 'Pinus', 'Pleistocene', 'Pleistocene Speciation',\n",
       "       'Pleistocene glaciation', 'Pleistocene glaciations',\n",
       "       'Pleistocene vicariance', 'Pliocene', 'Podocerus', 'Pole',\n",
       "       'Polytrichaceae', 'Polytrichum', 'Porangahau River', 'Potapov M',\n",
       "       'Protoranker', 'Rangauna Harbour |', 'Rhine', 'River',\n",
       "       'River 36°13’,174°28', 'Ross\\nIsland', 'Ross Desert',\n",
       "       'Ross Ice Shelf Polar Biology', 'Ross Island Granite Harbour',\n",
       "       'Ross Sea Region of Antarctica', 'Ross Sea Regions 2',\n",
       "       'Ross Sea Sector of Antarctica', 'Ross-Sea', 'S.',\n",
       "       'SOUTHERN VICTORIA', 'SOUTHERN VICTORIA LAND', 'Schlétterer',\n",
       "       'Scotia Sea', 'Scottnema', 'Seppelt', 'Serpentine Creek',\n",
       "       'Shag River', 'Shangri La', 'Signy Island South Orkney Islands',\n",
       "       'Soil', 'Solid Earth', 'South', 'South America', 'South Atlantic',\n",
       "       'South Georgia', 'South Island', 'South Islands', 'South Orkney',\n",
       "       'South Polar Region', 'South Sandwich Islands', 'South Shetland',\n",
       "       'South Shetland Island', 'South Shetlands', 'South Victoria Land',\n",
       "       'Southern Alps', 'Southern Hemisphere', 'Southern Ocean Islands',\n",
       "       'Southern Victoria Land', 'Southland', 'Sphagnum', 'Springtail',\n",
       "       'Springtail Pt', 'St. Lawrence River', 'Stewart Island',\n",
       "       'Strandtmann', 'Sub-Antarctic', 'Sub-Antarctic islands',\n",
       "       'S§ Covarrubias', 'Taq', 'Tauranga Harbour', 'Tereshkovoi Oasis',\n",
       "       'Terra Nova Bay/Jang Bogo Station', 'Terrestrial Antarctic',\n",
       "       'Terrestrial Fauna', 'Torricelli et al.',\n",
       "       'Transantarctic Mountain', 'Tris', 'Tukey', 'Upper',\n",
       "       'Usnea antarctica', 'Valley', 'Valleys', 'Victoria',\n",
       "       'Victoria Land of COI', 'Vill', 'Waikawa Harbour |',\n",
       "       'Waikouaiti River', 'Waimapu Estuary', 'West', 'West Antarctic',\n",
       "       'Western Antarctic Ice Sheet', 'Western Antarctic Peninsula',\n",
       "       'Western Antarctica', 'Western Australia', 'Whanganui River',\n",
       "       'Womersley', '_ the Dry Valleys', 'a Maritime Antarctic',\n",
       "       'a Tauranga Harbour', 'aurait', 'earth', 'g Tris-HCl Dissolve',\n",
       "       'gm~°', 'pre-Pleistocene', 'voy Ba ea', '°', '° 00', '°24’S',\n",
       "       '°40’S', '°S.', '’s Island'], dtype=object)"
      ]
     },
     "execution_count": 484,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not found in any of the referenced Antarctica gazetteers\")\n",
    "print(\"---------------------------------------------------------------------------\")\n",
    "print(len(locations_in_inital_set_lg[(locations_in_inital_set_lg.NZGazAnt==False) & \n",
    "                             (locations_in_inital_set_lg.ScarNZ==False) & \n",
    "                             (locations_in_inital_set_lg.ScarGlobal==False)]))\n",
    "locations_in_inital_set_lg[(locations_in_inital_set_lg.NZGazAnt==False) & \n",
    "                             (locations_in_inital_set_lg.ScarNZ==False) & \n",
    "                             (locations_in_inital_set_lg.ScarGlobal==False)].sort_values('Location').Location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T03:14:40.534935Z",
     "start_time": "2021-01-31T03:14:40.522449Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not matched in any of the referenced gazetteers\n",
      "-------------------------------------------------------------------\n",
      "1013\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['(Fanciulli et al.', '(South America', '015° Granite Harbour',\n",
       "       '4a-c', 'AAZ6358', 'ANTARCTIC COASTAL LANDSCAPES', 'ANTARCTICA',\n",
       "       'Antarctic', 'Antarctic B.', 'Antarctic Bryum',\n",
       "       'Antarctic Ceratodon', 'Antarctic Circle',\n",
       "       'Antarctic Coastal Landscapes', 'Antarctic Collembola',\n",
       "       'Antarctic Continent\\n', 'Antarctic Desert Ecosystems',\n",
       "       'Antarctic Dry Valley', 'Antarctic Ecology',\n",
       "       'Antarctic Hennidiella', 'Antarctic Ice-Free Coastal Landscapes',\n",
       "       'Antarctic Ice-Free Coastal Landscapes, Ecological Studies Analysis and Synthesis',\n",
       "       'Antarctic Ice-Free Landscapes', 'Antarctic Lake',\n",
       "       'Antarctic Map Folio Series - Folio 5', 'Antarctic Marion Island',\n",
       "       'Antarctic Oasis', 'Antarctic Penisula', 'Antarctic Soil',\n",
       "       'Antarctic South', 'Antarctic Terrestrial', 'Antarctic region',\n",
       "       'Antarctica Cape Bird', 'Antarctica Granite Harbour',\n",
       "       'Antarctica Miers Valley', 'Antarctica Peninsula',\n",
       "       'Antarctica Taylor Valley',\n",
       "       'Australasian Antarctic Expedition 1912', 'Avicennia marina',\n",
       "       'Beet et al.', 'Black IV', 'Botany Bay Physcia', 'Botany bay',\n",
       "       'Brighton River', 'Bunger Oasis',\n",
       "       'CONTINENT GRANITE HARBOUR Cape Geology', 'Cape 7',\n",
       "       'Cenozoic Islands', 'Chathams Islands', 'Cluster VI',\n",
       "       'Continental Antarctica', 'Contour', 'Cook Strait (Fig',\n",
       "       'Corbicula', 'Corophium', 'D. East Antarctica', 'Dnieper River',\n",
       "       'Driest area', 'Dry Valley Streams',\n",
       "       'Dry Valley Terrestrial Plant Communities', 'Dry Valleys',\n",
       "       'Dry Valleys’', 'Dry | Valleys', 'East Antarctic',\n",
       "       'East Antarctic ice', 'Eastern Antarctic Peninsula',\n",
       "       'Eastern Antarctica', 'Ecosystem',\n",
       "       'Enderby Land eastern Antarctica', 'Fig', 'GHOD', 'Glacier Mount',\n",
       "       'Glaciers', 'Glaciology Dry Valleys', 'Gomphiocephalus',\n",
       "       'Granite Harbour\\nGranite Harbour', 'Granite Harbour B.',\n",
       "       'Great Lakes - St. Lawrence River', 'Harbour',\n",
       "       'Harbour | Ross Sea', 'High Latitude Limnology',\n",
       "       'I. Pacific Insects', 'IV', 'Inner', 'Island Cape', 'Island INI',\n",
       "       'Isthmus', 'J. Great Lakes Res', 'Janetschek', 'Jn Antarctic',\n",
       "       'Jurassic', 'Kandpal et al.', 'Kar Plateau Antarctica',\n",
       "       'Kar plateau', 'Kilometers', 'L.', 'LTER', 'La 97', 'Lake',\n",
       "       'Lake Fryxell Basin', 'Lake North | Cape', 'Lakes',\n",
       "       'Lakes Rotorua', 'Latitude /Longitude', 'Laurentian Great Lakes',\n",
       "       'Lower Rhine', 'Lower Taylor Valley', 'Mackay Glacier\\n',\n",
       "       'Mackay Glacier region', 'Macquarie Islands', 'Marr Glacier east',\n",
       "       'Mawson Rock', 'McMurdo Dry Valley', 'McMurdo Sound Region',\n",
       "       'McMurdo Upland (Dry Valleys', 'Mcmurdo Dry Valleys',\n",
       "       'Megaleptoperla', 'Mi Mt', 'Modelling West Antarctic',\n",
       "       'Molodyezhnaya', 'Mount', 'Mount Seuss', 'Muelleria',\n",
       "       'NEW ZEALAND ESTUARY', 'Nanorchestidae', 'Navi Horiba',\n",
       "       'New Zealand Adrift', 'North American Great Lakes', 'North Base',\n",
       "       'North Island North Island', 'North Island P.',\n",
       "       'North Victoria Land', 'North, South', 'Northern Dry Valleys',\n",
       "       'Northern Victoria', 'Northern Victoria Land', 'Ny 7 North Island',\n",
       "       'O11 Taylor Valley', 'Oceans', 'Ongul Islands', 'Onyx river',\n",
       "       'Opalifski 1972', 'P.', 'Paracorophium', 'Physcia', 'Pinus',\n",
       "       'Pleistocene', 'Pleistocene Speciation', 'Pleistocene glaciation',\n",
       "       'Pleistocene glaciations', 'Pleistocene vicariance', 'Pliocene',\n",
       "       'Podocerus', 'Polytrichaceae', 'Polytrichum', 'Potapov M',\n",
       "       'Protoranker', 'Rangauna Harbour |', 'River 36°13’,174°28',\n",
       "       'Ross\\nIsland', 'Ross Desert', 'Ross Ice Shelf Polar Biology',\n",
       "       'Ross Island Granite Harbour', 'Ross Sea Region of Antarctica',\n",
       "       'Ross Sea Regions 2', 'Ross Sea Sector of Antarctica', 'Ross-Sea',\n",
       "       'S.', 'SOUTHERN VICTORIA', 'SOUTHERN VICTORIA LAND', 'Schlétterer',\n",
       "       'Scottnema', 'Seppelt', 'Signy Island South Orkney Islands',\n",
       "       'Soil', 'Solid Earth', 'South', 'South Atlantic', 'South Islands',\n",
       "       'South Orkney', 'South Polar Region', 'South Shetland',\n",
       "       'South Shetland Island', 'South Shetlands', 'South Victoria Land',\n",
       "       'Southern Hemisphere', 'Southern Ocean Islands',\n",
       "       'Southern Victoria Land', 'Sphagnum', 'Springtail',\n",
       "       'Springtail Pt', 'Strandtmann', 'Sub-Antarctic',\n",
       "       'Sub-Antarctic islands', 'S§ Covarrubias', 'Taq',\n",
       "       'Tereshkovoi Oasis', 'Terra Nova Bay/Jang Bogo Station',\n",
       "       'Terrestrial Antarctic', 'Terrestrial Fauna', 'Torricelli et al.',\n",
       "       'Transantarctic Mountain', 'Upper', 'Usnea antarctica', 'Valleys',\n",
       "       'Victoria Land of COI', 'Waikawa Harbour |', 'West Antarctic',\n",
       "       'Western Antarctic Ice Sheet', 'Western Antarctic Peninsula',\n",
       "       'Western Antarctica', 'Western Australia', '_ the Dry Valleys',\n",
       "       'a Maritime Antarctic', 'a Tauranga Harbour', 'aurait', 'earth',\n",
       "       'g Tris-HCl Dissolve', 'gm~°', 'pre-Pleistocene', 'voy Ba ea', '°',\n",
       "       '° 00', '°24’S', '°40’S', '°S.', '’s Island'], dtype=object)"
      ]
     },
     "execution_count": 486,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not matched in any of the referenced gazetteers\")\n",
    "print(\"-------------------------------------------------------------------\")\n",
    "print(len(locations_in_inital_set_lg[locations_in_inital_set_lg.exactMatch==False]))\n",
    "locations_in_inital_set_lg[locations_in_inital_set_lg.exactMatch==False].sort_values('Location').Location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 488,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T03:16:06.820433Z",
     "start_time": "2021-01-31T03:16:06.810721Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations subsequently found with close match in a gazetteer\n",
      "----------------------------------------------------------------------\n",
      "77\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['Antarctic Continent\\n', 'Antarctic Penisula',\n",
       "       'Antarctica Peninsula', 'Botany bay', 'Chathams Islands',\n",
       "       'D. East Antarctica', 'Dry Valleys', 'Dry Valleys’',\n",
       "       'East Antarctic', 'Eastern Antarctica', 'Granite Harbour B.',\n",
       "       'Heard Island', 'Hervey Bay', 'Kar plateau', 'Lakes Rotorua',\n",
       "       'Mackay Glacier\\n', 'Marion Island', 'Mawson Rock',\n",
       "       'McMurdo Dry Valley', 'Mcmurdo Dry Valleys', 'Molodyezhnaya',\n",
       "       'Mount Seuss', 'Ongul Islands', 'Onyx river', 'Orkney Islands',\n",
       "       'Ross\\nIsland', 'Shangri La', 'South Islands',\n",
       "       'South Shetland Island', 'Transantarctic Mountain',\n",
       "       'Waikawa Harbour |', 'West Antarctic',\n",
       "       'Western Antarctic Ice Sheet', 'Western Antarctica',\n",
       "       'a Tauranga Harbour'], dtype=object)"
      ]
     },
     "execution_count": 488,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations subsequently found with a close match in a gazetteer\")\n",
    "print(\"------------------------------------------------------------------------\")\n",
    "print(len(locations_in_inital_set_lg[locations_in_inital_set_lg.Close_Match==True]))\n",
    "locations_in_inital_set_lg[locations_in_inital_set_lg.Close_Match==True].sort_values('Location').Location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 492,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T03:18:37.225234Z",
     "start_time": "2021-01-31T03:18:37.215287Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations subsequently found with a partial match in a gazetteer\n",
      "-------------------------------------------------------------------------\n",
      "358\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['015° Granite Harbour', 'Antarctic Continent\\n',\n",
       "       'Antarctic Dry Valley', 'Antarctica Cape Bird',\n",
       "       'Antarctica Granite Harbour', 'Antarctica Miers Valley',\n",
       "       'Antarctica Taylor Valley', 'Botany Bay Physcia',\n",
       "       'CONTINENT GRANITE HARBOUR Cape Geology', 'Cook Strait (Fig',\n",
       "       'D. East Antarctica', 'Dry Valley Streams',\n",
       "       'Dry Valley Terrestrial Plant Communities',\n",
       "       'Eastern Antarctic Peninsula', 'Enderby Land eastern Antarctica',\n",
       "       'Granite Harbour\\nGranite Harbour', 'Granite Harbour B.',\n",
       "       'Great Lakes - St. Lawrence River', 'Harbour | Ross Sea',\n",
       "       'Kar Plateau Antarctica', 'Lake Fryxell Basin',\n",
       "       'Lower Taylor Valley', 'Mackay Glacier\\n', 'Mackay Glacier region',\n",
       "       'Marr Glacier east', 'McMurdo Dry Valley', 'McMurdo Sound Region',\n",
       "       'North Island North Island', 'North Island P.',\n",
       "       'North Victoria Land', 'Northern Victoria Land',\n",
       "       'Ny 7 North Island', 'O11 Taylor Valley',\n",
       "       'Ross Ice Shelf Polar Biology', 'Ross Island Granite Harbour',\n",
       "       'Ross Sea Region of Antarctica', 'Ross Sea Regions 2',\n",
       "       'Ross Sea Sector of Antarctica',\n",
       "       'Signy Island South Orkney Islands', 'South Victoria Land',\n",
       "       'Southern Ocean Islands', 'Southern Victoria Land',\n",
       "       'St. Lawrence River', 'Terra Nova Bay/Jang Bogo Station',\n",
       "       'Victoria Land of COI', 'Waikawa Harbour |',\n",
       "       'Western Antarctic Peninsula', 'a Tauranga Harbour'], dtype=object)"
      ]
     },
     "execution_count": 492,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations subsequently found with a partial match in a gazetteer\")\n",
    "print(\"-------------------------------------------------------------------------\")\n",
    "print(len(locations_in_inital_set_lg[locations_in_inital_set_lg.PartialMatch==True]))\n",
    "locations_in_inital_set_lg[locations_in_inital_set_lg.PartialMatch==True].sort_values('Location').Location.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 525,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T04:26:04.917792Z",
     "start_time": "2021-01-31T04:26:04.912502Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['thesis.txt', 'thesis (1).txt', 'gen-2015-0194.txt',\n",
       "       'fevo-07-00076.txt', 'greenslade2018.txt', 'thesis (2).txt',\n",
       "       'thesis (3).txt'], dtype=object)"
      ]
     },
     "execution_count": 525,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "locations_in_inital_set_lg[locations_in_inital_set_lg.PartialMatch==True].Source.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 493,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T03:20:33.261114Z",
     "start_time": "2021-01-31T03:20:33.249465Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted locations not found by exact, close or partial match in a gazetteer\n",
      "-----------------------------------------------------------------------------\n",
      "592\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['(Fanciulli et al.', '(South America', '4a-c', 'AAZ6358',\n",
       "       'ANTARCTIC COASTAL LANDSCAPES', 'ANTARCTICA', 'Antarctic',\n",
       "       'Antarctic B.', 'Antarctic Bryum', 'Antarctic Ceratodon',\n",
       "       'Antarctic Circle', 'Antarctic Coastal Landscapes',\n",
       "       'Antarctic Collembola', 'Antarctic Desert Ecosystems',\n",
       "       'Antarctic Ecology', 'Antarctic Hennidiella',\n",
       "       'Antarctic Ice-Free Coastal Landscapes',\n",
       "       'Antarctic Ice-Free Coastal Landscapes, Ecological Studies Analysis and Synthesis',\n",
       "       'Antarctic Ice-Free Landscapes', 'Antarctic Lake',\n",
       "       'Antarctic Map Folio Series - Folio 5', 'Antarctic Marion Island',\n",
       "       'Antarctic Oasis', 'Antarctic Soil', 'Antarctic South',\n",
       "       'Antarctic Terrestrial', 'Antarctic region',\n",
       "       'Australasian Antarctic Expedition 1912', 'Avicennia marina',\n",
       "       'Beet et al.', 'Black IV', 'Brighton River', 'Bunger Oasis',\n",
       "       'Cape 7', 'Cenozoic Islands', 'Cluster VI',\n",
       "       'Continental Antarctica', 'Contour', 'Corbicula', 'Corophium',\n",
       "       'Dnieper River', 'Driest area', 'Dry | Valleys',\n",
       "       'East Antarctic ice', 'Ecosystem', 'Fig', 'GHOD', 'Glacier Mount',\n",
       "       'Glaciers', 'Glaciology Dry Valleys', 'Gomphiocephalus', 'Harbour',\n",
       "       'High Latitude Limnology', 'I. Pacific Insects', 'IV', 'Inner',\n",
       "       'Island Cape', 'Island INI', 'Isthmus', 'J. Great Lakes Res',\n",
       "       'Janetschek', 'Jn Antarctic', 'Jurassic', 'Kandpal et al.',\n",
       "       'Kilometers', 'L.', 'LTER', 'La 97', 'Lake', 'Lake North | Cape',\n",
       "       'Lakes', 'Latitude /Longitude', 'Laurentian Great Lakes',\n",
       "       'Lower Rhine', 'Macquarie Islands', 'McMurdo Upland (Dry Valleys',\n",
       "       'Megaleptoperla', 'Mi Mt', 'Modelling West Antarctic', 'Mount',\n",
       "       'Muelleria', 'NEW ZEALAND ESTUARY', 'Nanorchestidae',\n",
       "       'Navi Horiba', 'New Zealand Adrift', 'North American Great Lakes',\n",
       "       'North Base', 'North, South', 'Northern Dry Valleys',\n",
       "       'Northern Victoria', 'Oceans', 'Opalifski 1972', 'P.',\n",
       "       'Paracorophium', 'Physcia', 'Pinus', 'Pleistocene',\n",
       "       'Pleistocene Speciation', 'Pleistocene glaciation',\n",
       "       'Pleistocene glaciations', 'Pleistocene vicariance', 'Pliocene',\n",
       "       'Podocerus', 'Polytrichaceae', 'Polytrichum', 'Potapov M',\n",
       "       'Protoranker', 'Rangauna Harbour |', 'River 36°13’,174°28',\n",
       "       'Ross Desert', 'Ross-Sea', 'S.', 'SOUTHERN VICTORIA',\n",
       "       'SOUTHERN VICTORIA LAND', 'Schlétterer', 'Scottnema', 'Seppelt',\n",
       "       'Soil', 'Solid Earth', 'South', 'South Atlantic', 'South Orkney',\n",
       "       'South Polar Region', 'South Shetland', 'South Shetlands',\n",
       "       'Southern Hemisphere', 'Sphagnum', 'Springtail', 'Springtail Pt',\n",
       "       'Strandtmann', 'Sub-Antarctic', 'Sub-Antarctic islands',\n",
       "       'S§ Covarrubias', 'Taq', 'Tereshkovoi Oasis',\n",
       "       'Terrestrial Antarctic', 'Terrestrial Fauna', 'Torricelli et al.',\n",
       "       'Upper', 'Usnea antarctica', 'Valleys', 'Western Australia',\n",
       "       '_ the Dry Valleys', 'a Maritime Antarctic', 'aurait', 'earth',\n",
       "       'g Tris-HCl Dissolve', 'gm~°', 'pre-Pleistocene', 'voy Ba ea', '°',\n",
       "       '° 00', '°24’S', '°40’S', '°S.', '’s Island'], dtype=object)"
      ]
     },
     "execution_count": 493,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Extracted locations not found by exact, close or partial match in a gazetteer\")\n",
    "print(\"-----------------------------------------------------------------------------\")\n",
    "print(len(locations_in_inital_set_lg[locations_in_inital_set_lg.Found==False]))\n",
    "locations_in_inital_set_lg[locations_in_inital_set_lg.Found==False].sort_values('Location').Location.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Species"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fraser's Comprehensive Species List "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2021-03-10T22:38:12.212Z"
    }
   },
   "outputs": [],
   "source": [
    "SpeciesReferenceList = pd.read_excel('Antarctic_Species_List.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:11:10.021411Z",
     "start_time": "2021-02-08T03:11:09.971449Z"
    }
   },
   "outputs": [],
   "source": [
    "SpeciesReferenceList['SpeciesString'] = SpeciesReferenceList[['GENUS', 'SPECIES']].agg(' '.join, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:11:11.351219Z",
     "start_time": "2021-02-08T03:11:11.307864Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Taxon_ID</th>\n",
       "      <th>KINGDOM</th>\n",
       "      <th>PHYLUM</th>\n",
       "      <th>CLASS</th>\n",
       "      <th>ORDER</th>\n",
       "      <th>FAMILY</th>\n",
       "      <th>GENUS</th>\n",
       "      <th>SPECIES</th>\n",
       "      <th>SpeciesString</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>115105</td>\n",
       "      <td>Protozoa</td>\n",
       "      <td>Choanozoa</td>\n",
       "      <td>Choanoflagellatea</td>\n",
       "      <td>Acanthoecida</td>\n",
       "      <td>Acanthoecidae</td>\n",
       "      <td>Acanthocorbis</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Acanthocorbis sp.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>102022</td>\n",
       "      <td>Animalia</td>\n",
       "      <td>Arthropoda</td>\n",
       "      <td>Maxillopoda</td>\n",
       "      <td>Cyclopoida</td>\n",
       "      <td>Cyclopidae</td>\n",
       "      <td>Diacyclops</td>\n",
       "      <td>mirnyi</td>\n",
       "      <td>Diacyclops mirnyi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100179</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Acarospora sp.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>110864</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>austroshetlandica</td>\n",
       "      <td>Acarospora austroshetlandica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>110865</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>badiofusca</td>\n",
       "      <td>Acarospora badiofusca</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>110866</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>cf. austroshetlandica</td>\n",
       "      <td>Acarospora cf. austroshetlandica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>110867</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>cf. gwynnii</td>\n",
       "      <td>Acarospora cf. gwynnii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>110868</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>cf. macrocyclos</td>\n",
       "      <td>Acarospora cf. macrocyclos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>110869</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>cf. wahlenbergii</td>\n",
       "      <td>Acarospora cf. wahlenbergii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>110870</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>convoluta</td>\n",
       "      <td>Acarospora convoluta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>110872</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>flavocordia</td>\n",
       "      <td>Acarospora flavocordia</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>100178</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>gwynnii</td>\n",
       "      <td>Acarospora gwynnii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>110873</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>macrocyclos</td>\n",
       "      <td>Acarospora macrocyclos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>110874</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>nitrophila</td>\n",
       "      <td>Acarospora nitrophila</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>110875</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>siplei</td>\n",
       "      <td>Acarospora siplei</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>110877</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>tyleri</td>\n",
       "      <td>Acarospora tyleri</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>110878</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>veronensis</td>\n",
       "      <td>Acarospora veronensis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>110879</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>wahlenbergii</td>\n",
       "      <td>Acarospora wahlenbergii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>110880</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Acarosporales</td>\n",
       "      <td>Acarosporaceae</td>\n",
       "      <td>Acarospora</td>\n",
       "      <td>williamsii</td>\n",
       "      <td>Acarospora williamsii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>116363</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Bacillariophyceae</td>\n",
       "      <td>Achnanthales</td>\n",
       "      <td>Achnanthaceae</td>\n",
       "      <td>Achnanthes</td>\n",
       "      <td>brevipes</td>\n",
       "      <td>Achnanthes brevipes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>116368</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Bacillariophyceae</td>\n",
       "      <td>Achnanthales</td>\n",
       "      <td>Achnanthaceae</td>\n",
       "      <td>Planothidium</td>\n",
       "      <td>delicatulum</td>\n",
       "      <td>Planothidium delicatulum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>116371</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Bacillariophyceae</td>\n",
       "      <td>Achnanthales</td>\n",
       "      <td>Achnanthaceae</td>\n",
       "      <td>Achnanthes</td>\n",
       "      <td>gainii</td>\n",
       "      <td>Achnanthes gainii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>116376</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Bacillariophyceae</td>\n",
       "      <td>Achnanthales</td>\n",
       "      <td>Achnanthaceae</td>\n",
       "      <td>Achnanthes</td>\n",
       "      <td>marginatula</td>\n",
       "      <td>Achnanthes marginatula</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>116380</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Bacillariophyceae</td>\n",
       "      <td>Achnanthales</td>\n",
       "      <td>Achnanthaceae</td>\n",
       "      <td>Achnanthes</td>\n",
       "      <td>nodosa</td>\n",
       "      <td>Achnanthes nodosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>116381</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Bacillariophyceae</td>\n",
       "      <td>Achnanthales</td>\n",
       "      <td>Achnanthaceae</td>\n",
       "      <td>Achnanthes</td>\n",
       "      <td>parvula</td>\n",
       "      <td>Achnanthes parvula</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>116382</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Bacillariophyceae</td>\n",
       "      <td>Achnanthales</td>\n",
       "      <td>Achnanthaceae</td>\n",
       "      <td>Achnanthes</td>\n",
       "      <td>petersenii</td>\n",
       "      <td>Achnanthes petersenii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>115106</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Bacillariophyceae</td>\n",
       "      <td>Achnanthales</td>\n",
       "      <td>Achnanthaceae</td>\n",
       "      <td>Achnanthes</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Achnanthes sp.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>116384</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Bacillariophyceae</td>\n",
       "      <td>Achnanthales</td>\n",
       "      <td>Achnanthaceae</td>\n",
       "      <td>Achnanthes</td>\n",
       "      <td>taylorensis</td>\n",
       "      <td>Achnanthes taylorensis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>114639</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Sordariomycetes</td>\n",
       "      <td>Hypocreales</td>\n",
       "      <td>Not assigned</td>\n",
       "      <td>Sarocladium</td>\n",
       "      <td>strictum</td>\n",
       "      <td>Sarocladium strictum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>110885</td>\n",
       "      <td>Plantae</td>\n",
       "      <td>Bryophyta</td>\n",
       "      <td>Bryopsida</td>\n",
       "      <td>Hypnales</td>\n",
       "      <td>Amblystegiaceae</td>\n",
       "      <td>Acrocladium</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Acrocladium sp.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2095</th>\n",
       "      <td>113165</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Eurotiomycetes</td>\n",
       "      <td>Verrucariales</td>\n",
       "      <td>Verrucariaceae</td>\n",
       "      <td>Verrucaria</td>\n",
       "      <td>maura</td>\n",
       "      <td>Verrucaria maura</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2096</th>\n",
       "      <td>113168</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Eurotiomycetes</td>\n",
       "      <td>Verrucariales</td>\n",
       "      <td>Verrucariaceae</td>\n",
       "      <td>Verrucaria</td>\n",
       "      <td>psychrophila</td>\n",
       "      <td>Verrucaria psychrophila</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2097</th>\n",
       "      <td>113170</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Eurotiomycetes</td>\n",
       "      <td>Verrucariales</td>\n",
       "      <td>Verrucariaceae</td>\n",
       "      <td>Verrucaria</td>\n",
       "      <td>serpuloides</td>\n",
       "      <td>Verrucaria serpuloides</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2098</th>\n",
       "      <td>113171</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Eurotiomycetes</td>\n",
       "      <td>Verrucariales</td>\n",
       "      <td>Verrucariaceae</td>\n",
       "      <td>Verrucaria</td>\n",
       "      <td>siplei</td>\n",
       "      <td>Verrucaria siplei</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2099</th>\n",
       "      <td>101283</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Eurotiomycetes</td>\n",
       "      <td>Verrucariales</td>\n",
       "      <td>Verrucariaceae</td>\n",
       "      <td>Verrucaria</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Verrucaria sp.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2100</th>\n",
       "      <td>113172</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Eurotiomycetes</td>\n",
       "      <td>Verrucariales</td>\n",
       "      <td>Verrucariaceae</td>\n",
       "      <td>Verrucaria</td>\n",
       "      <td>tesselatula</td>\n",
       "      <td>Verrucaria tesselatula</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2101</th>\n",
       "      <td>102264</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ciliophora</td>\n",
       "      <td>Oligohymenophorea</td>\n",
       "      <td>Peritrichida</td>\n",
       "      <td>Vorticellidae</td>\n",
       "      <td>Vorticella</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Vorticella sp.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2102</th>\n",
       "      <td>113179</td>\n",
       "      <td>Plantae</td>\n",
       "      <td>Bryophyta</td>\n",
       "      <td>Bryopsida</td>\n",
       "      <td>Hypnales</td>\n",
       "      <td>Amblystegiaceae</td>\n",
       "      <td>Warnstorfia</td>\n",
       "      <td>cf. laculosa</td>\n",
       "      <td>Warnstorfia cf. laculosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2103</th>\n",
       "      <td>113182</td>\n",
       "      <td>Plantae</td>\n",
       "      <td>Bryophyta</td>\n",
       "      <td>Bryopsida</td>\n",
       "      <td>Hypnales</td>\n",
       "      <td>Amblystegiaceae</td>\n",
       "      <td>Warnstorfia</td>\n",
       "      <td>fontinaliopsis</td>\n",
       "      <td>Warnstorfia fontinaliopsis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2104</th>\n",
       "      <td>113183</td>\n",
       "      <td>Plantae</td>\n",
       "      <td>Bryophyta</td>\n",
       "      <td>Bryopsida</td>\n",
       "      <td>Hypnales</td>\n",
       "      <td>Amblystegiaceae</td>\n",
       "      <td>Warnstorfia</td>\n",
       "      <td>laculosa</td>\n",
       "      <td>Warnstorfia laculosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2105</th>\n",
       "      <td>110065</td>\n",
       "      <td>Plantae</td>\n",
       "      <td>Bryophyta</td>\n",
       "      <td>Bryopsida</td>\n",
       "      <td>Hypnales</td>\n",
       "      <td>Amblystegiaceae</td>\n",
       "      <td>Warnstorfia</td>\n",
       "      <td>sarmentosa</td>\n",
       "      <td>Warnstorfia sarmentosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2106</th>\n",
       "      <td>117850</td>\n",
       "      <td>Plantae</td>\n",
       "      <td>Chlorophyta</td>\n",
       "      <td>Chlorophyceae</td>\n",
       "      <td>Sphaeropleales</td>\n",
       "      <td>Scenedesmaceae</td>\n",
       "      <td>Westella</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Westella sp.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2107</th>\n",
       "      <td>113190</td>\n",
       "      <td>Plantae</td>\n",
       "      <td>Bryophyta</td>\n",
       "      <td>Bryopsida</td>\n",
       "      <td>Pottiales</td>\n",
       "      <td>Pottiaceae</td>\n",
       "      <td>Willia</td>\n",
       "      <td>austroleucophaea</td>\n",
       "      <td>Willia austroleucophaea</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2108</th>\n",
       "      <td>117854</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Xanthophyceae</td>\n",
       "      <td>Tribonematales</td>\n",
       "      <td>Xanthonemataceae</td>\n",
       "      <td>Xanthonema</td>\n",
       "      <td>hormidioides</td>\n",
       "      <td>Xanthonema hormidioides</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2109</th>\n",
       "      <td>117855</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Xanthophyceae</td>\n",
       "      <td>Tribonematales</td>\n",
       "      <td>Xanthonemataceae</td>\n",
       "      <td>Xanthonema</td>\n",
       "      <td>sessile</td>\n",
       "      <td>Xanthonema sessile</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2110</th>\n",
       "      <td>117856</td>\n",
       "      <td>Chromista</td>\n",
       "      <td>Ochrophyta</td>\n",
       "      <td>Xanthophyceae</td>\n",
       "      <td>Tribonematales</td>\n",
       "      <td>Xanthonemataceae</td>\n",
       "      <td>Xanthonema</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Xanthonema sp.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2111</th>\n",
       "      <td>110213</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Teloschistales</td>\n",
       "      <td>Teloschistaceae</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>candelaria</td>\n",
       "      <td>Xanthoria candelaria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2112</th>\n",
       "      <td>113197</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Teloschistales</td>\n",
       "      <td>Teloschistaceae</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>cf. candelaria</td>\n",
       "      <td>Xanthoria cf. candelaria</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2113</th>\n",
       "      <td>113198</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Teloschistales</td>\n",
       "      <td>Teloschistaceae</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>cf. elegans</td>\n",
       "      <td>Xanthoria cf. elegans</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2114</th>\n",
       "      <td>113199</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Teloschistales</td>\n",
       "      <td>Teloschistaceae</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>cf. mawsonii</td>\n",
       "      <td>Xanthoria cf. mawsonii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2115</th>\n",
       "      <td>101488</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Teloschistales</td>\n",
       "      <td>Teloschistaceae</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>elegans</td>\n",
       "      <td>Xanthoria elegans</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2116</th>\n",
       "      <td>113200</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Teloschistales</td>\n",
       "      <td>Teloschistaceae</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>lychnea</td>\n",
       "      <td>Xanthoria lychnea</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2117</th>\n",
       "      <td>101298</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Teloschistales</td>\n",
       "      <td>Teloschistaceae</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>mawsonii</td>\n",
       "      <td>Xanthoria mawsonii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2118</th>\n",
       "      <td>113201</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Teloschistales</td>\n",
       "      <td>Teloschistaceae</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>siplei</td>\n",
       "      <td>Xanthoria siplei</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2119</th>\n",
       "      <td>101489</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lecanoromycetes</td>\n",
       "      <td>Teloschistales</td>\n",
       "      <td>Teloschistaceae</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Xanthoria sp.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2120</th>\n",
       "      <td>117857</td>\n",
       "      <td>Bacteria</td>\n",
       "      <td>Cyanobacteria</td>\n",
       "      <td>Cyanophyceae</td>\n",
       "      <td>Pleurocapsales</td>\n",
       "      <td>Pleurocapsaceae</td>\n",
       "      <td>Xenococcus</td>\n",
       "      <td>schousboei</td>\n",
       "      <td>Xenococcus schousboei</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2121</th>\n",
       "      <td>101848</td>\n",
       "      <td>Animalia</td>\n",
       "      <td>Arthropoda</td>\n",
       "      <td>Arachnida</td>\n",
       "      <td>Sarcoptiformes</td>\n",
       "      <td>Avenzoariidae</td>\n",
       "      <td>Zachvatkinia</td>\n",
       "      <td>hydrobatidii</td>\n",
       "      <td>Zachvatkinia hydrobatidii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2122</th>\n",
       "      <td>114592</td>\n",
       "      <td>Animalia</td>\n",
       "      <td>Arthropoda</td>\n",
       "      <td>Arachnida</td>\n",
       "      <td>Sarcoptiformes</td>\n",
       "      <td>Avenzoariidae</td>\n",
       "      <td>Zachvatkinia</td>\n",
       "      <td>stercorarii</td>\n",
       "      <td>Zachvatkinia stercorarii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2123</th>\n",
       "      <td>113203</td>\n",
       "      <td>Fungi</td>\n",
       "      <td>Ascomycota</td>\n",
       "      <td>Lichinomycetes</td>\n",
       "      <td>Lichinales</td>\n",
       "      <td>Lichinaceae</td>\n",
       "      <td>Zahlbrucknerella</td>\n",
       "      <td>patagonica</td>\n",
       "      <td>Zahlbrucknerella patagonica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2124</th>\n",
       "      <td>117858</td>\n",
       "      <td>Plantae</td>\n",
       "      <td>Charophyta</td>\n",
       "      <td>Conjugatophyceae (zygnemaphyceae)</td>\n",
       "      <td>Zygnematales</td>\n",
       "      <td>Zygnemataceae</td>\n",
       "      <td>Zygnema</td>\n",
       "      <td>sp.</td>\n",
       "      <td>Zygnema sp.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2125 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Taxon_ID    KINGDOM         PHYLUM                              CLASS  \\\n",
       "0       115105   Protozoa     Choanozoa                   Choanoflagellatea   \n",
       "1       102022   Animalia     Arthropoda                        Maxillopoda   \n",
       "2       100179      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "3       110864      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "4       110865      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "5       110866      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "6       110867      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "7       110868      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "8       110869      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "9       110870      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "10      110872      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "11      100178      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "12      110873      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "13      110874      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "14      110875      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "15      110877      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "16      110878      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "17      110879      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "18      110880      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "19      116363  Chromista     Ochrophyta                  Bacillariophyceae   \n",
       "20      116368  Chromista     Ochrophyta                  Bacillariophyceae   \n",
       "21      116371  Chromista     Ochrophyta                  Bacillariophyceae   \n",
       "22      116376  Chromista     Ochrophyta                  Bacillariophyceae   \n",
       "23      116380  Chromista     Ochrophyta                  Bacillariophyceae   \n",
       "24      116381  Chromista     Ochrophyta                  Bacillariophyceae   \n",
       "25      116382  Chromista     Ochrophyta                  Bacillariophyceae   \n",
       "26      115106  Chromista     Ochrophyta                  Bacillariophyceae   \n",
       "27      116384  Chromista     Ochrophyta                  Bacillariophyceae   \n",
       "28      114639      Fungi     Ascomycota                    Sordariomycetes   \n",
       "29      110885    Plantae      Bryophyta                          Bryopsida   \n",
       "...        ...        ...            ...                                ...   \n",
       "2095    113165      Fungi     Ascomycota                     Eurotiomycetes   \n",
       "2096    113168      Fungi     Ascomycota                     Eurotiomycetes   \n",
       "2097    113170      Fungi     Ascomycota                     Eurotiomycetes   \n",
       "2098    113171      Fungi     Ascomycota                     Eurotiomycetes   \n",
       "2099    101283      Fungi     Ascomycota                     Eurotiomycetes   \n",
       "2100    113172      Fungi     Ascomycota                     Eurotiomycetes   \n",
       "2101    102264  Chromista     Ciliophora                  Oligohymenophorea   \n",
       "2102    113179    Plantae      Bryophyta                          Bryopsida   \n",
       "2103    113182    Plantae      Bryophyta                          Bryopsida   \n",
       "2104    113183    Plantae      Bryophyta                          Bryopsida   \n",
       "2105    110065    Plantae      Bryophyta                          Bryopsida   \n",
       "2106    117850    Plantae    Chlorophyta                      Chlorophyceae   \n",
       "2107    113190    Plantae      Bryophyta                          Bryopsida   \n",
       "2108    117854  Chromista     Ochrophyta                      Xanthophyceae   \n",
       "2109    117855  Chromista     Ochrophyta                      Xanthophyceae   \n",
       "2110    117856  Chromista     Ochrophyta                      Xanthophyceae   \n",
       "2111    110213      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "2112    113197      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "2113    113198      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "2114    113199      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "2115    101488      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "2116    113200      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "2117    101298      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "2118    113201      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "2119    101489      Fungi     Ascomycota                    Lecanoromycetes   \n",
       "2120    117857   Bacteria  Cyanobacteria                       Cyanophyceae   \n",
       "2121    101848   Animalia     Arthropoda                          Arachnida   \n",
       "2122    114592   Animalia     Arthropoda                          Arachnida   \n",
       "2123    113203      Fungi     Ascomycota                     Lichinomycetes   \n",
       "2124    117858    Plantae     Charophyta  Conjugatophyceae (zygnemaphyceae)   \n",
       "\n",
       "               ORDER            FAMILY             GENUS  \\\n",
       "0       Acanthoecida     Acanthoecidae     Acanthocorbis   \n",
       "1         Cyclopoida        Cyclopidae        Diacyclops   \n",
       "2      Acarosporales    Acarosporaceae        Acarospora   \n",
       "3      Acarosporales    Acarosporaceae        Acarospora   \n",
       "4      Acarosporales    Acarosporaceae        Acarospora   \n",
       "5      Acarosporales    Acarosporaceae        Acarospora   \n",
       "6      Acarosporales    Acarosporaceae        Acarospora   \n",
       "7      Acarosporales    Acarosporaceae        Acarospora   \n",
       "8      Acarosporales    Acarosporaceae        Acarospora   \n",
       "9      Acarosporales    Acarosporaceae        Acarospora   \n",
       "10     Acarosporales    Acarosporaceae        Acarospora   \n",
       "11     Acarosporales    Acarosporaceae        Acarospora   \n",
       "12     Acarosporales    Acarosporaceae        Acarospora   \n",
       "13     Acarosporales    Acarosporaceae        Acarospora   \n",
       "14     Acarosporales    Acarosporaceae        Acarospora   \n",
       "15     Acarosporales    Acarosporaceae        Acarospora   \n",
       "16     Acarosporales    Acarosporaceae        Acarospora   \n",
       "17     Acarosporales    Acarosporaceae        Acarospora   \n",
       "18     Acarosporales    Acarosporaceae        Acarospora   \n",
       "19      Achnanthales     Achnanthaceae        Achnanthes   \n",
       "20      Achnanthales     Achnanthaceae      Planothidium   \n",
       "21      Achnanthales     Achnanthaceae        Achnanthes   \n",
       "22      Achnanthales     Achnanthaceae        Achnanthes   \n",
       "23      Achnanthales     Achnanthaceae        Achnanthes   \n",
       "24      Achnanthales     Achnanthaceae        Achnanthes   \n",
       "25      Achnanthales     Achnanthaceae        Achnanthes   \n",
       "26      Achnanthales     Achnanthaceae        Achnanthes   \n",
       "27      Achnanthales     Achnanthaceae        Achnanthes   \n",
       "28       Hypocreales      Not assigned       Sarocladium   \n",
       "29          Hypnales   Amblystegiaceae       Acrocladium   \n",
       "...              ...               ...               ...   \n",
       "2095   Verrucariales    Verrucariaceae        Verrucaria   \n",
       "2096   Verrucariales    Verrucariaceae        Verrucaria   \n",
       "2097   Verrucariales    Verrucariaceae        Verrucaria   \n",
       "2098   Verrucariales    Verrucariaceae        Verrucaria   \n",
       "2099   Verrucariales    Verrucariaceae        Verrucaria   \n",
       "2100   Verrucariales    Verrucariaceae        Verrucaria   \n",
       "2101    Peritrichida     Vorticellidae        Vorticella   \n",
       "2102        Hypnales   Amblystegiaceae       Warnstorfia   \n",
       "2103        Hypnales   Amblystegiaceae       Warnstorfia   \n",
       "2104        Hypnales   Amblystegiaceae       Warnstorfia   \n",
       "2105        Hypnales   Amblystegiaceae       Warnstorfia   \n",
       "2106  Sphaeropleales    Scenedesmaceae          Westella   \n",
       "2107       Pottiales        Pottiaceae            Willia   \n",
       "2108  Tribonematales  Xanthonemataceae        Xanthonema   \n",
       "2109  Tribonematales  Xanthonemataceae        Xanthonema   \n",
       "2110  Tribonematales  Xanthonemataceae        Xanthonema   \n",
       "2111  Teloschistales   Teloschistaceae         Xanthoria   \n",
       "2112  Teloschistales   Teloschistaceae         Xanthoria   \n",
       "2113  Teloschistales   Teloschistaceae         Xanthoria   \n",
       "2114  Teloschistales   Teloschistaceae         Xanthoria   \n",
       "2115  Teloschistales   Teloschistaceae         Xanthoria   \n",
       "2116  Teloschistales   Teloschistaceae         Xanthoria   \n",
       "2117  Teloschistales   Teloschistaceae         Xanthoria   \n",
       "2118  Teloschistales   Teloschistaceae         Xanthoria   \n",
       "2119  Teloschistales   Teloschistaceae         Xanthoria   \n",
       "2120  Pleurocapsales   Pleurocapsaceae        Xenococcus   \n",
       "2121  Sarcoptiformes     Avenzoariidae      Zachvatkinia   \n",
       "2122  Sarcoptiformes     Avenzoariidae      Zachvatkinia   \n",
       "2123      Lichinales       Lichinaceae  Zahlbrucknerella   \n",
       "2124    Zygnematales     Zygnemataceae           Zygnema   \n",
       "\n",
       "                    SPECIES                     SpeciesString  \n",
       "0                       sp.                 Acanthocorbis sp.  \n",
       "1                    mirnyi                 Diacyclops mirnyi  \n",
       "2                       sp.                    Acarospora sp.  \n",
       "3         austroshetlandica      Acarospora austroshetlandica  \n",
       "4                badiofusca             Acarospora badiofusca  \n",
       "5     cf. austroshetlandica  Acarospora cf. austroshetlandica  \n",
       "6               cf. gwynnii            Acarospora cf. gwynnii  \n",
       "7           cf. macrocyclos        Acarospora cf. macrocyclos  \n",
       "8          cf. wahlenbergii       Acarospora cf. wahlenbergii  \n",
       "9                 convoluta              Acarospora convoluta  \n",
       "10              flavocordia            Acarospora flavocordia  \n",
       "11                  gwynnii                Acarospora gwynnii  \n",
       "12              macrocyclos            Acarospora macrocyclos  \n",
       "13               nitrophila             Acarospora nitrophila  \n",
       "14                   siplei                 Acarospora siplei  \n",
       "15                   tyleri                 Acarospora tyleri  \n",
       "16               veronensis             Acarospora veronensis  \n",
       "17             wahlenbergii           Acarospora wahlenbergii  \n",
       "18               williamsii             Acarospora williamsii  \n",
       "19                 brevipes               Achnanthes brevipes  \n",
       "20              delicatulum          Planothidium delicatulum  \n",
       "21                   gainii                 Achnanthes gainii  \n",
       "22              marginatula            Achnanthes marginatula  \n",
       "23                   nodosa                 Achnanthes nodosa  \n",
       "24                  parvula                Achnanthes parvula  \n",
       "25               petersenii             Achnanthes petersenii  \n",
       "26                      sp.                    Achnanthes sp.  \n",
       "27              taylorensis            Achnanthes taylorensis  \n",
       "28                 strictum              Sarocladium strictum  \n",
       "29                      sp.                   Acrocladium sp.  \n",
       "...                     ...                               ...  \n",
       "2095                  maura                  Verrucaria maura  \n",
       "2096           psychrophila           Verrucaria psychrophila  \n",
       "2097            serpuloides            Verrucaria serpuloides  \n",
       "2098                 siplei                 Verrucaria siplei  \n",
       "2099                    sp.                    Verrucaria sp.  \n",
       "2100            tesselatula            Verrucaria tesselatula  \n",
       "2101                    sp.                    Vorticella sp.  \n",
       "2102           cf. laculosa          Warnstorfia cf. laculosa  \n",
       "2103         fontinaliopsis        Warnstorfia fontinaliopsis  \n",
       "2104               laculosa              Warnstorfia laculosa  \n",
       "2105             sarmentosa            Warnstorfia sarmentosa  \n",
       "2106                    sp.                      Westella sp.  \n",
       "2107       austroleucophaea           Willia austroleucophaea  \n",
       "2108           hormidioides           Xanthonema hormidioides  \n",
       "2109                sessile                Xanthonema sessile  \n",
       "2110                    sp.                    Xanthonema sp.  \n",
       "2111             candelaria              Xanthoria candelaria  \n",
       "2112         cf. candelaria          Xanthoria cf. candelaria  \n",
       "2113            cf. elegans             Xanthoria cf. elegans  \n",
       "2114           cf. mawsonii            Xanthoria cf. mawsonii  \n",
       "2115                elegans                 Xanthoria elegans  \n",
       "2116                lychnea                 Xanthoria lychnea  \n",
       "2117               mawsonii                Xanthoria mawsonii  \n",
       "2118                 siplei                  Xanthoria siplei  \n",
       "2119                    sp.                     Xanthoria sp.  \n",
       "2120             schousboei             Xenococcus schousboei  \n",
       "2121           hydrobatidii         Zachvatkinia hydrobatidii  \n",
       "2122            stercorarii          Zachvatkinia stercorarii  \n",
       "2123             patagonica       Zahlbrucknerella patagonica  \n",
       "2124                    sp.                       Zygnema sp.  \n",
       "\n",
       "[2125 rows x 9 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SpeciesReferenceList"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Locating Genus and Species Mentions in Document"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Constructing lists of the unique words in genus and species names we can check through the document to see how the NLP model is currently parsing these words and how often they are not being picked up as part of a named entity or even as nouns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:11:54.907357Z",
     "start_time": "2021-02-08T03:11:54.902001Z"
    }
   },
   "outputs": [],
   "source": [
    "uniqueGenus = SpeciesReferenceList.GENUS.unique()\n",
    "uniqueSpecies = SpeciesReferenceList.SPECIES.unique()\n",
    "uniqueSpeciesStrings = SpeciesReferenceList.SpeciesString.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T01:33:41.167500Z",
     "start_time": "2021-02-25T01:33:41.165089Z"
    }
   },
   "outputs": [],
   "source": [
    "# 'Diplosphaera' in uniqueGenus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:12:19.205918Z",
     "start_time": "2021-02-08T03:12:19.203136Z"
    }
   },
   "outputs": [],
   "source": [
    "genus = []\n",
    "species = []\n",
    "for token in document:\n",
    "    if str(token) in uniqueGenus:\n",
    "        genus.append([token.text, token.lemma_, token.idx, \n",
    "                      token.pos_, token.tag_, token.dep_, \n",
    "                      document[token.i+1].text, token.ent_type_, token.sent])\n",
    "    if token.pos_ != 'PUNCT' and str(token) in uniqueSpecies:\n",
    "        species.append([token.text, token.lemma_, token.idx, \n",
    "                         token.pos_, token.tag_, token.dep_, \n",
    "                         document[token.i-1].text, document[token.i-1].shape_, document[token.i+1].text,\n",
    "                         token.ent_type_, token.sent])\n",
    "genusInDoc = pd.DataFrame(genus,columns=['Token', 'Lemma', 'PositionofWord', \n",
    "                                 'PartOfSpeech', 'PartOfSpeech_Fine', 'Dependency',\n",
    "                                'FollowingToken', 'NamedEntityType', 'Sentence'])\n",
    "speciesInDoc = pd.DataFrame(species,columns=['Token', 'Lemma', 'PositionofWord', \n",
    "                                 'PartOfSpeech', 'PartOfSpeech_Fine', 'Dependency',\n",
    "                                'PreviousToken','PreviousToken_Shape','FollowingToken', 'NamedEntityType', 'Sentence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T23:31:33.291218Z",
     "start_time": "2021-01-26T23:31:33.222275Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Token</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>PositionofWord</th>\n",
       "      <th>PartOfSpeech</th>\n",
       "      <th>PartOfSpeech_Fine</th>\n",
       "      <th>Dependency</th>\n",
       "      <th>FollowingToken</th>\n",
       "      <th>NamedEntityType</th>\n",
       "      <th>Sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>1330</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(The, native, species, Cryptopygus, antarcticu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>1459</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>viatica</td>\n",
       "      <td></td>\n",
       "      <td>(Hypogastrura, viatica, (, Tullberg, ))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sanionia</td>\n",
       "      <td>Sanionia</td>\n",
       "      <td>7748</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>pobj</td>\n",
       "      <td>as</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>(Extensive, moss, carpets, cover, the, center,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sanionia</td>\n",
       "      <td>Sanionia</td>\n",
       "      <td>7780</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>uncinata</td>\n",
       "      <td></td>\n",
       "      <td>(Sanionia, uncinata, (, Hedw, ., ))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sanionia</td>\n",
       "      <td>Sanionia</td>\n",
       "      <td>7880</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>amod</td>\n",
       "      <td>uncinata</td>\n",
       "      <td></td>\n",
       "      <td>(Sanionia, uncinata, forms, monospecific, comm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Polytrichastrum</td>\n",
       "      <td>Polytrichastrum</td>\n",
       "      <td>7955</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>alpinum</td>\n",
       "      <td>WORK_OF_ART</td>\n",
       "      <td>(Sanionia, uncinata, forms, monospecific, comm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Bryum</td>\n",
       "      <td>Bryum</td>\n",
       "      <td>8000</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>pseudotriquetrum</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(G.L., Sm, ., and, Bryum, pseudotriquetrum, (,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Sanionia</td>\n",
       "      <td>sanionia</td>\n",
       "      <td>8101</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>compound</td>\n",
       "      <td>georgicouncinata</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>(Sanionia, georgicouncinata, is, often, found,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Warnstorfia</td>\n",
       "      <td>Warnstorfia</td>\n",
       "      <td>8147</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>fontinaliopsis</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>(Sanionia, georgicouncinata, is, often, found,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>Xanthoria</td>\n",
       "      <td>8286</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>spp</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>(Different, crustose, lichen, species, ,, such...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Caloplaca</td>\n",
       "      <td>Caloplaca</td>\n",
       "      <td>8304</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>spp</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>(Different, crustose, lichen, species, ,, such...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Prasiola</td>\n",
       "      <td>prasiola</td>\n",
       "      <td>8367</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>crispa</td>\n",
       "      <td>PRODUCT</td>\n",
       "      <td>(The, green, alga, Prasiola, crispa, (, Lightf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Pygoscelis</td>\n",
       "      <td>Pygoscelis</td>\n",
       "      <td>8523</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>papua</td>\n",
       "      <td>NORP</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Pygoscelis</td>\n",
       "      <td>Pygoscelis</td>\n",
       "      <td>8578</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Macronectes</td>\n",
       "      <td>Macronectes</td>\n",
       "      <td>8643</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNPS</td>\n",
       "      <td>compound</td>\n",
       "      <td>giganteus</td>\n",
       "      <td>NORP</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Larus</td>\n",
       "      <td>Larus</td>\n",
       "      <td>8694</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>dominicanus</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Mirounga</td>\n",
       "      <td>Mirounga</td>\n",
       "      <td>8783</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>leonina</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Leptonychotes</td>\n",
       "      <td>Leptonychotes</td>\n",
       "      <td>8834</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNPS</td>\n",
       "      <td>appos</td>\n",
       "      <td>weddellii</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Arctocephalus</td>\n",
       "      <td>arctocephalus</td>\n",
       "      <td>8900</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>gazella</td>\n",
       "      <td></td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>11720</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>viatica</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(The, cosmopolitan, species, Hypogastrura, via...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>Friesea</td>\n",
       "      <td>15446</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>bispinosa</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>Friesea</td>\n",
       "      <td>15481</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>grisea</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>Friesea</td>\n",
       "      <td>15514</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>woyciechowskii</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Tullbergia</td>\n",
       "      <td>Tullbergia</td>\n",
       "      <td>15551</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>mixta</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Archisotoma</td>\n",
       "      <td>Archisotoma</td>\n",
       "      <td>15584</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>brucei</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>15622</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>15672</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>badasa</td>\n",
       "      <td>NORP</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>15799</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>viatica</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(Two, species, are, non, -, indigenous, in, An...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>18170</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(These, species, are, Cryptopygus, antarcticus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>18244</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>badasa</td>\n",
       "      <td>NORP</td>\n",
       "      <td>(These, species, are, Cryptopygus, antarcticus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>18623</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>viatica</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(Hypogastrura, viatica, was, the, only, specie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>friesea</td>\n",
       "      <td>19140</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>compound</td>\n",
       "      <td>bispinosa</td>\n",
       "      <td></td>\n",
       "      <td>(Friesea, bispinosa, ,, F., grisea, ,, and, F....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>friesea</td>\n",
       "      <td>19351</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>compound</td>\n",
       "      <td>grisea</td>\n",
       "      <td></td>\n",
       "      <td>(Friesea, grisea, can, be, considered, as, eur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Tullbergia</td>\n",
       "      <td>Tullbergia</td>\n",
       "      <td>20186</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>mixta</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(The, family, Tullbergiidae, was, represented,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Archisotoma</td>\n",
       "      <td>Archisotoma</td>\n",
       "      <td>20508</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>brucei</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>(Archisotoma, brucei, only, was, found, in, tw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>20688</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td></td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, was, f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Prasiola</td>\n",
       "      <td>prasiola</td>\n",
       "      <td>20940</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nmod</td>\n",
       "      <td>crispa</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(This, species, presents, the, highest, densit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>21523</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>badasa</td>\n",
       "      <td></td>\n",
       "      <td>(Cryptopygus, badasa, also, appeared, in, a, h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>friesea</td>\n",
       "      <td>26262</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>compound</td>\n",
       "      <td>grisea</td>\n",
       "      <td></td>\n",
       "      <td>((, 2014, ), for, the, maritime, Antarctica, r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Tullbergia</td>\n",
       "      <td>Tullbergia</td>\n",
       "      <td>26326</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>mixta</td>\n",
       "      <td>GPE</td>\n",
       "      <td>((, 2014, ), for, the, maritime, Antarctica, r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>friesea</td>\n",
       "      <td>26376</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>compound</td>\n",
       "      <td>bispinosa</td>\n",
       "      <td></td>\n",
       "      <td>(In, addition, ,, four, other, species, —, Fri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Archisotoma</td>\n",
       "      <td>Archisotoma</td>\n",
       "      <td>26395</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>brucei</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(In, addition, ,, four, other, species, —, Fri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>26415</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>viatica</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(In, addition, ,, four, other, species, —, Fri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>26566</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(The, most, abundant, species, was, Cryptopygu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>27189</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>viatica</td>\n",
       "      <td></td>\n",
       "      <td>(Hypogastrura, viatica, and, Mesaphorura, macr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Prasiola</td>\n",
       "      <td>Prasiola</td>\n",
       "      <td>34145</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>crispa</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(On, Barrientos, Island, ,, environments, domi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>34337</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td></td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, has, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>35339</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td></td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, Was, v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>35622</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>badasa</td>\n",
       "      <td>NORP</td>\n",
       "      <td>(For, example, ,, Cryptopygus, badasa, ,, the,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>35673</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>pobj</td>\n",
       "      <td>present</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(For, example, ,, Cryptopygus, badasa, ,, the,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>Friesea</td>\n",
       "      <td>36268</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>is</td>\n",
       "      <td>PRODUCT</td>\n",
       "      <td>(The, genus, Friesea, is, a, perfect, example,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>friesea</td>\n",
       "      <td>37331</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>compound</td>\n",
       "      <td>bispinosa</td>\n",
       "      <td></td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>Archisotoma</td>\n",
       "      <td>archisotoma</td>\n",
       "      <td>39242</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>brucei</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>(Archisotoma, brucei, is, usually, classified,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>40793</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>viatica</td>\n",
       "      <td></td>\n",
       "      <td>(—, Hypogastrura, viatica, and, Mesaphorura, m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>44560</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td></td>\n",
       "      <td>(1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Gomphiocephalus</td>\n",
       "      <td>Gomphiocephalus</td>\n",
       "      <td>45883</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>hodgsoni</td>\n",
       "      <td></td>\n",
       "      <td>(Analysis, of, gut, contents, of, Gomphiocepha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>46164</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>ORG</td>\n",
       "      <td>(Response, of, plants, and, the, dominant, mic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>48467</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>pobj</td>\n",
       "      <td>and</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(org/10.1673/031.012.11101, \\n, Greenslade, P,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>Antarctophorus</td>\n",
       "      <td>Antarctophorus</td>\n",
       "      <td>52334</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>—</td>\n",
       "      <td></td>\n",
       "      <td>(Antarctophorus, —, a, new, genus, of, Isotomi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>Friesea</td>\n",
       "      <td>friesea</td>\n",
       "      <td>54876</td>\n",
       "      <td>ADV</td>\n",
       "      <td>RB</td>\n",
       "      <td>compound</td>\n",
       "      <td>woyciechowskii</td>\n",
       "      <td></td>\n",
       "      <td>((, 1980, ), Friesea, woyciechowskii)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>66 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              Token            Lemma  PositionofWord PartOfSpeech  \\\n",
       "0       Cryptopygus      Cryptopygus            1330        PROPN   \n",
       "1      Hypogastrura     Hypogastrura            1459        PROPN   \n",
       "2          Sanionia         Sanionia            7748        PROPN   \n",
       "3          Sanionia         Sanionia            7780        PROPN   \n",
       "4          Sanionia         Sanionia            7880        PROPN   \n",
       "5   Polytrichastrum  Polytrichastrum            7955        PROPN   \n",
       "6             Bryum            Bryum            8000        PROPN   \n",
       "7          Sanionia         sanionia            8101          ADJ   \n",
       "8       Warnstorfia      Warnstorfia            8147        PROPN   \n",
       "9         Xanthoria        Xanthoria            8286        PROPN   \n",
       "10        Caloplaca        Caloplaca            8304        PROPN   \n",
       "11         Prasiola         prasiola            8367         NOUN   \n",
       "12       Pygoscelis       Pygoscelis            8523        PROPN   \n",
       "13       Pygoscelis       Pygoscelis            8578        PROPN   \n",
       "14      Macronectes      Macronectes            8643        PROPN   \n",
       "15            Larus            Larus            8694        PROPN   \n",
       "16         Mirounga         Mirounga            8783        PROPN   \n",
       "17    Leptonychotes    Leptonychotes            8834        PROPN   \n",
       "18    Arctocephalus    arctocephalus            8900         NOUN   \n",
       "19     Hypogastrura     Hypogastrura           11720        PROPN   \n",
       "20          Friesea          Friesea           15446        PROPN   \n",
       "21          Friesea          Friesea           15481        PROPN   \n",
       "22          Friesea          Friesea           15514        PROPN   \n",
       "23       Tullbergia       Tullbergia           15551        PROPN   \n",
       "24      Archisotoma      Archisotoma           15584        PROPN   \n",
       "25      Cryptopygus      Cryptopygus           15622        PROPN   \n",
       "26      Cryptopygus      Cryptopygus           15672        PROPN   \n",
       "27     Hypogastrura     Hypogastrura           15799        PROPN   \n",
       "28      Cryptopygus      Cryptopygus           18170        PROPN   \n",
       "29      Cryptopygus      Cryptopygus           18244        PROPN   \n",
       "..              ...              ...             ...          ...   \n",
       "36     Hypogastrura     Hypogastrura           18623        PROPN   \n",
       "37          Friesea          friesea           19140          ADJ   \n",
       "38          Friesea          friesea           19351          ADJ   \n",
       "39       Tullbergia       Tullbergia           20186        PROPN   \n",
       "40      Archisotoma      Archisotoma           20508        PROPN   \n",
       "41      Cryptopygus      Cryptopygus           20688        PROPN   \n",
       "42         Prasiola         prasiola           20940         NOUN   \n",
       "43      Cryptopygus      Cryptopygus           21523        PROPN   \n",
       "44          Friesea          friesea           26262          ADJ   \n",
       "45       Tullbergia       Tullbergia           26326        PROPN   \n",
       "46          Friesea          friesea           26376          ADJ   \n",
       "47      Archisotoma      Archisotoma           26395        PROPN   \n",
       "48     Hypogastrura     Hypogastrura           26415        PROPN   \n",
       "49      Cryptopygus      Cryptopygus           26566        PROPN   \n",
       "50     Hypogastrura     Hypogastrura           27189        PROPN   \n",
       "51         Prasiola         Prasiola           34145        PROPN   \n",
       "52      Cryptopygus      Cryptopygus           34337        PROPN   \n",
       "53      Cryptopygus      Cryptopygus           35339        PROPN   \n",
       "54      Cryptopygus      Cryptopygus           35622        PROPN   \n",
       "55      Cryptopygus      Cryptopygus           35673        PROPN   \n",
       "56          Friesea          Friesea           36268        PROPN   \n",
       "57          Friesea          friesea           37331          ADJ   \n",
       "58      Archisotoma      archisotoma           39242         NOUN   \n",
       "59     Hypogastrura     Hypogastrura           40793        PROPN   \n",
       "60      Cryptopygus      Cryptopygus           44560        PROPN   \n",
       "61  Gomphiocephalus  Gomphiocephalus           45883        PROPN   \n",
       "62      Cryptopygus      Cryptopygus           46164        PROPN   \n",
       "63     Hypogastrura     Hypogastrura           48467        PROPN   \n",
       "64   Antarctophorus   Antarctophorus           52334        PROPN   \n",
       "65          Friesea          friesea           54876          ADV   \n",
       "\n",
       "   PartOfSpeech_Fine Dependency    FollowingToken NamedEntityType  \\\n",
       "0                NNP   compound       antarcticus             ORG   \n",
       "1                NNP   compound           viatica                   \n",
       "2                NNP       pobj                as          PERSON   \n",
       "3                NNP   compound          uncinata                   \n",
       "4                NNP       amod          uncinata                   \n",
       "5                NNP   compound           alpinum     WORK_OF_ART   \n",
       "6                NNP   compound  pseudotriquetrum             ORG   \n",
       "7                 JJ   compound  georgicouncinata          PERSON   \n",
       "8                NNP   compound    fontinaliopsis          PERSON   \n",
       "9                NNP      nsubj               spp          PERSON   \n",
       "10               NNP      nsubj               spp          PERSON   \n",
       "11                NN   compound            crispa         PRODUCT   \n",
       "12               NNP   compound             papua            NORP   \n",
       "13               NNP   compound       antarcticus             ORG   \n",
       "14              NNPS   compound         giganteus            NORP   \n",
       "15               NNP   compound       dominicanus             ORG   \n",
       "16               NNP   compound           leonina             ORG   \n",
       "17              NNPS      appos         weddellii             GPE   \n",
       "18                NN   compound           gazella                   \n",
       "19               NNP   compound           viatica             GPE   \n",
       "20               NNP   compound         bispinosa             ORG   \n",
       "21               NNP   compound            grisea             ORG   \n",
       "22               NNP      nsubj    woyciechowskii             ORG   \n",
       "23               NNP   compound             mixta             GPE   \n",
       "24               NNP   compound            brucei             ORG   \n",
       "25               NNP   compound       antarcticus             ORG   \n",
       "26               NNP   compound            badasa            NORP   \n",
       "27               NNP   compound           viatica             GPE   \n",
       "28               NNP   compound       antarcticus             ORG   \n",
       "29               NNP   compound            badasa            NORP   \n",
       "..               ...        ...               ...             ...   \n",
       "36               NNP   compound           viatica             GPE   \n",
       "37                JJ   compound         bispinosa                   \n",
       "38                JJ   compound            grisea                   \n",
       "39               NNP   compound             mixta             GPE   \n",
       "40               NNP   compound            brucei          PERSON   \n",
       "41               NNP   compound       antarcticus                   \n",
       "42                NN       nmod            crispa             GPE   \n",
       "43               NNP   compound            badasa                   \n",
       "44                JJ   compound            grisea                   \n",
       "45               NNP   compound             mixta             GPE   \n",
       "46                JJ   compound         bispinosa                   \n",
       "47               NNP   compound            brucei             ORG   \n",
       "48               NNP   compound           viatica             GPE   \n",
       "49               NNP   compound       antarcticus             ORG   \n",
       "50               NNP   compound           viatica                   \n",
       "51               NNP   compound            crispa             GPE   \n",
       "52               NNP   compound       antarcticus                   \n",
       "53               NNP   compound       antarcticus                   \n",
       "54               NNP   compound            badasa            NORP   \n",
       "55               NNP       pobj           present             ORG   \n",
       "56               NNP      nsubj                is         PRODUCT   \n",
       "57                JJ   compound         bispinosa                   \n",
       "58                NN   compound            brucei          PERSON   \n",
       "59               NNP   compound           viatica                   \n",
       "60               NNP   compound       antarcticus                   \n",
       "61               NNP   compound          hodgsoni                   \n",
       "62               NNP   compound       antarcticus             ORG   \n",
       "63               NNP       pobj               and             GPE   \n",
       "64               NNP      nsubj                 —                   \n",
       "65                RB   compound    woyciechowskii                   \n",
       "\n",
       "                                             Sentence  \n",
       "0   (The, native, species, Cryptopygus, antarcticu...  \n",
       "1             (Hypogastrura, viatica, (, Tullberg, ))  \n",
       "2   (Extensive, moss, carpets, cover, the, center,...  \n",
       "3                 (Sanionia, uncinata, (, Hedw, ., ))  \n",
       "4   (Sanionia, uncinata, forms, monospecific, comm...  \n",
       "5   (Sanionia, uncinata, forms, monospecific, comm...  \n",
       "6   (G.L., Sm, ., and, Bryum, pseudotriquetrum, (,...  \n",
       "7   (Sanionia, georgicouncinata, is, often, found,...  \n",
       "8   (Sanionia, georgicouncinata, is, often, found,...  \n",
       "9   (Different, crustose, lichen, species, ,, such...  \n",
       "10  (Different, crustose, lichen, species, ,, such...  \n",
       "11  (The, green, alga, Prasiola, crispa, (, Lightf...  \n",
       "12  (The, local, fauna, is, diverse, and, includes...  \n",
       "13  (The, local, fauna, is, diverse, and, includes...  \n",
       "14  (The, local, fauna, is, diverse, and, includes...  \n",
       "15  (The, local, fauna, is, diverse, and, includes...  \n",
       "16  (The, local, fauna, is, diverse, and, includes...  \n",
       "17  (The, local, fauna, is, diverse, and, includes...  \n",
       "18  (The, local, fauna, is, diverse, and, includes...  \n",
       "19  (The, cosmopolitan, species, Hypogastrura, via...  \n",
       "20  (Eight, species, are, native, in, Antarctica, ...  \n",
       "21  (Eight, species, are, native, in, Antarctica, ...  \n",
       "22  (Eight, species, are, native, in, Antarctica, ...  \n",
       "23  (Eight, species, are, native, in, Antarctica, ...  \n",
       "24  (Eight, species, are, native, in, Antarctica, ...  \n",
       "25  (Eight, species, are, native, in, Antarctica, ...  \n",
       "26  (Eight, species, are, native, in, Antarctica, ...  \n",
       "27  (Two, species, are, non, -, indigenous, in, An...  \n",
       "28  (These, species, are, Cryptopygus, antarcticus...  \n",
       "29  (These, species, are, Cryptopygus, antarcticus...  \n",
       "..                                                ...  \n",
       "36  (Hypogastrura, viatica, was, the, only, specie...  \n",
       "37  (Friesea, bispinosa, ,, F., grisea, ,, and, F....  \n",
       "38  (Friesea, grisea, can, be, considered, as, eur...  \n",
       "39  (The, family, Tullbergiidae, was, represented,...  \n",
       "40  (Archisotoma, brucei, only, was, found, in, tw...  \n",
       "41  (Cryptopygus, antarcticus, antarcticus, was, f...  \n",
       "42  (This, species, presents, the, highest, densit...  \n",
       "43  (Cryptopygus, badasa, also, appeared, in, a, h...  \n",
       "44  ((, 2014, ), for, the, maritime, Antarctica, r...  \n",
       "45  ((, 2014, ), for, the, maritime, Antarctica, r...  \n",
       "46  (In, addition, ,, four, other, species, —, Fri...  \n",
       "47  (In, addition, ,, four, other, species, —, Fri...  \n",
       "48  (In, addition, ,, four, other, species, —, Fri...  \n",
       "49  (The, most, abundant, species, was, Cryptopygu...  \n",
       "50  (Hypogastrura, viatica, and, Mesaphorura, macr...  \n",
       "51  (On, Barrientos, Island, ,, environments, domi...  \n",
       "52  (Cryptopygus, antarcticus, antarcticus, has, a...  \n",
       "53  (Cryptopygus, antarcticus, antarcticus, Was, v...  \n",
       "54  (For, example, ,, Cryptopygus, badasa, ,, the,...  \n",
       "55  (For, example, ,, Cryptopygus, badasa, ,, the,...  \n",
       "56  (The, genus, Friesea, is, a, perfect, example,...  \n",
       "57  (Friesea, bispinosa, and, F., woyciechowskii, ...  \n",
       "58  (Archisotoma, brucei, is, usually, classified,...  \n",
       "59  (—, Hypogastrura, viatica, and, Mesaphorura, m...  \n",
       "60  (1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...  \n",
       "61  (Analysis, of, gut, contents, of, Gomphiocepha...  \n",
       "62  (Response, of, plants, and, the, dominant, mic...  \n",
       "63  (org/10.1673/031.012.11101, \\n, Greenslade, P,...  \n",
       "64  (Antarctophorus, —, a, new, genus, of, Isotomi...  \n",
       "65              ((, 1980, ), Friesea, woyciechowskii)  \n",
       "\n",
       "[66 rows x 9 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genusInDoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T23:31:46.739127Z",
     "start_time": "2021-01-26T23:31:46.659301Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Token</th>\n",
       "      <th>Lemma</th>\n",
       "      <th>PositionofWord</th>\n",
       "      <th>PartOfSpeech</th>\n",
       "      <th>PartOfSpeech_Fine</th>\n",
       "      <th>Dependency</th>\n",
       "      <th>PreviousToken</th>\n",
       "      <th>PreviousToken_Shape</th>\n",
       "      <th>FollowingToken</th>\n",
       "      <th>NamedEntityType</th>\n",
       "      <th>Sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>1342</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td></td>\n",
       "      <td>(The, native, species, Cryptopygus, antarcticu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>1354</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>xxxx</td>\n",
       "      <td>Willem</td>\n",
       "      <td></td>\n",
       "      <td>(The, native, species, Cryptopygus, antarcticu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>viatica</td>\n",
       "      <td>viatica</td>\n",
       "      <td>1472</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>ROOT</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(Hypogastrura, viatica, (, Tullberg, ))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>1525</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>conj</td>\n",
       "      <td>Mesaphorura</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>Rusek</td>\n",
       "      <td></td>\n",
       "      <td>((, Hypogastruridae, ), and, Mesaphorura, macr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>uncinata</td>\n",
       "      <td>uncinata</td>\n",
       "      <td>7789</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>ROOT</td>\n",
       "      <td>Sanionia</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(Sanionia, uncinata, (, Hedw, ., ))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>uncinata</td>\n",
       "      <td>uncinata</td>\n",
       "      <td>7889</td>\n",
       "      <td>ADJ</td>\n",
       "      <td>JJ</td>\n",
       "      <td>amod</td>\n",
       "      <td>Sanionia</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>forms</td>\n",
       "      <td></td>\n",
       "      <td>(Sanionia, uncinata, forms, monospecific, comm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>alpinum</td>\n",
       "      <td>alpinum</td>\n",
       "      <td>7971</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>pobj</td>\n",
       "      <td>Polytrichastrum</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(Sanionia, uncinata, forms, monospecific, comm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>pseudotriquetrum</td>\n",
       "      <td>pseudotriquetrum</td>\n",
       "      <td>8006</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>conj</td>\n",
       "      <td>Bryum</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(G.L., Sm, ., and, Bryum, pseudotriquetrum, (,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>fontinaliopsis</td>\n",
       "      <td>fontinaliopsis</td>\n",
       "      <td>8159</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>pobj</td>\n",
       "      <td>Warnstorfia</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(Sanionia, georgicouncinata, is, often, found,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>crispa</td>\n",
       "      <td>crispa</td>\n",
       "      <td>8376</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>appos</td>\n",
       "      <td>Prasiola</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(The, green, alga, Prasiola, crispa, (, Lightf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>papua</td>\n",
       "      <td>papua</td>\n",
       "      <td>8534</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>parataxis</td>\n",
       "      <td>Pygoscelis</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>8589</td>\n",
       "      <td>PROPN</td>\n",
       "      <td>NNP</td>\n",
       "      <td>appos</td>\n",
       "      <td>Pygoscelis</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>giganteus</td>\n",
       "      <td>giganteus</td>\n",
       "      <td>8655</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>appos</td>\n",
       "      <td>Macronectes</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>dominicanus</td>\n",
       "      <td>dominicanus</td>\n",
       "      <td>8700</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Larus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>Lichtenstein</td>\n",
       "      <td></td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>leonina</td>\n",
       "      <td>leonina</td>\n",
       "      <td>8792</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>parataxis</td>\n",
       "      <td>Mirounga</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>weddellii</td>\n",
       "      <td>weddellii</td>\n",
       "      <td>8848</td>\n",
       "      <td>VERB</td>\n",
       "      <td>VBP</td>\n",
       "      <td>advmod</td>\n",
       "      <td>Leptonychotes</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>gazella</td>\n",
       "      <td>gazella</td>\n",
       "      <td>8914</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>parataxis</td>\n",
       "      <td>Arctocephalus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>viatica</td>\n",
       "      <td>viatica</td>\n",
       "      <td>11733</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(The, cosmopolitan, species, Hypogastrura, via...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>11774</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Mesaphorura</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>Rusek</td>\n",
       "      <td></td>\n",
       "      <td>(The, cosmopolitan, species, Hypogastrura, via...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>grisea</td>\n",
       "      <td>grisea</td>\n",
       "      <td>15489</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>conj</td>\n",
       "      <td>Friesea</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>woyciechowskii</td>\n",
       "      <td>woyciechowskii</td>\n",
       "      <td>15522</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Friesea</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>Weiner</td>\n",
       "      <td></td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>mixta</td>\n",
       "      <td>mixta</td>\n",
       "      <td>15562</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Tullbergia</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>Wahlgren</td>\n",
       "      <td></td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>brucei</td>\n",
       "      <td>brucei</td>\n",
       "      <td>15596</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>conj</td>\n",
       "      <td>Archisotoma</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>15634</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td></td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>15646</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>xxxx</td>\n",
       "      <td>Willem</td>\n",
       "      <td></td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>badasa</td>\n",
       "      <td>badasa</td>\n",
       "      <td>15684</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>Greenslade</td>\n",
       "      <td></td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>octooculata</td>\n",
       "      <td>octooculata</td>\n",
       "      <td>15725</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>conj</td>\n",
       "      <td>Folsomotoma</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>viatica</td>\n",
       "      <td>viatica</td>\n",
       "      <td>15812</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>conj</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>and</td>\n",
       "      <td></td>\n",
       "      <td>(Two, species, are, non, -, indigenous, in, An...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>15836</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>conj</td>\n",
       "      <td>Mesaphorura</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>(</td>\n",
       "      <td></td>\n",
       "      <td>(Two, species, are, non, -, indigenous, in, An...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>grisea</td>\n",
       "      <td>grisea</td>\n",
       "      <td>16360</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>appos</td>\n",
       "      <td>F.</td>\n",
       "      <td>X.</td>\n",
       "      <td>and</td>\n",
       "      <td></td>\n",
       "      <td>(In, 2012, campaign, the, number, of, sampling...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>viatica</td>\n",
       "      <td>viatica</td>\n",
       "      <td>27520</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>appos</td>\n",
       "      <td>H.</td>\n",
       "      <td>X.</td>\n",
       "      <td>,</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(The, first, species, ,, H., viatica, ,, is, w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>28002</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>dobj</td>\n",
       "      <td>a.</td>\n",
       "      <td>x.</td>\n",
       "      <td>on</td>\n",
       "      <td></td>\n",
       "      <td>(Its, presence, could, affect, the, native, Co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>viatica</td>\n",
       "      <td>viatica</td>\n",
       "      <td>28095</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubjpass</td>\n",
       "      <td>H.</td>\n",
       "      <td>X.</td>\n",
       "      <td>was</td>\n",
       "      <td></td>\n",
       "      <td>(On, Barrientos, Island, ,, H., viatica, was, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>28178</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>M.</td>\n",
       "      <td>X.</td>\n",
       "      <td>was</td>\n",
       "      <td></td>\n",
       "      <td>(On, Barrientos, Island, ,, H., viatica, was, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>viatica</td>\n",
       "      <td>viatica</td>\n",
       "      <td>28244</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>pobj</td>\n",
       "      <td>H.</td>\n",
       "      <td>X.</td>\n",
       "      <td>could</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(The, presence, of, H., viatica, could, be, a,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>28414</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>pobj</td>\n",
       "      <td>M.</td>\n",
       "      <td>X.</td>\n",
       "      <td>may</td>\n",
       "      <td></td>\n",
       "      <td>(The, presence, of, M., macrochaeta, may, be, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>viatica</td>\n",
       "      <td>viatica</td>\n",
       "      <td>29656</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>attr</td>\n",
       "      <td>H.</td>\n",
       "      <td>X.</td>\n",
       "      <td>found</td>\n",
       "      <td>GPE</td>\n",
       "      <td>(The, first, non, -, indigenous, invertebrate,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>30184</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubjpass</td>\n",
       "      <td>M.</td>\n",
       "      <td>X.</td>\n",
       "      <td>was</td>\n",
       "      <td></td>\n",
       "      <td>(The, species, M., macrochaeta, was, reported,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>crispa</td>\n",
       "      <td>crispa</td>\n",
       "      <td>34154</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>pobj</td>\n",
       "      <td>Prasiola</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>,</td>\n",
       "      <td></td>\n",
       "      <td>(On, Barrientos, Island, ,, environments, domi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>34349</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td></td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, has, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>34361</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>xxxx</td>\n",
       "      <td>has</td>\n",
       "      <td></td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, has, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>crispa</td>\n",
       "      <td>crispa</td>\n",
       "      <td>34671</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>pobj</td>\n",
       "      <td>P.</td>\n",
       "      <td>X.</td>\n",
       "      <td>is</td>\n",
       "      <td></td>\n",
       "      <td>(Behind, its, predilection, for, P., crispa, i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>34973</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>pobj</td>\n",
       "      <td>a.</td>\n",
       "      <td>x.</td>\n",
       "      <td>when</td>\n",
       "      <td></td>\n",
       "      <td>(Therefore, ,, mosses, are, less, consumed, by...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>35351</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td></td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, Was, v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>35363</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>xxxx</td>\n",
       "      <td>Was</td>\n",
       "      <td></td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, Was, v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>badasa</td>\n",
       "      <td>badasa</td>\n",
       "      <td>35634</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>,</td>\n",
       "      <td></td>\n",
       "      <td>(For, example, ,, Cryptopygus, badasa, ,, the,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>grisea</td>\n",
       "      <td>grisea</td>\n",
       "      <td>36456</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>attr</td>\n",
       "      <td>F.</td>\n",
       "      <td>X.</td>\n",
       "      <td>.</td>\n",
       "      <td></td>\n",
       "      <td>(On, Barrientos, Island, ,, the, species, with...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>grisea</td>\n",
       "      <td>grisea</td>\n",
       "      <td>37037</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>F.</td>\n",
       "      <td>X.</td>\n",
       "      <td>abundance</td>\n",
       "      <td></td>\n",
       "      <td>(Therefore, ,, it, is, not, surprising, that, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>grisea</td>\n",
       "      <td>grisea</td>\n",
       "      <td>37227</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubjpass</td>\n",
       "      <td>F.</td>\n",
       "      <td>X.</td>\n",
       "      <td>was</td>\n",
       "      <td></td>\n",
       "      <td>(This, indicates, that, F., grisea, was, widel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>woyciechowskii</td>\n",
       "      <td>woyciechowskii</td>\n",
       "      <td>37356</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>F.</td>\n",
       "      <td>X.</td>\n",
       "      <td>,</td>\n",
       "      <td></td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>grisea</td>\n",
       "      <td>grisea</td>\n",
       "      <td>37436</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>pobj</td>\n",
       "      <td>F.</td>\n",
       "      <td>X.</td>\n",
       "      <td>,</td>\n",
       "      <td></td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>octooculata</td>\n",
       "      <td>octooculata</td>\n",
       "      <td>37729</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>Folsomotoma</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>is</td>\n",
       "      <td></td>\n",
       "      <td>(Folsomotoma, octooculata, is, an, epedaphic, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>brucei</td>\n",
       "      <td>brucei</td>\n",
       "      <td>39254</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubjpass</td>\n",
       "      <td>Archisotoma</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>is</td>\n",
       "      <td></td>\n",
       "      <td>(Archisotoma, brucei, is, usually, classified,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>brucei</td>\n",
       "      <td>brucei</td>\n",
       "      <td>39711</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>pobj</td>\n",
       "      <td>A.</td>\n",
       "      <td>X.</td>\n",
       "      <td>at</td>\n",
       "      <td></td>\n",
       "      <td>(While, this, is, an, unusual, location, for, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>viatica</td>\n",
       "      <td>viatica</td>\n",
       "      <td>40806</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>Hypogastrura</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>and</td>\n",
       "      <td></td>\n",
       "      <td>(—, Hypogastrura, viatica, and, Mesaphorura, m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>macrochaeta</td>\n",
       "      <td>40830</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Mesaphorura</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>—</td>\n",
       "      <td></td>\n",
       "      <td>(—, Hypogastrura, viatica, and, Mesaphorura, m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>44572</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>Willem</td>\n",
       "      <td></td>\n",
       "      <td>(1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>hodgsoni</td>\n",
       "      <td>hodgsoni</td>\n",
       "      <td>45899</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>compound</td>\n",
       "      <td>Gomphiocephalus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>carpenter</td>\n",
       "      <td></td>\n",
       "      <td>(Analysis, of, gut, contents, of, Gomphiocepha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>antarcticus</td>\n",
       "      <td>antarcticus</td>\n",
       "      <td>46176</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>appos</td>\n",
       "      <td>Cryptopygus</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>,</td>\n",
       "      <td></td>\n",
       "      <td>(Response, of, plants, and, the, dominant, mic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>woyciechowskii</td>\n",
       "      <td>woyciechowskii</td>\n",
       "      <td>54884</td>\n",
       "      <td>NOUN</td>\n",
       "      <td>NN</td>\n",
       "      <td>ROOT</td>\n",
       "      <td>Friesea</td>\n",
       "      <td>Xxxxx</td>\n",
       "      <td>sp</td>\n",
       "      <td></td>\n",
       "      <td>((, 1980, ), Friesea, woyciechowskii)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>111 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                Token             Lemma  PositionofWord PartOfSpeech  \\\n",
       "0         antarcticus       antarcticus            1342         NOUN   \n",
       "1         antarcticus       antarcticus            1354         NOUN   \n",
       "2             viatica           viatica            1472         NOUN   \n",
       "3         macrochaeta       macrochaeta            1525         NOUN   \n",
       "4            uncinata          uncinata            7789         NOUN   \n",
       "5            uncinata          uncinata            7889          ADJ   \n",
       "6             alpinum           alpinum            7971         NOUN   \n",
       "7    pseudotriquetrum  pseudotriquetrum            8006         NOUN   \n",
       "8      fontinaliopsis    fontinaliopsis            8159         NOUN   \n",
       "9              crispa            crispa            8376         NOUN   \n",
       "10              papua             papua            8534         NOUN   \n",
       "11        antarcticus       antarcticus            8589        PROPN   \n",
       "12          giganteus         giganteus            8655         NOUN   \n",
       "13        dominicanus       dominicanus            8700         NOUN   \n",
       "14            leonina           leonina            8792         NOUN   \n",
       "15          weddellii         weddellii            8848         VERB   \n",
       "16            gazella           gazella            8914         NOUN   \n",
       "17            viatica           viatica           11733         NOUN   \n",
       "18        macrochaeta       macrochaeta           11774         NOUN   \n",
       "19             grisea            grisea           15489         NOUN   \n",
       "20     woyciechowskii    woyciechowskii           15522         NOUN   \n",
       "21              mixta             mixta           15562         NOUN   \n",
       "22             brucei            brucei           15596         NOUN   \n",
       "23        antarcticus       antarcticus           15634         NOUN   \n",
       "24        antarcticus       antarcticus           15646         NOUN   \n",
       "25             badasa            badasa           15684         NOUN   \n",
       "26        octooculata       octooculata           15725         NOUN   \n",
       "27            viatica           viatica           15812         NOUN   \n",
       "28        macrochaeta       macrochaeta           15836         NOUN   \n",
       "29             grisea            grisea           16360         NOUN   \n",
       "..                ...               ...             ...          ...   \n",
       "81            viatica           viatica           27520         NOUN   \n",
       "82        antarcticus       antarcticus           28002         NOUN   \n",
       "83            viatica           viatica           28095         NOUN   \n",
       "84        macrochaeta       macrochaeta           28178         NOUN   \n",
       "85            viatica           viatica           28244         NOUN   \n",
       "86        macrochaeta       macrochaeta           28414         NOUN   \n",
       "87            viatica           viatica           29656         NOUN   \n",
       "88        macrochaeta       macrochaeta           30184         NOUN   \n",
       "89             crispa            crispa           34154         NOUN   \n",
       "90        antarcticus       antarcticus           34349         NOUN   \n",
       "91        antarcticus       antarcticus           34361         NOUN   \n",
       "92             crispa            crispa           34671         NOUN   \n",
       "93        antarcticus       antarcticus           34973         NOUN   \n",
       "94        antarcticus       antarcticus           35351         NOUN   \n",
       "95        antarcticus       antarcticus           35363         NOUN   \n",
       "96             badasa            badasa           35634         NOUN   \n",
       "97             grisea            grisea           36456         NOUN   \n",
       "98             grisea            grisea           37037         NOUN   \n",
       "99             grisea            grisea           37227         NOUN   \n",
       "100    woyciechowskii    woyciechowskii           37356         NOUN   \n",
       "101            grisea            grisea           37436         NOUN   \n",
       "102       octooculata       octooculata           37729         NOUN   \n",
       "103            brucei            brucei           39254         NOUN   \n",
       "104            brucei            brucei           39711         NOUN   \n",
       "105           viatica           viatica           40806         NOUN   \n",
       "106       macrochaeta       macrochaeta           40830         NOUN   \n",
       "107       antarcticus       antarcticus           44572         NOUN   \n",
       "108          hodgsoni          hodgsoni           45899         NOUN   \n",
       "109       antarcticus       antarcticus           46176         NOUN   \n",
       "110    woyciechowskii    woyciechowskii           54884         NOUN   \n",
       "\n",
       "    PartOfSpeech_Fine Dependency    PreviousToken PreviousToken_Shape  \\\n",
       "0                  NN   compound      Cryptopygus               Xxxxx   \n",
       "1                  NN   compound      antarcticus                xxxx   \n",
       "2                  NN       ROOT     Hypogastrura               Xxxxx   \n",
       "3                  NN       conj      Mesaphorura               Xxxxx   \n",
       "4                  NN       ROOT         Sanionia               Xxxxx   \n",
       "5                  JJ       amod         Sanionia               Xxxxx   \n",
       "6                  NN       pobj  Polytrichastrum               Xxxxx   \n",
       "7                  NN       conj            Bryum               Xxxxx   \n",
       "8                  NN       pobj      Warnstorfia               Xxxxx   \n",
       "9                  NN      appos         Prasiola               Xxxxx   \n",
       "10                 NN  parataxis       Pygoscelis               Xxxxx   \n",
       "11                NNP      appos       Pygoscelis               Xxxxx   \n",
       "12                 NN      appos      Macronectes               Xxxxx   \n",
       "13                 NN   compound            Larus               Xxxxx   \n",
       "14                 NN  parataxis         Mirounga               Xxxxx   \n",
       "15                VBP     advmod    Leptonychotes               Xxxxx   \n",
       "16                 NN  parataxis    Arctocephalus               Xxxxx   \n",
       "17                 NN      nsubj     Hypogastrura               Xxxxx   \n",
       "18                 NN   compound      Mesaphorura               Xxxxx   \n",
       "19                 NN       conj          Friesea               Xxxxx   \n",
       "20                 NN   compound          Friesea               Xxxxx   \n",
       "21                 NN   compound       Tullbergia               Xxxxx   \n",
       "22                 NN       conj      Archisotoma               Xxxxx   \n",
       "23                 NN   compound      Cryptopygus               Xxxxx   \n",
       "24                 NN   compound      antarcticus                xxxx   \n",
       "25                 NN   compound      Cryptopygus               Xxxxx   \n",
       "26                 NN       conj      Folsomotoma               Xxxxx   \n",
       "27                 NN       conj     Hypogastrura               Xxxxx   \n",
       "28                 NN       conj      Mesaphorura               Xxxxx   \n",
       "29                 NN      appos               F.                  X.   \n",
       "..                ...        ...              ...                 ...   \n",
       "81                 NN      appos               H.                  X.   \n",
       "82                 NN       dobj               a.                  x.   \n",
       "83                 NN  nsubjpass               H.                  X.   \n",
       "84                 NN      nsubj               M.                  X.   \n",
       "85                 NN       pobj               H.                  X.   \n",
       "86                 NN       pobj               M.                  X.   \n",
       "87                 NN       attr               H.                  X.   \n",
       "88                 NN  nsubjpass               M.                  X.   \n",
       "89                 NN       pobj         Prasiola               Xxxxx   \n",
       "90                 NN   compound      Cryptopygus               Xxxxx   \n",
       "91                 NN      nsubj      antarcticus                xxxx   \n",
       "92                 NN       pobj               P.                  X.   \n",
       "93                 NN       pobj               a.                  x.   \n",
       "94                 NN   compound      Cryptopygus               Xxxxx   \n",
       "95                 NN      nsubj      antarcticus                xxxx   \n",
       "96                 NN      nsubj      Cryptopygus               Xxxxx   \n",
       "97                 NN       attr               F.                  X.   \n",
       "98                 NN   compound               F.                  X.   \n",
       "99                 NN  nsubjpass               F.                  X.   \n",
       "100                NN      nsubj               F.                  X.   \n",
       "101                NN       pobj               F.                  X.   \n",
       "102                NN      nsubj      Folsomotoma               Xxxxx   \n",
       "103                NN  nsubjpass      Archisotoma               Xxxxx   \n",
       "104                NN       pobj               A.                  X.   \n",
       "105                NN      nsubj     Hypogastrura               Xxxxx   \n",
       "106                NN   compound      Mesaphorura               Xxxxx   \n",
       "107                NN   compound      Cryptopygus               Xxxxx   \n",
       "108                NN   compound  Gomphiocephalus               Xxxxx   \n",
       "109                NN      appos      Cryptopygus               Xxxxx   \n",
       "110                NN       ROOT          Friesea               Xxxxx   \n",
       "\n",
       "    FollowingToken NamedEntityType  \\\n",
       "0      antarcticus                   \n",
       "1           Willem                   \n",
       "2                (                   \n",
       "3            Rusek                   \n",
       "4                (                   \n",
       "5            forms                   \n",
       "6                (                   \n",
       "7                (                   \n",
       "8                (                   \n",
       "9                (                   \n",
       "10               (                   \n",
       "11               (                   \n",
       "12               (                   \n",
       "13    Lichtenstein                   \n",
       "14               (                   \n",
       "15               (                   \n",
       "16               (                   \n",
       "17               (                   \n",
       "18           Rusek                   \n",
       "19               (                   \n",
       "20          Weiner                   \n",
       "21        Wahlgren                   \n",
       "22               (                   \n",
       "23     antarcticus                   \n",
       "24          Willem                   \n",
       "25      Greenslade                   \n",
       "26               (                   \n",
       "27             and                   \n",
       "28               (                   \n",
       "29             and                   \n",
       "..             ...             ...   \n",
       "81               ,             GPE   \n",
       "82              on                   \n",
       "83             was                   \n",
       "84             was                   \n",
       "85           could             GPE   \n",
       "86             may                   \n",
       "87           found             GPE   \n",
       "88             was                   \n",
       "89               ,                   \n",
       "90     antarcticus                   \n",
       "91             has                   \n",
       "92              is                   \n",
       "93            when                   \n",
       "94     antarcticus                   \n",
       "95             Was                   \n",
       "96               ,                   \n",
       "97               .                   \n",
       "98       abundance                   \n",
       "99             was                   \n",
       "100              ,                   \n",
       "101              ,                   \n",
       "102             is                   \n",
       "103             is                   \n",
       "104             at                   \n",
       "105            and                   \n",
       "106              —                   \n",
       "107         Willem                   \n",
       "108      carpenter                   \n",
       "109              ,                   \n",
       "110             sp                   \n",
       "\n",
       "                                              Sentence  \n",
       "0    (The, native, species, Cryptopygus, antarcticu...  \n",
       "1    (The, native, species, Cryptopygus, antarcticu...  \n",
       "2              (Hypogastrura, viatica, (, Tullberg, ))  \n",
       "3    ((, Hypogastruridae, ), and, Mesaphorura, macr...  \n",
       "4                  (Sanionia, uncinata, (, Hedw, ., ))  \n",
       "5    (Sanionia, uncinata, forms, monospecific, comm...  \n",
       "6    (Sanionia, uncinata, forms, monospecific, comm...  \n",
       "7    (G.L., Sm, ., and, Bryum, pseudotriquetrum, (,...  \n",
       "8    (Sanionia, georgicouncinata, is, often, found,...  \n",
       "9    (The, green, alga, Prasiola, crispa, (, Lightf...  \n",
       "10   (The, local, fauna, is, diverse, and, includes...  \n",
       "11   (The, local, fauna, is, diverse, and, includes...  \n",
       "12   (The, local, fauna, is, diverse, and, includes...  \n",
       "13   (The, local, fauna, is, diverse, and, includes...  \n",
       "14   (The, local, fauna, is, diverse, and, includes...  \n",
       "15   (The, local, fauna, is, diverse, and, includes...  \n",
       "16   (The, local, fauna, is, diverse, and, includes...  \n",
       "17   (The, cosmopolitan, species, Hypogastrura, via...  \n",
       "18   (The, cosmopolitan, species, Hypogastrura, via...  \n",
       "19   (Eight, species, are, native, in, Antarctica, ...  \n",
       "20   (Eight, species, are, native, in, Antarctica, ...  \n",
       "21   (Eight, species, are, native, in, Antarctica, ...  \n",
       "22   (Eight, species, are, native, in, Antarctica, ...  \n",
       "23   (Eight, species, are, native, in, Antarctica, ...  \n",
       "24   (Eight, species, are, native, in, Antarctica, ...  \n",
       "25   (Eight, species, are, native, in, Antarctica, ...  \n",
       "26   (Eight, species, are, native, in, Antarctica, ...  \n",
       "27   (Two, species, are, non, -, indigenous, in, An...  \n",
       "28   (Two, species, are, non, -, indigenous, in, An...  \n",
       "29   (In, 2012, campaign, the, number, of, sampling...  \n",
       "..                                                 ...  \n",
       "81   (The, first, species, ,, H., viatica, ,, is, w...  \n",
       "82   (Its, presence, could, affect, the, native, Co...  \n",
       "83   (On, Barrientos, Island, ,, H., viatica, was, ...  \n",
       "84   (On, Barrientos, Island, ,, H., viatica, was, ...  \n",
       "85   (The, presence, of, H., viatica, could, be, a,...  \n",
       "86   (The, presence, of, M., macrochaeta, may, be, ...  \n",
       "87   (The, first, non, -, indigenous, invertebrate,...  \n",
       "88   (The, species, M., macrochaeta, was, reported,...  \n",
       "89   (On, Barrientos, Island, ,, environments, domi...  \n",
       "90   (Cryptopygus, antarcticus, antarcticus, has, a...  \n",
       "91   (Cryptopygus, antarcticus, antarcticus, has, a...  \n",
       "92   (Behind, its, predilection, for, P., crispa, i...  \n",
       "93   (Therefore, ,, mosses, are, less, consumed, by...  \n",
       "94   (Cryptopygus, antarcticus, antarcticus, Was, v...  \n",
       "95   (Cryptopygus, antarcticus, antarcticus, Was, v...  \n",
       "96   (For, example, ,, Cryptopygus, badasa, ,, the,...  \n",
       "97   (On, Barrientos, Island, ,, the, species, with...  \n",
       "98   (Therefore, ,, it, is, not, surprising, that, ...  \n",
       "99   (This, indicates, that, F., grisea, was, widel...  \n",
       "100  (Friesea, bispinosa, and, F., woyciechowskii, ...  \n",
       "101  (Friesea, bispinosa, and, F., woyciechowskii, ...  \n",
       "102  (Folsomotoma, octooculata, is, an, epedaphic, ...  \n",
       "103  (Archisotoma, brucei, is, usually, classified,...  \n",
       "104  (While, this, is, an, unusual, location, for, ...  \n",
       "105  (—, Hypogastrura, viatica, and, Mesaphorura, m...  \n",
       "106  (—, Hypogastrura, viatica, and, Mesaphorura, m...  \n",
       "107  (1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...  \n",
       "108  (Analysis, of, gut, contents, of, Gomphiocepha...  \n",
       "109  (Response, of, plants, and, the, dominant, mic...  \n",
       "110              ((, 1980, ), Friesea, woyciechowskii)  \n",
       "\n",
       "[111 rows x 11 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "speciesInDoc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:11:22.967892Z",
     "start_time": "2021-02-08T03:11:22.965249Z"
    }
   },
   "outputs": [],
   "source": [
    "from spacy.matcher import Matcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-27T01:27:45.101155Z",
     "start_time": "2021-02-27T01:27:43.948481Z"
    }
   },
   "outputs": [],
   "source": [
    "def findSpecies(doc):\n",
    "    \n",
    "    \"\"\"\n",
    "    Dictionary approach.\n",
    "    \n",
    "    This code runs through token by token looking to match Genus + species, \n",
    "    Genus + species + species, G. species etc based on words found in the genus \n",
    "    and species columns of the supplied list of species.\n",
    "    \n",
    "    Also finds examples where a known genus is followed by an unrecognised species \n",
    "    if that example also appears in an abbreviated form elsewhere. \n",
    "    \"\"\"\n",
    "\n",
    "    foundSpeciesList = []\n",
    "    foundSpecies = ''\n",
    "    foundSpeciestokens = []\n",
    "    foundSpeciesDict ={}\n",
    "    possiblesNames =[]\n",
    "    possiblesTokens = []\n",
    "\n",
    "    \n",
    "    for token in doc:\n",
    "        if str(token) in uniqueGenus:\n",
    "            foundSpecies = foundSpecies + str(token)\n",
    "            foundSpeciestokens.append(token)\n",
    "            j=1\n",
    "            onlyGenus = True\n",
    "            while (doc[token.i+j].text in uniqueSpecies):\n",
    "                foundSpecies += ' ' + str(doc[token.i+j])\n",
    "                foundSpeciestokens.append(doc[token.i+j])\n",
    "                j+=1\n",
    "                onlyGenus = False\n",
    "                \n",
    "            key = ''\n",
    "            \n",
    "            if onlyGenus ==False:\n",
    "                for tkn in foundSpeciestokens[:-1]:\n",
    "                    key += tkn.text[0] + '. '\n",
    "                key+= foundSpeciestokens[-1].text\n",
    "                if key not in foundSpeciesDict:\n",
    "                    foundSpeciesDict[key]=[foundSpecies, True]\n",
    "#                     print('adding ', key, ': [', foundSpecies, True,'] to dict' )\n",
    "\n",
    "                foundSpeciesList.append([foundSpecies, foundSpecies, True, token.i, token.i+j, token.idx, token.sent])\n",
    "#                 print('DEBUG 1 - adding ',foundSpecies, foundSpecies, True, token.i, 'to DF')\n",
    "\n",
    "                \n",
    "            else:\n",
    "                if doc[token.i+1].is_alpha:\n",
    "                    candidate = token.text + ' ' + doc[token.i+1].text\n",
    "#                     if candidate not in possiblesNames:\n",
    "                    possiblesNames.append(candidate)\n",
    "                    possiblesTokens.append((token, doc[token.i+1]))\n",
    "#                         if token.i ==1428:\n",
    "#                             print(candidate)\n",
    "                else:\n",
    "                    foundSpeciesList.append([foundSpecies, foundSpecies, True, token.i, token.i+1, token.idx, token.sent])\n",
    "\n",
    "            foundSpecies=''\n",
    "            foundSpeciestokens=[]\n",
    "\n",
    "\n",
    "        elif token.shape_ == 'X.':\n",
    "            foundSpeciestokens=[]\n",
    "            foundSpecies = foundSpecies + str(token)\n",
    "            foundSpeciestokens.append(token)\n",
    "            j=1\n",
    "            while ((doc[token.i+j].is_alpha and doc[token.i+j].text in uniqueSpecies) or doc[token.i+j].shape_ == 'x.'):\n",
    "                foundSpecies += ' ' + str(doc[token.i+j])\n",
    "                foundSpeciestokens.append(doc[token.i+j])\n",
    "                j+=1\n",
    "            if doc[token.i+j-1].text in uniqueSpecies:\n",
    "#                 print(doc[token.i+j-1].text, 'in uniqueSpecies')\n",
    "                if foundSpecies in foundSpeciesDict:\n",
    "                    foundSpeciesList.append([foundSpecies, foundSpeciesDict.get(foundSpecies)[0], foundSpeciesDict.get(foundSpecies)[1], token.i, token.i+j, token.idx, token.sent])\n",
    "#                     print('DEBUG 2 -adding ',foundSpecies, foundSpeciesDict.get(foundSpecies)[0], foundSpeciesDict.get(foundSpecies)[1],token.i, 'to DF')\n",
    "                else:\n",
    "#                     print(\"looking for\", foundSpeciestokens)\n",
    "                    matcher = Matcher(nlp.vocab)\n",
    "                    pattern = [{\"SHAPE\": \"Xxxxx\", \n",
    "                                   \"TEXT\"[0]: foundSpeciestokens[0].text[0]}\n",
    "                              ]\n",
    "                    \n",
    "                    for tkn in foundSpeciestokens[1:-1]:\n",
    "                        pattern.append({\"SHAPE\": \"xxxx\", \n",
    "                                       \"TEXT\"[0]: tkn[0].text[0]})\n",
    "                    pattern.append({\"TEXT\": foundSpeciestokens[-1].text})\n",
    "#                     print(\"looking for\", pattern)                \n",
    "                    matcher.add(\"Species_Pattern\", None, pattern)\n",
    "\n",
    "                    matches = matcher(doc)\n",
    "                    if len(matches)>0:\n",
    "                        start = matches[0][1]\n",
    "                        end =  matches[0][2]\n",
    "                        matched_span = doc[start:end]\n",
    "                        inList = False\n",
    "                        if doc[start].text in uniqueGenus:\n",
    "                            inList = True\n",
    "#                             print(doc[start].text, 'in UniqueGenus')\n",
    "\n",
    "                        foundSpeciesList.append([foundSpecies, matched_span.text, inList, token.i, token.i+j, token.idx, token.sent])\n",
    "#                         print('DEBUG 3 -adding ',foundSpecies, matched_span.text, inList, token.i, 'to DF')\n",
    "\n",
    "                        foundSpeciesDict[foundSpecies]=[matched_span.text, inList]\n",
    "\n",
    "                        if inList==False:\n",
    "                            for match_id, start, end in matches:\n",
    "                                matched_span = doc[start:end]\n",
    "                                foundSpeciesList.append([matched_span.text, matched_span.text, inList, start,end, doc[start].idx, doc[start].sent])\n",
    "#                                 print('DEBUG 4 -adding ',matched_span.text, matched_span.text, inList, start, 'to DF')\n",
    "            foundSpecies=''\n",
    "            foundSpeciestokens=[]\n",
    "                \n",
    "#     print(possiblesTokens)\n",
    "    for tkns in possiblesTokens:\n",
    "        shortPattern = [{\"SHAPE\": \"X.\", \"TEXT\"[0]: tkns[0].text[0]},\n",
    "                           {\"TEXT\": doc[tkns[0].i+1].text}\n",
    "                          ]\n",
    "        fullPattern = [{\"SHAPE\": \"Xxxxx\", \"TEXT\": tkns[0].text},\n",
    "                           {\"TEXT\": doc[tkns[0].i+1].text}\n",
    "                          ]\n",
    "#         print('candidate = ', tkns)\n",
    "#         print(\"looking for\", shortPattern) \n",
    "        matcher = Matcher(nlp.vocab)\n",
    "        matcher.add(\"abbreviated_Species_Pattern\", None, shortPattern)\n",
    "        matches = matcher(doc)\n",
    "        if len(matches)>0:\n",
    "#             print('found abbreviated version of candidate')\n",
    "            matcher.add(\"full_Species_Pattern\", None, fullPattern)\n",
    "            \n",
    "            matches = matcher(doc)\n",
    "            for match_id, start, end in matches:\n",
    "                matched_span = doc[start:end]\n",
    "                foundSpeciesList.append([matched_span.text, tkns[0].text + ' '+ tkns[1].text, False, start, end, doc[start].idx, doc[start].sent])\n",
    "#                 print('DEBUG 5 -adding ',matched_span.text, tkns[0].text + ' '+ tkns[1].text, False ,start, 'to DF')\n",
    "        else:\n",
    "            foundSpeciesList.append([tkns[0].text, tkns[0].text, False, tkns[0].i, tkns[0].i+1, doc[tkns[0].i].idx, doc[tkns[0].i].sent])\n",
    "\n",
    "\n",
    "\n",
    "    FoundSpeciesDF = pd.DataFrame(foundSpeciesList,columns=['Found as', 'Full name', 'InDefinitiveList', 'Start', 'End','PositionOfFirstToken','Sentence'])\n",
    "    FoundSpeciesDF.sort_values('Start', inplace=True)\n",
    "    FoundSpeciesDF.reset_index(inplace=True,drop=True)\n",
    "    FoundSpeciesDF['InReferenceList']= [True if x in uniqueSpeciesStrings else False for x in FoundSpeciesDF['Full name']]\n",
    "#     FoundSpeciesDF['InReferenceList']=FoundSpeciesDF['Full name']\n",
    "    FoundSpeciesDF.drop_duplicates(keep='first', inplace=True)\n",
    "    return FoundSpeciesDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-27T01:06:41.902608Z",
     "start_time": "2021-02-27T01:06:41.888343Z"
    }
   },
   "outputs": [],
   "source": [
    "file = open(\"FraserSet/BodyOnly/Clean/s42003-018-0260-y.txt\")\n",
    "doc = file.read()\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-27T01:06:44.310425Z",
     "start_time": "2021-02-27T01:06:43.488190Z"
    }
   },
   "outputs": [],
   "source": [
    "#using large Spacy model\n",
    "documenttest = nlp_lg(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-27T01:24:01.010679Z",
     "start_time": "2021-02-27T01:24:00.724863Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test = findSpecies(documenttest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-27T01:25:39.578479Z",
     "start_time": "2021-02-27T01:25:39.566220Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{False}"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(list(test.duplicated()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T01:43:12.921895Z",
     "start_time": "2021-02-25T01:43:12.917667Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Diplosphaera"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documenttest[3434]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T23:32:43.161882Z",
     "start_time": "2021-01-26T23:32:42.783274Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Found as</th>\n",
       "      <th>Full name</th>\n",
       "      <th>InDefinitiveList</th>\n",
       "      <th>Start</th>\n",
       "      <th>End</th>\n",
       "      <th>PositionOfFirstToken</th>\n",
       "      <th>Sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>226</td>\n",
       "      <td>229</td>\n",
       "      <td>1330</td>\n",
       "      <td>(The, native, species, Cryptopygus, antarcticu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>247</td>\n",
       "      <td>249</td>\n",
       "      <td>1459</td>\n",
       "      <td>(Hypogastrura, viatica, (, Tullberg, ))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>256</td>\n",
       "      <td>258</td>\n",
       "      <td>1513</td>\n",
       "      <td>((, Hypogastruridae, ), and, Mesaphorura, macr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sanionia uncinata</td>\n",
       "      <td>Sanionia uncinata</td>\n",
       "      <td>True</td>\n",
       "      <td>1453</td>\n",
       "      <td>1455</td>\n",
       "      <td>7780</td>\n",
       "      <td>(Sanionia, uncinata, (, Hedw, ., ))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S. georgicouncinata</td>\n",
       "      <td>Sanionia georgicouncinata</td>\n",
       "      <td>False</td>\n",
       "      <td>1461</td>\n",
       "      <td>1463</td>\n",
       "      <td>7817</td>\n",
       "      <td>(Loeske, and, S., georgicouncinata, (, Mill, .)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Sanionia uncinata</td>\n",
       "      <td>Sanionia uncinata</td>\n",
       "      <td>True</td>\n",
       "      <td>1475</td>\n",
       "      <td>1477</td>\n",
       "      <td>7880</td>\n",
       "      <td>(Sanionia, uncinata, forms, monospecific, comm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Polytrichastrum alpinum</td>\n",
       "      <td>Polytrichastrum alpinum</td>\n",
       "      <td>True</td>\n",
       "      <td>1484</td>\n",
       "      <td>1486</td>\n",
       "      <td>7955</td>\n",
       "      <td>(Sanionia, uncinata, forms, monospecific, comm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Bryum pseudotriquetrum</td>\n",
       "      <td>Bryum pseudotriquetrum</td>\n",
       "      <td>True</td>\n",
       "      <td>1494</td>\n",
       "      <td>1496</td>\n",
       "      <td>8000</td>\n",
       "      <td>(G.L., Sm, ., and, Bryum, pseudotriquetrum, (,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Sanionia georgicouncinata</td>\n",
       "      <td>Sanionia georgicouncinata</td>\n",
       "      <td>False</td>\n",
       "      <td>1517</td>\n",
       "      <td>1519</td>\n",
       "      <td>8101</td>\n",
       "      <td>(Sanionia, georgicouncinata, is, often, found,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Warnstorfia fontinaliopsis</td>\n",
       "      <td>Warnstorfia fontinaliopsis</td>\n",
       "      <td>True</td>\n",
       "      <td>1523</td>\n",
       "      <td>1525</td>\n",
       "      <td>8147</td>\n",
       "      <td>(Sanionia, georgicouncinata, is, often, found,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>True</td>\n",
       "      <td>1566</td>\n",
       "      <td>1568</td>\n",
       "      <td>8367</td>\n",
       "      <td>(The, green, alga, Prasiola, crispa, (, Lightf...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Pygoscelis papua</td>\n",
       "      <td>Pygoscelis papua</td>\n",
       "      <td>True</td>\n",
       "      <td>1592</td>\n",
       "      <td>1594</td>\n",
       "      <td>8523</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Pygoscelis antarcticus</td>\n",
       "      <td>Pygoscelis antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>1604</td>\n",
       "      <td>1606</td>\n",
       "      <td>8578</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Macronectes giganteus</td>\n",
       "      <td>Macronectes giganteus</td>\n",
       "      <td>True</td>\n",
       "      <td>1617</td>\n",
       "      <td>1619</td>\n",
       "      <td>8643</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Larus dominicanus</td>\n",
       "      <td>Larus dominicanus</td>\n",
       "      <td>True</td>\n",
       "      <td>1629</td>\n",
       "      <td>1631</td>\n",
       "      <td>8694</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Mirounga leonina</td>\n",
       "      <td>Mirounga leonina</td>\n",
       "      <td>True</td>\n",
       "      <td>1647</td>\n",
       "      <td>1649</td>\n",
       "      <td>8783</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Leptonychotes weddellii</td>\n",
       "      <td>Leptonychotes weddellii</td>\n",
       "      <td>True</td>\n",
       "      <td>1659</td>\n",
       "      <td>1661</td>\n",
       "      <td>8834</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Arctocephalus gazella</td>\n",
       "      <td>Arctocephalus gazella</td>\n",
       "      <td>True</td>\n",
       "      <td>1673</td>\n",
       "      <td>1675</td>\n",
       "      <td>8900</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>2226</td>\n",
       "      <td>2228</td>\n",
       "      <td>11720</td>\n",
       "      <td>(The, cosmopolitan, species, Hypogastrura, via...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>2234</td>\n",
       "      <td>2236</td>\n",
       "      <td>11762</td>\n",
       "      <td>(The, cosmopolitan, species, Hypogastrura, via...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>2942</td>\n",
       "      <td>2944</td>\n",
       "      <td>15446</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>2948</td>\n",
       "      <td>2950</td>\n",
       "      <td>15481</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>True</td>\n",
       "      <td>2956</td>\n",
       "      <td>2958</td>\n",
       "      <td>15514</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Tullbergia mixta</td>\n",
       "      <td>Tullbergia mixta</td>\n",
       "      <td>True</td>\n",
       "      <td>2962</td>\n",
       "      <td>2964</td>\n",
       "      <td>15551</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>True</td>\n",
       "      <td>2968</td>\n",
       "      <td>2970</td>\n",
       "      <td>15584</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>2976</td>\n",
       "      <td>2979</td>\n",
       "      <td>15622</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Cryptopygus badasa</td>\n",
       "      <td>Cryptopygus badasa</td>\n",
       "      <td>True</td>\n",
       "      <td>2983</td>\n",
       "      <td>2985</td>\n",
       "      <td>15672</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>3007</td>\n",
       "      <td>3009</td>\n",
       "      <td>15799</td>\n",
       "      <td>(Two, species, are, non, -, indigenous, in, An...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>3010</td>\n",
       "      <td>3012</td>\n",
       "      <td>15824</td>\n",
       "      <td>(Two, species, are, non, -, indigenous, in, An...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>3116</td>\n",
       "      <td>3118</td>\n",
       "      <td>16357</td>\n",
       "      <td>(In, 2012, campaign, the, number, of, sampling...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5393</td>\n",
       "      <td>5395</td>\n",
       "      <td>27189</td>\n",
       "      <td>(Hypogastrura, viatica, and, Mesaphorura, macr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5396</td>\n",
       "      <td>5398</td>\n",
       "      <td>27214</td>\n",
       "      <td>(Hypogastrura, viatica, and, Mesaphorura, macr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>H. viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5449</td>\n",
       "      <td>5451</td>\n",
       "      <td>27517</td>\n",
       "      <td>(The, first, species, ,, H., viatica, ,, is, w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>5541</td>\n",
       "      <td>5544</td>\n",
       "      <td>27996</td>\n",
       "      <td>(Its, presence, could, affect, the, native, Co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>H. viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5562</td>\n",
       "      <td>5564</td>\n",
       "      <td>28092</td>\n",
       "      <td>(On, Barrientos, Island, ,, H., viatica, was, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5580</td>\n",
       "      <td>5582</td>\n",
       "      <td>28175</td>\n",
       "      <td>(On, Barrientos, Island, ,, H., viatica, was, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>H. viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5595</td>\n",
       "      <td>5597</td>\n",
       "      <td>28241</td>\n",
       "      <td>(The, presence, of, H., viatica, could, be, a,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5627</td>\n",
       "      <td>5629</td>\n",
       "      <td>28411</td>\n",
       "      <td>(The, presence, of, M., macrochaeta, may, be, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>H. viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5860</td>\n",
       "      <td>5862</td>\n",
       "      <td>29653</td>\n",
       "      <td>(The, first, non, -, indigenous, invertebrate,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5952</td>\n",
       "      <td>5954</td>\n",
       "      <td>30181</td>\n",
       "      <td>(The, species, M., macrochaeta, was, reported,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>True</td>\n",
       "      <td>6646</td>\n",
       "      <td>6648</td>\n",
       "      <td>34145</td>\n",
       "      <td>(On, Barrientos, Island, ,, environments, domi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>6680</td>\n",
       "      <td>6683</td>\n",
       "      <td>34337</td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, has, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>P. crispa</td>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>True</td>\n",
       "      <td>6742</td>\n",
       "      <td>6744</td>\n",
       "      <td>34668</td>\n",
       "      <td>(Behind, its, predilection, for, P., crispa, i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>6789</td>\n",
       "      <td>6792</td>\n",
       "      <td>34967</td>\n",
       "      <td>(Therefore, ,, mosses, are, less, consumed, by...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>6863</td>\n",
       "      <td>6866</td>\n",
       "      <td>35339</td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, Was, v...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Cryptopygus badasa</td>\n",
       "      <td>Cryptopygus badasa</td>\n",
       "      <td>True</td>\n",
       "      <td>6909</td>\n",
       "      <td>6911</td>\n",
       "      <td>35622</td>\n",
       "      <td>(For, example, ,, Cryptopygus, badasa, ,, the,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>7054</td>\n",
       "      <td>7056</td>\n",
       "      <td>36453</td>\n",
       "      <td>(On, Barrientos, Island, ,, the, species, with...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>7165</td>\n",
       "      <td>7167</td>\n",
       "      <td>37034</td>\n",
       "      <td>(Therefore, ,, it, is, not, surprising, that, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>7198</td>\n",
       "      <td>7200</td>\n",
       "      <td>37224</td>\n",
       "      <td>(This, indicates, that, F., grisea, was, widel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>7216</td>\n",
       "      <td>7218</td>\n",
       "      <td>37331</td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>F. woyciechowskii</td>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>True</td>\n",
       "      <td>7219</td>\n",
       "      <td>7221</td>\n",
       "      <td>37353</td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>7234</td>\n",
       "      <td>7236</td>\n",
       "      <td>37433</td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>True</td>\n",
       "      <td>7548</td>\n",
       "      <td>7550</td>\n",
       "      <td>39242</td>\n",
       "      <td>(Archisotoma, brucei, is, usually, classified,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>A. brucei</td>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>True</td>\n",
       "      <td>7631</td>\n",
       "      <td>7633</td>\n",
       "      <td>39708</td>\n",
       "      <td>(While, this, is, an, unusual, location, for, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>7830</td>\n",
       "      <td>7832</td>\n",
       "      <td>40793</td>\n",
       "      <td>(—, Hypogastrura, viatica, and, Mesaphorura, m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>7833</td>\n",
       "      <td>7835</td>\n",
       "      <td>40818</td>\n",
       "      <td>(—, Hypogastrura, viatica, and, Mesaphorura, m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>Cryptopygus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>8537</td>\n",
       "      <td>8539</td>\n",
       "      <td>44560</td>\n",
       "      <td>(1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>Gomphiocephalus hodgsoni</td>\n",
       "      <td>Gomphiocephalus hodgsoni</td>\n",
       "      <td>True</td>\n",
       "      <td>8790</td>\n",
       "      <td>8792</td>\n",
       "      <td>45883</td>\n",
       "      <td>(Analysis, of, gut, contents, of, Gomphiocepha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>Cryptopygus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>8841</td>\n",
       "      <td>8843</td>\n",
       "      <td>46164</td>\n",
       "      <td>(Response, of, plants, and, the, dominant, mic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>True</td>\n",
       "      <td>10455</td>\n",
       "      <td>10457</td>\n",
       "      <td>54876</td>\n",
       "      <td>((, 1980, ), Friesea, woyciechowskii)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>106 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Found as                            Full name  \\\n",
       "0    Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "1                   Hypogastrura viatica                 Hypogastrura viatica   \n",
       "2                Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "3                      Sanionia uncinata                    Sanionia uncinata   \n",
       "4                    S. georgicouncinata            Sanionia georgicouncinata   \n",
       "5                      Sanionia uncinata                    Sanionia uncinata   \n",
       "6                Polytrichastrum alpinum              Polytrichastrum alpinum   \n",
       "7                 Bryum pseudotriquetrum               Bryum pseudotriquetrum   \n",
       "8              Sanionia georgicouncinata            Sanionia georgicouncinata   \n",
       "9             Warnstorfia fontinaliopsis           Warnstorfia fontinaliopsis   \n",
       "10                       Prasiola crispa                      Prasiola crispa   \n",
       "11                      Pygoscelis papua                     Pygoscelis papua   \n",
       "12                Pygoscelis antarcticus               Pygoscelis antarcticus   \n",
       "13                 Macronectes giganteus                Macronectes giganteus   \n",
       "14                     Larus dominicanus                    Larus dominicanus   \n",
       "15                      Mirounga leonina                     Mirounga leonina   \n",
       "16               Leptonychotes weddellii              Leptonychotes weddellii   \n",
       "17                 Arctocephalus gazella                Arctocephalus gazella   \n",
       "18                  Hypogastrura viatica                 Hypogastrura viatica   \n",
       "19               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "20                     Friesea bispinosa                    Friesea bispinosa   \n",
       "21                        Friesea grisea                       Friesea grisea   \n",
       "22                Friesea woyciechowskii               Friesea woyciechowskii   \n",
       "23                      Tullbergia mixta                     Tullbergia mixta   \n",
       "24                    Archisotoma brucei                   Archisotoma brucei   \n",
       "25   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "26                    Cryptopygus badasa                   Cryptopygus badasa   \n",
       "27                  Hypogastrura viatica                 Hypogastrura viatica   \n",
       "28               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "29                             F. grisea                       Friesea grisea   \n",
       "..                                   ...                                  ...   \n",
       "76                  Hypogastrura viatica                 Hypogastrura viatica   \n",
       "77               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "78                            H. viatica                 Hypogastrura viatica   \n",
       "79                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "80                            H. viatica                 Hypogastrura viatica   \n",
       "81                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "82                            H. viatica                 Hypogastrura viatica   \n",
       "83                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "84                            H. viatica                 Hypogastrura viatica   \n",
       "85                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "86                       Prasiola crispa                      Prasiola crispa   \n",
       "87   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "88                             P. crispa                      Prasiola crispa   \n",
       "89                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "90   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "91                    Cryptopygus badasa                   Cryptopygus badasa   \n",
       "92                             F. grisea                       Friesea grisea   \n",
       "93                             F. grisea                       Friesea grisea   \n",
       "94                             F. grisea                       Friesea grisea   \n",
       "95                     Friesea bispinosa                    Friesea bispinosa   \n",
       "96                     F. woyciechowskii               Friesea woyciechowskii   \n",
       "97                             F. grisea                       Friesea grisea   \n",
       "98                    Archisotoma brucei                   Archisotoma brucei   \n",
       "99                             A. brucei                   Archisotoma brucei   \n",
       "100                 Hypogastrura viatica                 Hypogastrura viatica   \n",
       "101              Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "102              Cryptopygus antarcticus              Cryptopygus antarcticus   \n",
       "103             Gomphiocephalus hodgsoni             Gomphiocephalus hodgsoni   \n",
       "104              Cryptopygus antarcticus              Cryptopygus antarcticus   \n",
       "105               Friesea woyciechowskii               Friesea woyciechowskii   \n",
       "\n",
       "     InDefinitiveList  Start    End  PositionOfFirstToken  \\\n",
       "0                True    226    229                  1330   \n",
       "1                True    247    249                  1459   \n",
       "2               False    256    258                  1513   \n",
       "3                True   1453   1455                  7780   \n",
       "4               False   1461   1463                  7817   \n",
       "5                True   1475   1477                  7880   \n",
       "6                True   1484   1486                  7955   \n",
       "7                True   1494   1496                  8000   \n",
       "8               False   1517   1519                  8101   \n",
       "9                True   1523   1525                  8147   \n",
       "10               True   1566   1568                  8367   \n",
       "11               True   1592   1594                  8523   \n",
       "12               True   1604   1606                  8578   \n",
       "13               True   1617   1619                  8643   \n",
       "14               True   1629   1631                  8694   \n",
       "15               True   1647   1649                  8783   \n",
       "16               True   1659   1661                  8834   \n",
       "17               True   1673   1675                  8900   \n",
       "18               True   2226   2228                 11720   \n",
       "19              False   2234   2236                 11762   \n",
       "20              False   2942   2944                 15446   \n",
       "21               True   2948   2950                 15481   \n",
       "22               True   2956   2958                 15514   \n",
       "23               True   2962   2964                 15551   \n",
       "24               True   2968   2970                 15584   \n",
       "25               True   2976   2979                 15622   \n",
       "26               True   2983   2985                 15672   \n",
       "27               True   3007   3009                 15799   \n",
       "28              False   3010   3012                 15824   \n",
       "29               True   3116   3118                 16357   \n",
       "..                ...    ...    ...                   ...   \n",
       "76               True   5393   5395                 27189   \n",
       "77              False   5396   5398                 27214   \n",
       "78               True   5449   5451                 27517   \n",
       "79               True   5541   5544                 27996   \n",
       "80               True   5562   5564                 28092   \n",
       "81              False   5580   5582                 28175   \n",
       "82               True   5595   5597                 28241   \n",
       "83              False   5627   5629                 28411   \n",
       "84               True   5860   5862                 29653   \n",
       "85              False   5952   5954                 30181   \n",
       "86               True   6646   6648                 34145   \n",
       "87               True   6680   6683                 34337   \n",
       "88               True   6742   6744                 34668   \n",
       "89               True   6789   6792                 34967   \n",
       "90               True   6863   6866                 35339   \n",
       "91               True   6909   6911                 35622   \n",
       "92               True   7054   7056                 36453   \n",
       "93               True   7165   7167                 37034   \n",
       "94               True   7198   7200                 37224   \n",
       "95              False   7216   7218                 37331   \n",
       "96               True   7219   7221                 37353   \n",
       "97               True   7234   7236                 37433   \n",
       "98               True   7548   7550                 39242   \n",
       "99               True   7631   7633                 39708   \n",
       "100              True   7830   7832                 40793   \n",
       "101             False   7833   7835                 40818   \n",
       "102              True   8537   8539                 44560   \n",
       "103              True   8790   8792                 45883   \n",
       "104              True   8841   8843                 46164   \n",
       "105              True  10455  10457                 54876   \n",
       "\n",
       "                                              Sentence  \n",
       "0    (The, native, species, Cryptopygus, antarcticu...  \n",
       "1              (Hypogastrura, viatica, (, Tullberg, ))  \n",
       "2    ((, Hypogastruridae, ), and, Mesaphorura, macr...  \n",
       "3                  (Sanionia, uncinata, (, Hedw, ., ))  \n",
       "4      (Loeske, and, S., georgicouncinata, (, Mill, .)  \n",
       "5    (Sanionia, uncinata, forms, monospecific, comm...  \n",
       "6    (Sanionia, uncinata, forms, monospecific, comm...  \n",
       "7    (G.L., Sm, ., and, Bryum, pseudotriquetrum, (,...  \n",
       "8    (Sanionia, georgicouncinata, is, often, found,...  \n",
       "9    (Sanionia, georgicouncinata, is, often, found,...  \n",
       "10   (The, green, alga, Prasiola, crispa, (, Lightf...  \n",
       "11   (The, local, fauna, is, diverse, and, includes...  \n",
       "12   (The, local, fauna, is, diverse, and, includes...  \n",
       "13   (The, local, fauna, is, diverse, and, includes...  \n",
       "14   (The, local, fauna, is, diverse, and, includes...  \n",
       "15   (The, local, fauna, is, diverse, and, includes...  \n",
       "16   (The, local, fauna, is, diverse, and, includes...  \n",
       "17   (The, local, fauna, is, diverse, and, includes...  \n",
       "18   (The, cosmopolitan, species, Hypogastrura, via...  \n",
       "19   (The, cosmopolitan, species, Hypogastrura, via...  \n",
       "20   (Eight, species, are, native, in, Antarctica, ...  \n",
       "21   (Eight, species, are, native, in, Antarctica, ...  \n",
       "22   (Eight, species, are, native, in, Antarctica, ...  \n",
       "23   (Eight, species, are, native, in, Antarctica, ...  \n",
       "24   (Eight, species, are, native, in, Antarctica, ...  \n",
       "25   (Eight, species, are, native, in, Antarctica, ...  \n",
       "26   (Eight, species, are, native, in, Antarctica, ...  \n",
       "27   (Two, species, are, non, -, indigenous, in, An...  \n",
       "28   (Two, species, are, non, -, indigenous, in, An...  \n",
       "29   (In, 2012, campaign, the, number, of, sampling...  \n",
       "..                                                 ...  \n",
       "76   (Hypogastrura, viatica, and, Mesaphorura, macr...  \n",
       "77   (Hypogastrura, viatica, and, Mesaphorura, macr...  \n",
       "78   (The, first, species, ,, H., viatica, ,, is, w...  \n",
       "79   (Its, presence, could, affect, the, native, Co...  \n",
       "80   (On, Barrientos, Island, ,, H., viatica, was, ...  \n",
       "81   (On, Barrientos, Island, ,, H., viatica, was, ...  \n",
       "82   (The, presence, of, H., viatica, could, be, a,...  \n",
       "83   (The, presence, of, M., macrochaeta, may, be, ...  \n",
       "84   (The, first, non, -, indigenous, invertebrate,...  \n",
       "85   (The, species, M., macrochaeta, was, reported,...  \n",
       "86   (On, Barrientos, Island, ,, environments, domi...  \n",
       "87   (Cryptopygus, antarcticus, antarcticus, has, a...  \n",
       "88   (Behind, its, predilection, for, P., crispa, i...  \n",
       "89   (Therefore, ,, mosses, are, less, consumed, by...  \n",
       "90   (Cryptopygus, antarcticus, antarcticus, Was, v...  \n",
       "91   (For, example, ,, Cryptopygus, badasa, ,, the,...  \n",
       "92   (On, Barrientos, Island, ,, the, species, with...  \n",
       "93   (Therefore, ,, it, is, not, surprising, that, ...  \n",
       "94   (This, indicates, that, F., grisea, was, widel...  \n",
       "95   (Friesea, bispinosa, and, F., woyciechowskii, ...  \n",
       "96   (Friesea, bispinosa, and, F., woyciechowskii, ...  \n",
       "97   (Friesea, bispinosa, and, F., woyciechowskii, ...  \n",
       "98   (Archisotoma, brucei, is, usually, classified,...  \n",
       "99   (While, this, is, an, unusual, location, for, ...  \n",
       "100  (—, Hypogastrura, viatica, and, Mesaphorura, m...  \n",
       "101  (—, Hypogastrura, viatica, and, Mesaphorura, m...  \n",
       "102  (1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...  \n",
       "103  (Analysis, of, gut, contents, of, Gomphiocepha...  \n",
       "104  (Response, of, plants, and, the, dominant, mic...  \n",
       "105              ((, 1980, ), Friesea, woyciechowskii)  \n",
       "\n",
       "[106 rows x 7 columns]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SpeciesFound = findSpecies(document)\n",
    "SpeciesFound"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T23:32:49.765773Z",
     "start_time": "2021-01-26T23:32:49.756676Z"
    }
   },
   "outputs": [],
   "source": [
    "SpeciesFound['InReferenceList']= [True if x in uniqueSpeciesStrings else False for x in SpeciesFound['Full name']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T23:32:55.486456Z",
     "start_time": "2021-01-26T23:32:55.404547Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Found as</th>\n",
       "      <th>Full name</th>\n",
       "      <th>InDefinitiveList</th>\n",
       "      <th>Start</th>\n",
       "      <th>End</th>\n",
       "      <th>PositionOfFirstToken</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>InReferenceList</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>226</td>\n",
       "      <td>229</td>\n",
       "      <td>1330</td>\n",
       "      <td>(The, native, species, Cryptopygus, antarcticu...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>247</td>\n",
       "      <td>249</td>\n",
       "      <td>1459</td>\n",
       "      <td>(Hypogastrura, viatica, (, Tullberg, ))</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>256</td>\n",
       "      <td>258</td>\n",
       "      <td>1513</td>\n",
       "      <td>((, Hypogastruridae, ), and, Mesaphorura, macr...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sanionia uncinata</td>\n",
       "      <td>Sanionia uncinata</td>\n",
       "      <td>True</td>\n",
       "      <td>1453</td>\n",
       "      <td>1455</td>\n",
       "      <td>7780</td>\n",
       "      <td>(Sanionia, uncinata, (, Hedw, ., ))</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S. georgicouncinata</td>\n",
       "      <td>Sanionia georgicouncinata</td>\n",
       "      <td>False</td>\n",
       "      <td>1461</td>\n",
       "      <td>1463</td>\n",
       "      <td>7817</td>\n",
       "      <td>(Loeske, and, S., georgicouncinata, (, Mill, .)</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Sanionia uncinata</td>\n",
       "      <td>Sanionia uncinata</td>\n",
       "      <td>True</td>\n",
       "      <td>1475</td>\n",
       "      <td>1477</td>\n",
       "      <td>7880</td>\n",
       "      <td>(Sanionia, uncinata, forms, monospecific, comm...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Polytrichastrum alpinum</td>\n",
       "      <td>Polytrichastrum alpinum</td>\n",
       "      <td>True</td>\n",
       "      <td>1484</td>\n",
       "      <td>1486</td>\n",
       "      <td>7955</td>\n",
       "      <td>(Sanionia, uncinata, forms, monospecific, comm...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Bryum pseudotriquetrum</td>\n",
       "      <td>Bryum pseudotriquetrum</td>\n",
       "      <td>True</td>\n",
       "      <td>1494</td>\n",
       "      <td>1496</td>\n",
       "      <td>8000</td>\n",
       "      <td>(G.L., Sm, ., and, Bryum, pseudotriquetrum, (,...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Sanionia georgicouncinata</td>\n",
       "      <td>Sanionia georgicouncinata</td>\n",
       "      <td>False</td>\n",
       "      <td>1517</td>\n",
       "      <td>1519</td>\n",
       "      <td>8101</td>\n",
       "      <td>(Sanionia, georgicouncinata, is, often, found,...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Warnstorfia fontinaliopsis</td>\n",
       "      <td>Warnstorfia fontinaliopsis</td>\n",
       "      <td>True</td>\n",
       "      <td>1523</td>\n",
       "      <td>1525</td>\n",
       "      <td>8147</td>\n",
       "      <td>(Sanionia, georgicouncinata, is, often, found,...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>True</td>\n",
       "      <td>1566</td>\n",
       "      <td>1568</td>\n",
       "      <td>8367</td>\n",
       "      <td>(The, green, alga, Prasiola, crispa, (, Lightf...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Pygoscelis papua</td>\n",
       "      <td>Pygoscelis papua</td>\n",
       "      <td>True</td>\n",
       "      <td>1592</td>\n",
       "      <td>1594</td>\n",
       "      <td>8523</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Pygoscelis antarcticus</td>\n",
       "      <td>Pygoscelis antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>1604</td>\n",
       "      <td>1606</td>\n",
       "      <td>8578</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Macronectes giganteus</td>\n",
       "      <td>Macronectes giganteus</td>\n",
       "      <td>True</td>\n",
       "      <td>1617</td>\n",
       "      <td>1619</td>\n",
       "      <td>8643</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Larus dominicanus</td>\n",
       "      <td>Larus dominicanus</td>\n",
       "      <td>True</td>\n",
       "      <td>1629</td>\n",
       "      <td>1631</td>\n",
       "      <td>8694</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Mirounga leonina</td>\n",
       "      <td>Mirounga leonina</td>\n",
       "      <td>True</td>\n",
       "      <td>1647</td>\n",
       "      <td>1649</td>\n",
       "      <td>8783</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Leptonychotes weddellii</td>\n",
       "      <td>Leptonychotes weddellii</td>\n",
       "      <td>True</td>\n",
       "      <td>1659</td>\n",
       "      <td>1661</td>\n",
       "      <td>8834</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Arctocephalus gazella</td>\n",
       "      <td>Arctocephalus gazella</td>\n",
       "      <td>True</td>\n",
       "      <td>1673</td>\n",
       "      <td>1675</td>\n",
       "      <td>8900</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>2226</td>\n",
       "      <td>2228</td>\n",
       "      <td>11720</td>\n",
       "      <td>(The, cosmopolitan, species, Hypogastrura, via...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>2234</td>\n",
       "      <td>2236</td>\n",
       "      <td>11762</td>\n",
       "      <td>(The, cosmopolitan, species, Hypogastrura, via...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>2942</td>\n",
       "      <td>2944</td>\n",
       "      <td>15446</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>2948</td>\n",
       "      <td>2950</td>\n",
       "      <td>15481</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>True</td>\n",
       "      <td>2956</td>\n",
       "      <td>2958</td>\n",
       "      <td>15514</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Tullbergia mixta</td>\n",
       "      <td>Tullbergia mixta</td>\n",
       "      <td>True</td>\n",
       "      <td>2962</td>\n",
       "      <td>2964</td>\n",
       "      <td>15551</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>True</td>\n",
       "      <td>2968</td>\n",
       "      <td>2970</td>\n",
       "      <td>15584</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>2976</td>\n",
       "      <td>2979</td>\n",
       "      <td>15622</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Cryptopygus badasa</td>\n",
       "      <td>Cryptopygus badasa</td>\n",
       "      <td>True</td>\n",
       "      <td>2983</td>\n",
       "      <td>2985</td>\n",
       "      <td>15672</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>3007</td>\n",
       "      <td>3009</td>\n",
       "      <td>15799</td>\n",
       "      <td>(Two, species, are, non, -, indigenous, in, An...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>3010</td>\n",
       "      <td>3012</td>\n",
       "      <td>15824</td>\n",
       "      <td>(Two, species, are, non, -, indigenous, in, An...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>3116</td>\n",
       "      <td>3118</td>\n",
       "      <td>16357</td>\n",
       "      <td>(In, 2012, campaign, the, number, of, sampling...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5393</td>\n",
       "      <td>5395</td>\n",
       "      <td>27189</td>\n",
       "      <td>(Hypogastrura, viatica, and, Mesaphorura, macr...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5396</td>\n",
       "      <td>5398</td>\n",
       "      <td>27214</td>\n",
       "      <td>(Hypogastrura, viatica, and, Mesaphorura, macr...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>H. viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5449</td>\n",
       "      <td>5451</td>\n",
       "      <td>27517</td>\n",
       "      <td>(The, first, species, ,, H., viatica, ,, is, w...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>5541</td>\n",
       "      <td>5544</td>\n",
       "      <td>27996</td>\n",
       "      <td>(Its, presence, could, affect, the, native, Co...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>H. viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5562</td>\n",
       "      <td>5564</td>\n",
       "      <td>28092</td>\n",
       "      <td>(On, Barrientos, Island, ,, H., viatica, was, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5580</td>\n",
       "      <td>5582</td>\n",
       "      <td>28175</td>\n",
       "      <td>(On, Barrientos, Island, ,, H., viatica, was, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>H. viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5595</td>\n",
       "      <td>5597</td>\n",
       "      <td>28241</td>\n",
       "      <td>(The, presence, of, H., viatica, could, be, a,...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5627</td>\n",
       "      <td>5629</td>\n",
       "      <td>28411</td>\n",
       "      <td>(The, presence, of, M., macrochaeta, may, be, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>H. viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>5860</td>\n",
       "      <td>5862</td>\n",
       "      <td>29653</td>\n",
       "      <td>(The, first, non, -, indigenous, invertebrate,...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5952</td>\n",
       "      <td>5954</td>\n",
       "      <td>30181</td>\n",
       "      <td>(The, species, M., macrochaeta, was, reported,...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>True</td>\n",
       "      <td>6646</td>\n",
       "      <td>6648</td>\n",
       "      <td>34145</td>\n",
       "      <td>(On, Barrientos, Island, ,, environments, domi...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>6680</td>\n",
       "      <td>6683</td>\n",
       "      <td>34337</td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, has, a...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>P. crispa</td>\n",
       "      <td>Prasiola crispa</td>\n",
       "      <td>True</td>\n",
       "      <td>6742</td>\n",
       "      <td>6744</td>\n",
       "      <td>34668</td>\n",
       "      <td>(Behind, its, predilection, for, P., crispa, i...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>6789</td>\n",
       "      <td>6792</td>\n",
       "      <td>34967</td>\n",
       "      <td>(Therefore, ,, mosses, are, less, consumed, by...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>6863</td>\n",
       "      <td>6866</td>\n",
       "      <td>35339</td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, Was, v...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Cryptopygus badasa</td>\n",
       "      <td>Cryptopygus badasa</td>\n",
       "      <td>True</td>\n",
       "      <td>6909</td>\n",
       "      <td>6911</td>\n",
       "      <td>35622</td>\n",
       "      <td>(For, example, ,, Cryptopygus, badasa, ,, the,...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>7054</td>\n",
       "      <td>7056</td>\n",
       "      <td>36453</td>\n",
       "      <td>(On, Barrientos, Island, ,, the, species, with...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>7165</td>\n",
       "      <td>7167</td>\n",
       "      <td>37034</td>\n",
       "      <td>(Therefore, ,, it, is, not, surprising, that, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>7198</td>\n",
       "      <td>7200</td>\n",
       "      <td>37224</td>\n",
       "      <td>(This, indicates, that, F., grisea, was, widel...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>7216</td>\n",
       "      <td>7218</td>\n",
       "      <td>37331</td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>F. woyciechowskii</td>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>True</td>\n",
       "      <td>7219</td>\n",
       "      <td>7221</td>\n",
       "      <td>37353</td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>F. grisea</td>\n",
       "      <td>Friesea grisea</td>\n",
       "      <td>True</td>\n",
       "      <td>7234</td>\n",
       "      <td>7236</td>\n",
       "      <td>37433</td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>True</td>\n",
       "      <td>7548</td>\n",
       "      <td>7550</td>\n",
       "      <td>39242</td>\n",
       "      <td>(Archisotoma, brucei, is, usually, classified,...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>A. brucei</td>\n",
       "      <td>Archisotoma brucei</td>\n",
       "      <td>True</td>\n",
       "      <td>7631</td>\n",
       "      <td>7633</td>\n",
       "      <td>39708</td>\n",
       "      <td>(While, this, is, an, unusual, location, for, ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>Hypogastrura viatica</td>\n",
       "      <td>True</td>\n",
       "      <td>7830</td>\n",
       "      <td>7832</td>\n",
       "      <td>40793</td>\n",
       "      <td>(—, Hypogastrura, viatica, and, Mesaphorura, m...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>7833</td>\n",
       "      <td>7835</td>\n",
       "      <td>40818</td>\n",
       "      <td>(—, Hypogastrura, viatica, and, Mesaphorura, m...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>Cryptopygus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>8537</td>\n",
       "      <td>8539</td>\n",
       "      <td>44560</td>\n",
       "      <td>(1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>Gomphiocephalus hodgsoni</td>\n",
       "      <td>Gomphiocephalus hodgsoni</td>\n",
       "      <td>True</td>\n",
       "      <td>8790</td>\n",
       "      <td>8792</td>\n",
       "      <td>45883</td>\n",
       "      <td>(Analysis, of, gut, contents, of, Gomphiocepha...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>Cryptopygus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>8841</td>\n",
       "      <td>8843</td>\n",
       "      <td>46164</td>\n",
       "      <td>(Response, of, plants, and, the, dominant, mic...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>Friesea woyciechowskii</td>\n",
       "      <td>True</td>\n",
       "      <td>10455</td>\n",
       "      <td>10457</td>\n",
       "      <td>54876</td>\n",
       "      <td>((, 1980, ), Friesea, woyciechowskii)</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>106 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Found as                            Full name  \\\n",
       "0    Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "1                   Hypogastrura viatica                 Hypogastrura viatica   \n",
       "2                Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "3                      Sanionia uncinata                    Sanionia uncinata   \n",
       "4                    S. georgicouncinata            Sanionia georgicouncinata   \n",
       "5                      Sanionia uncinata                    Sanionia uncinata   \n",
       "6                Polytrichastrum alpinum              Polytrichastrum alpinum   \n",
       "7                 Bryum pseudotriquetrum               Bryum pseudotriquetrum   \n",
       "8              Sanionia georgicouncinata            Sanionia georgicouncinata   \n",
       "9             Warnstorfia fontinaliopsis           Warnstorfia fontinaliopsis   \n",
       "10                       Prasiola crispa                      Prasiola crispa   \n",
       "11                      Pygoscelis papua                     Pygoscelis papua   \n",
       "12                Pygoscelis antarcticus               Pygoscelis antarcticus   \n",
       "13                 Macronectes giganteus                Macronectes giganteus   \n",
       "14                     Larus dominicanus                    Larus dominicanus   \n",
       "15                      Mirounga leonina                     Mirounga leonina   \n",
       "16               Leptonychotes weddellii              Leptonychotes weddellii   \n",
       "17                 Arctocephalus gazella                Arctocephalus gazella   \n",
       "18                  Hypogastrura viatica                 Hypogastrura viatica   \n",
       "19               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "20                     Friesea bispinosa                    Friesea bispinosa   \n",
       "21                        Friesea grisea                       Friesea grisea   \n",
       "22                Friesea woyciechowskii               Friesea woyciechowskii   \n",
       "23                      Tullbergia mixta                     Tullbergia mixta   \n",
       "24                    Archisotoma brucei                   Archisotoma brucei   \n",
       "25   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "26                    Cryptopygus badasa                   Cryptopygus badasa   \n",
       "27                  Hypogastrura viatica                 Hypogastrura viatica   \n",
       "28               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "29                             F. grisea                       Friesea grisea   \n",
       "..                                   ...                                  ...   \n",
       "76                  Hypogastrura viatica                 Hypogastrura viatica   \n",
       "77               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "78                            H. viatica                 Hypogastrura viatica   \n",
       "79                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "80                            H. viatica                 Hypogastrura viatica   \n",
       "81                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "82                            H. viatica                 Hypogastrura viatica   \n",
       "83                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "84                            H. viatica                 Hypogastrura viatica   \n",
       "85                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "86                       Prasiola crispa                      Prasiola crispa   \n",
       "87   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "88                             P. crispa                      Prasiola crispa   \n",
       "89                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "90   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "91                    Cryptopygus badasa                   Cryptopygus badasa   \n",
       "92                             F. grisea                       Friesea grisea   \n",
       "93                             F. grisea                       Friesea grisea   \n",
       "94                             F. grisea                       Friesea grisea   \n",
       "95                     Friesea bispinosa                    Friesea bispinosa   \n",
       "96                     F. woyciechowskii               Friesea woyciechowskii   \n",
       "97                             F. grisea                       Friesea grisea   \n",
       "98                    Archisotoma brucei                   Archisotoma brucei   \n",
       "99                             A. brucei                   Archisotoma brucei   \n",
       "100                 Hypogastrura viatica                 Hypogastrura viatica   \n",
       "101              Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "102              Cryptopygus antarcticus              Cryptopygus antarcticus   \n",
       "103             Gomphiocephalus hodgsoni             Gomphiocephalus hodgsoni   \n",
       "104              Cryptopygus antarcticus              Cryptopygus antarcticus   \n",
       "105               Friesea woyciechowskii               Friesea woyciechowskii   \n",
       "\n",
       "     InDefinitiveList  Start    End  PositionOfFirstToken  \\\n",
       "0                True    226    229                  1330   \n",
       "1                True    247    249                  1459   \n",
       "2               False    256    258                  1513   \n",
       "3                True   1453   1455                  7780   \n",
       "4               False   1461   1463                  7817   \n",
       "5                True   1475   1477                  7880   \n",
       "6                True   1484   1486                  7955   \n",
       "7                True   1494   1496                  8000   \n",
       "8               False   1517   1519                  8101   \n",
       "9                True   1523   1525                  8147   \n",
       "10               True   1566   1568                  8367   \n",
       "11               True   1592   1594                  8523   \n",
       "12               True   1604   1606                  8578   \n",
       "13               True   1617   1619                  8643   \n",
       "14               True   1629   1631                  8694   \n",
       "15               True   1647   1649                  8783   \n",
       "16               True   1659   1661                  8834   \n",
       "17               True   1673   1675                  8900   \n",
       "18               True   2226   2228                 11720   \n",
       "19              False   2234   2236                 11762   \n",
       "20              False   2942   2944                 15446   \n",
       "21               True   2948   2950                 15481   \n",
       "22               True   2956   2958                 15514   \n",
       "23               True   2962   2964                 15551   \n",
       "24               True   2968   2970                 15584   \n",
       "25               True   2976   2979                 15622   \n",
       "26               True   2983   2985                 15672   \n",
       "27               True   3007   3009                 15799   \n",
       "28              False   3010   3012                 15824   \n",
       "29               True   3116   3118                 16357   \n",
       "..                ...    ...    ...                   ...   \n",
       "76               True   5393   5395                 27189   \n",
       "77              False   5396   5398                 27214   \n",
       "78               True   5449   5451                 27517   \n",
       "79               True   5541   5544                 27996   \n",
       "80               True   5562   5564                 28092   \n",
       "81              False   5580   5582                 28175   \n",
       "82               True   5595   5597                 28241   \n",
       "83              False   5627   5629                 28411   \n",
       "84               True   5860   5862                 29653   \n",
       "85              False   5952   5954                 30181   \n",
       "86               True   6646   6648                 34145   \n",
       "87               True   6680   6683                 34337   \n",
       "88               True   6742   6744                 34668   \n",
       "89               True   6789   6792                 34967   \n",
       "90               True   6863   6866                 35339   \n",
       "91               True   6909   6911                 35622   \n",
       "92               True   7054   7056                 36453   \n",
       "93               True   7165   7167                 37034   \n",
       "94               True   7198   7200                 37224   \n",
       "95              False   7216   7218                 37331   \n",
       "96               True   7219   7221                 37353   \n",
       "97               True   7234   7236                 37433   \n",
       "98               True   7548   7550                 39242   \n",
       "99               True   7631   7633                 39708   \n",
       "100              True   7830   7832                 40793   \n",
       "101             False   7833   7835                 40818   \n",
       "102              True   8537   8539                 44560   \n",
       "103              True   8790   8792                 45883   \n",
       "104              True   8841   8843                 46164   \n",
       "105              True  10455  10457                 54876   \n",
       "\n",
       "                                              Sentence  InReferenceList  \n",
       "0    (The, native, species, Cryptopygus, antarcticu...            False  \n",
       "1              (Hypogastrura, viatica, (, Tullberg, ))             True  \n",
       "2    ((, Hypogastruridae, ), and, Mesaphorura, macr...            False  \n",
       "3                  (Sanionia, uncinata, (, Hedw, ., ))             True  \n",
       "4      (Loeske, and, S., georgicouncinata, (, Mill, .)            False  \n",
       "5    (Sanionia, uncinata, forms, monospecific, comm...             True  \n",
       "6    (Sanionia, uncinata, forms, monospecific, comm...             True  \n",
       "7    (G.L., Sm, ., and, Bryum, pseudotriquetrum, (,...             True  \n",
       "8    (Sanionia, georgicouncinata, is, often, found,...            False  \n",
       "9    (Sanionia, georgicouncinata, is, often, found,...             True  \n",
       "10   (The, green, alga, Prasiola, crispa, (, Lightf...             True  \n",
       "11   (The, local, fauna, is, diverse, and, includes...             True  \n",
       "12   (The, local, fauna, is, diverse, and, includes...            False  \n",
       "13   (The, local, fauna, is, diverse, and, includes...             True  \n",
       "14   (The, local, fauna, is, diverse, and, includes...             True  \n",
       "15   (The, local, fauna, is, diverse, and, includes...             True  \n",
       "16   (The, local, fauna, is, diverse, and, includes...             True  \n",
       "17   (The, local, fauna, is, diverse, and, includes...             True  \n",
       "18   (The, cosmopolitan, species, Hypogastrura, via...             True  \n",
       "19   (The, cosmopolitan, species, Hypogastrura, via...            False  \n",
       "20   (Eight, species, are, native, in, Antarctica, ...            False  \n",
       "21   (Eight, species, are, native, in, Antarctica, ...             True  \n",
       "22   (Eight, species, are, native, in, Antarctica, ...             True  \n",
       "23   (Eight, species, are, native, in, Antarctica, ...             True  \n",
       "24   (Eight, species, are, native, in, Antarctica, ...             True  \n",
       "25   (Eight, species, are, native, in, Antarctica, ...            False  \n",
       "26   (Eight, species, are, native, in, Antarctica, ...             True  \n",
       "27   (Two, species, are, non, -, indigenous, in, An...             True  \n",
       "28   (Two, species, are, non, -, indigenous, in, An...            False  \n",
       "29   (In, 2012, campaign, the, number, of, sampling...             True  \n",
       "..                                                 ...              ...  \n",
       "76   (Hypogastrura, viatica, and, Mesaphorura, macr...             True  \n",
       "77   (Hypogastrura, viatica, and, Mesaphorura, macr...            False  \n",
       "78   (The, first, species, ,, H., viatica, ,, is, w...             True  \n",
       "79   (Its, presence, could, affect, the, native, Co...            False  \n",
       "80   (On, Barrientos, Island, ,, H., viatica, was, ...             True  \n",
       "81   (On, Barrientos, Island, ,, H., viatica, was, ...            False  \n",
       "82   (The, presence, of, H., viatica, could, be, a,...             True  \n",
       "83   (The, presence, of, M., macrochaeta, may, be, ...            False  \n",
       "84   (The, first, non, -, indigenous, invertebrate,...             True  \n",
       "85   (The, species, M., macrochaeta, was, reported,...            False  \n",
       "86   (On, Barrientos, Island, ,, environments, domi...             True  \n",
       "87   (Cryptopygus, antarcticus, antarcticus, has, a...            False  \n",
       "88   (Behind, its, predilection, for, P., crispa, i...             True  \n",
       "89   (Therefore, ,, mosses, are, less, consumed, by...            False  \n",
       "90   (Cryptopygus, antarcticus, antarcticus, Was, v...            False  \n",
       "91   (For, example, ,, Cryptopygus, badasa, ,, the,...             True  \n",
       "92   (On, Barrientos, Island, ,, the, species, with...             True  \n",
       "93   (Therefore, ,, it, is, not, surprising, that, ...             True  \n",
       "94   (This, indicates, that, F., grisea, was, widel...             True  \n",
       "95   (Friesea, bispinosa, and, F., woyciechowskii, ...            False  \n",
       "96   (Friesea, bispinosa, and, F., woyciechowskii, ...             True  \n",
       "97   (Friesea, bispinosa, and, F., woyciechowskii, ...             True  \n",
       "98   (Archisotoma, brucei, is, usually, classified,...             True  \n",
       "99   (While, this, is, an, unusual, location, for, ...             True  \n",
       "100  (—, Hypogastrura, viatica, and, Mesaphorura, m...             True  \n",
       "101  (—, Hypogastrura, viatica, and, Mesaphorura, m...            False  \n",
       "102  (1324, -, 1345, \\n, Broady, PA, (, 1979, ), Fe...             True  \n",
       "103  (Analysis, of, gut, contents, of, Gomphiocepha...             True  \n",
       "104  (Response, of, plants, and, the, dominant, mic...             True  \n",
       "105              ((, 1980, ), Friesea, woyciechowskii)             True  \n",
       "\n",
       "[106 rows x 8 columns]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SpeciesFound"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following species were found in the document based on rules and at least one token being found in the defintiive list but the complete name was not found in that list. e.g. \"Cryptopygus antarcticus antarcticus\" is not in the list as an exact match but both 'Cryptopygus' and 'antarcticus' are (for Cryptopygus antarcticus). Also, the algorithm finds \"Mesaphorura macrochaeta\" despite \"Mesaphorura\" not being present in the reference list because 'macrochaeta' is present in the entry for Tullbergia macrochaeta and other rules applied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-26T23:33:29.852035Z",
     "start_time": "2021-01-26T23:33:29.806932Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Found as</th>\n",
       "      <th>Full name</th>\n",
       "      <th>InDefinitiveList</th>\n",
       "      <th>Start</th>\n",
       "      <th>End</th>\n",
       "      <th>PositionOfFirstToken</th>\n",
       "      <th>Sentence</th>\n",
       "      <th>InReferenceList</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>226</td>\n",
       "      <td>229</td>\n",
       "      <td>1330</td>\n",
       "      <td>(The, native, species, Cryptopygus, antarcticu...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>256</td>\n",
       "      <td>258</td>\n",
       "      <td>1513</td>\n",
       "      <td>((, Hypogastruridae, ), and, Mesaphorura, macr...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S. georgicouncinata</td>\n",
       "      <td>Sanionia georgicouncinata</td>\n",
       "      <td>False</td>\n",
       "      <td>1461</td>\n",
       "      <td>1463</td>\n",
       "      <td>7817</td>\n",
       "      <td>(Loeske, and, S., georgicouncinata, (, Mill, .)</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Sanionia georgicouncinata</td>\n",
       "      <td>Sanionia georgicouncinata</td>\n",
       "      <td>False</td>\n",
       "      <td>1517</td>\n",
       "      <td>1519</td>\n",
       "      <td>8101</td>\n",
       "      <td>(Sanionia, georgicouncinata, is, often, found,...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Pygoscelis antarcticus</td>\n",
       "      <td>Pygoscelis antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>1604</td>\n",
       "      <td>1606</td>\n",
       "      <td>8578</td>\n",
       "      <td>(The, local, fauna, is, diverse, and, includes...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>2234</td>\n",
       "      <td>2236</td>\n",
       "      <td>11762</td>\n",
       "      <td>(The, cosmopolitan, species, Hypogastrura, via...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>2942</td>\n",
       "      <td>2944</td>\n",
       "      <td>15446</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>2976</td>\n",
       "      <td>2979</td>\n",
       "      <td>15622</td>\n",
       "      <td>(Eight, species, are, native, in, Antarctica, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>3010</td>\n",
       "      <td>3012</td>\n",
       "      <td>15824</td>\n",
       "      <td>(Two, species, are, non, -, indigenous, in, An...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>F. bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>3472</td>\n",
       "      <td>3474</td>\n",
       "      <td>17728</td>\n",
       "      <td>(obtaining, seven, species, ,, with, a, new, r...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>3556</td>\n",
       "      <td>3559</td>\n",
       "      <td>18170</td>\n",
       "      <td>(These, species, are, Cryptopygus, antarcticus...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>3634</td>\n",
       "      <td>3636</td>\n",
       "      <td>18551</td>\n",
       "      <td>(This, group, is, formed, by, Archisotoma, bru...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>3641</td>\n",
       "      <td>3643</td>\n",
       "      <td>18598</td>\n",
       "      <td>(This, group, is, formed, by, Archisotoma, bru...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>3711</td>\n",
       "      <td>3713</td>\n",
       "      <td>19018</td>\n",
       "      <td>(The, same, happens, with, F., woyciechowskii,...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>3733</td>\n",
       "      <td>3735</td>\n",
       "      <td>19140</td>\n",
       "      <td>(Friesea, bispinosa, ,, F., grisea, ,, and, F....</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>3921</td>\n",
       "      <td>3923</td>\n",
       "      <td>20089</td>\n",
       "      <td>(The, family, Tullbergiidae, was, represented,...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>4041</td>\n",
       "      <td>4044</td>\n",
       "      <td>20688</td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, was, f...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>4095</td>\n",
       "      <td>4098</td>\n",
       "      <td>21013</td>\n",
       "      <td>(The, least, favorable, substrates, for, C., a...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>4134</td>\n",
       "      <td>4137</td>\n",
       "      <td>21240</td>\n",
       "      <td>(Electrical, conductivity, had, a, negative, c...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>4288</td>\n",
       "      <td>4291</td>\n",
       "      <td>21967</td>\n",
       "      <td>(It, was, also, present, in, all, substrates, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>4887</td>\n",
       "      <td>4890</td>\n",
       "      <td>24464</td>\n",
       "      <td>(At, these, locations, ,, the, presence, of, l...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>F. bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>5101</td>\n",
       "      <td>5103</td>\n",
       "      <td>25577</td>\n",
       "      <td>(Folsomotoma, octooculata, and, F., bispinosa,...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>5130</td>\n",
       "      <td>5133</td>\n",
       "      <td>25734</td>\n",
       "      <td>(The, other, four, species, included, in, this...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5137</td>\n",
       "      <td>5139</td>\n",
       "      <td>25763</td>\n",
       "      <td>(The, other, four, species, included, in, this...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>5226</td>\n",
       "      <td>5229</td>\n",
       "      <td>26232</td>\n",
       "      <td>((, 2014, ), for, the, maritime, Antarctica, r...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>5253</td>\n",
       "      <td>5255</td>\n",
       "      <td>26376</td>\n",
       "      <td>(In, addition, ,, four, other, species, —, Fri...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5263</td>\n",
       "      <td>5265</td>\n",
       "      <td>26441</td>\n",
       "      <td>(In, addition, ,, four, other, species, —, Fri...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>5284</td>\n",
       "      <td>5287</td>\n",
       "      <td>26566</td>\n",
       "      <td>(The, most, abundant, species, was, Cryptopygu...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5396</td>\n",
       "      <td>5398</td>\n",
       "      <td>27214</td>\n",
       "      <td>(Hypogastrura, viatica, and, Mesaphorura, macr...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>5541</td>\n",
       "      <td>5544</td>\n",
       "      <td>27996</td>\n",
       "      <td>(Its, presence, could, affect, the, native, Co...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5580</td>\n",
       "      <td>5582</td>\n",
       "      <td>28175</td>\n",
       "      <td>(On, Barrientos, Island, ,, H., viatica, was, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5627</td>\n",
       "      <td>5629</td>\n",
       "      <td>28411</td>\n",
       "      <td>(The, presence, of, M., macrochaeta, may, be, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>M. macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>5952</td>\n",
       "      <td>5954</td>\n",
       "      <td>30181</td>\n",
       "      <td>(The, species, M., macrochaeta, was, reported,...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>6680</td>\n",
       "      <td>6683</td>\n",
       "      <td>34337</td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, has, a...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>C. a. antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>6789</td>\n",
       "      <td>6792</td>\n",
       "      <td>34967</td>\n",
       "      <td>(Therefore, ,, mosses, are, less, consumed, by...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>Cryptopygus antarcticus antarcticus</td>\n",
       "      <td>True</td>\n",
       "      <td>6863</td>\n",
       "      <td>6866</td>\n",
       "      <td>35339</td>\n",
       "      <td>(Cryptopygus, antarcticus, antarcticus, Was, v...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>Friesea bispinosa</td>\n",
       "      <td>False</td>\n",
       "      <td>7216</td>\n",
       "      <td>7218</td>\n",
       "      <td>37331</td>\n",
       "      <td>(Friesea, bispinosa, and, F., woyciechowskii, ...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>Mesaphorura macrochaeta</td>\n",
       "      <td>False</td>\n",
       "      <td>7833</td>\n",
       "      <td>7835</td>\n",
       "      <td>40818</td>\n",
       "      <td>(—, Hypogastrura, viatica, and, Mesaphorura, m...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Found as                            Full name  \\\n",
       "0    Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "2                Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "4                    S. georgicouncinata            Sanionia georgicouncinata   \n",
       "8              Sanionia georgicouncinata            Sanionia georgicouncinata   \n",
       "12                Pygoscelis antarcticus               Pygoscelis antarcticus   \n",
       "19               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "20                     Friesea bispinosa                    Friesea bispinosa   \n",
       "25   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "28               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "31                          F. bispinosa                    Friesea bispinosa   \n",
       "32   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "38                     Friesea bispinosa                    Friesea bispinosa   \n",
       "40               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "43                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "44                     Friesea bispinosa                    Friesea bispinosa   \n",
       "49               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "52   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "54                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "55                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "57                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "59                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "61                          F. bispinosa                    Friesea bispinosa   \n",
       "62                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "64                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "66                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "71                     Friesea bispinosa                    Friesea bispinosa   \n",
       "74               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "75   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "77               Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "79                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "81                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "83                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "85                        M. macrochaeta              Mesaphorura macrochaeta   \n",
       "87   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "89                     C. a. antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "90   Cryptopygus antarcticus antarcticus  Cryptopygus antarcticus antarcticus   \n",
       "95                     Friesea bispinosa                    Friesea bispinosa   \n",
       "101              Mesaphorura macrochaeta              Mesaphorura macrochaeta   \n",
       "\n",
       "     InDefinitiveList  Start   End  PositionOfFirstToken  \\\n",
       "0                True    226   229                  1330   \n",
       "2               False    256   258                  1513   \n",
       "4               False   1461  1463                  7817   \n",
       "8               False   1517  1519                  8101   \n",
       "12               True   1604  1606                  8578   \n",
       "19              False   2234  2236                 11762   \n",
       "20              False   2942  2944                 15446   \n",
       "25               True   2976  2979                 15622   \n",
       "28              False   3010  3012                 15824   \n",
       "31              False   3472  3474                 17728   \n",
       "32               True   3556  3559                 18170   \n",
       "38              False   3634  3636                 18551   \n",
       "40              False   3641  3643                 18598   \n",
       "43              False   3711  3713                 19018   \n",
       "44              False   3733  3735                 19140   \n",
       "49              False   3921  3923                 20089   \n",
       "52               True   4041  4044                 20688   \n",
       "54               True   4095  4098                 21013   \n",
       "55               True   4134  4137                 21240   \n",
       "57               True   4288  4291                 21967   \n",
       "59               True   4887  4890                 24464   \n",
       "61              False   5101  5103                 25577   \n",
       "62               True   5130  5133                 25734   \n",
       "64              False   5137  5139                 25763   \n",
       "66               True   5226  5229                 26232   \n",
       "71              False   5253  5255                 26376   \n",
       "74              False   5263  5265                 26441   \n",
       "75               True   5284  5287                 26566   \n",
       "77              False   5396  5398                 27214   \n",
       "79               True   5541  5544                 27996   \n",
       "81              False   5580  5582                 28175   \n",
       "83              False   5627  5629                 28411   \n",
       "85              False   5952  5954                 30181   \n",
       "87               True   6680  6683                 34337   \n",
       "89               True   6789  6792                 34967   \n",
       "90               True   6863  6866                 35339   \n",
       "95              False   7216  7218                 37331   \n",
       "101             False   7833  7835                 40818   \n",
       "\n",
       "                                              Sentence  InReferenceList  \n",
       "0    (The, native, species, Cryptopygus, antarcticu...            False  \n",
       "2    ((, Hypogastruridae, ), and, Mesaphorura, macr...            False  \n",
       "4      (Loeske, and, S., georgicouncinata, (, Mill, .)            False  \n",
       "8    (Sanionia, georgicouncinata, is, often, found,...            False  \n",
       "12   (The, local, fauna, is, diverse, and, includes...            False  \n",
       "19   (The, cosmopolitan, species, Hypogastrura, via...            False  \n",
       "20   (Eight, species, are, native, in, Antarctica, ...            False  \n",
       "25   (Eight, species, are, native, in, Antarctica, ...            False  \n",
       "28   (Two, species, are, non, -, indigenous, in, An...            False  \n",
       "31   (obtaining, seven, species, ,, with, a, new, r...            False  \n",
       "32   (These, species, are, Cryptopygus, antarcticus...            False  \n",
       "38   (This, group, is, formed, by, Archisotoma, bru...            False  \n",
       "40   (This, group, is, formed, by, Archisotoma, bru...            False  \n",
       "43   (The, same, happens, with, F., woyciechowskii,...            False  \n",
       "44   (Friesea, bispinosa, ,, F., grisea, ,, and, F....            False  \n",
       "49   (The, family, Tullbergiidae, was, represented,...            False  \n",
       "52   (Cryptopygus, antarcticus, antarcticus, was, f...            False  \n",
       "54   (The, least, favorable, substrates, for, C., a...            False  \n",
       "55   (Electrical, conductivity, had, a, negative, c...            False  \n",
       "57   (It, was, also, present, in, all, substrates, ...            False  \n",
       "59   (At, these, locations, ,, the, presence, of, l...            False  \n",
       "61   (Folsomotoma, octooculata, and, F., bispinosa,...            False  \n",
       "62   (The, other, four, species, included, in, this...            False  \n",
       "64   (The, other, four, species, included, in, this...            False  \n",
       "66   ((, 2014, ), for, the, maritime, Antarctica, r...            False  \n",
       "71   (In, addition, ,, four, other, species, —, Fri...            False  \n",
       "74   (In, addition, ,, four, other, species, —, Fri...            False  \n",
       "75   (The, most, abundant, species, was, Cryptopygu...            False  \n",
       "77   (Hypogastrura, viatica, and, Mesaphorura, macr...            False  \n",
       "79   (Its, presence, could, affect, the, native, Co...            False  \n",
       "81   (On, Barrientos, Island, ,, H., viatica, was, ...            False  \n",
       "83   (The, presence, of, M., macrochaeta, may, be, ...            False  \n",
       "85   (The, species, M., macrochaeta, was, reported,...            False  \n",
       "87   (Cryptopygus, antarcticus, antarcticus, has, a...            False  \n",
       "89   (Therefore, ,, mosses, are, less, consumed, by...            False  \n",
       "90   (Cryptopygus, antarcticus, antarcticus, Was, v...            False  \n",
       "95   (Friesea, bispinosa, and, F., woyciechowskii, ...            False  \n",
       "101  (—, Hypogastrura, viatica, and, Mesaphorura, m...            False  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SpeciesFound[SpeciesFound.InReferenceList ==False]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Second Document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 645,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T23:30:13.301106Z",
     "start_time": "2021-01-31T23:30:13.299043Z"
    }
   },
   "outputs": [],
   "source": [
    "# file = open(\"PDFs/thesis (3).txt\")\n",
    "# document2 = file.read()\n",
    "# file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 646,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T23:30:15.333418Z",
     "start_time": "2021-01-31T23:30:15.330825Z"
    }
   },
   "outputs": [],
   "source": [
    "# document2 = nlp_lg(document2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 647,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T23:30:17.479911Z",
     "start_time": "2021-01-31T23:30:17.476493Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# SpeciesFound2 = findSpecies(document2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 644,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-31T23:30:02.897598Z",
     "start_time": "2021-01-31T23:30:02.895012Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# SpeciesFound2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extracting and Saving Species for each Document in a Folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:13:49.397861Z",
     "start_time": "2021-02-08T03:13:49.395261Z"
    }
   },
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-08T03:13:50.903734Z",
     "start_time": "2021-02-08T03:13:50.897946Z"
    }
   },
   "outputs": [],
   "source": [
    "def extractSpeciesfromFolderofDocuments(folder, NLPmodel='large'):\n",
    "    filelist = [x for x in os.listdir(folder) if x.endswith('txt')]\n",
    "    for filename in filelist:\n",
    "        file = open(folder+'/'+ filename)\n",
    "        doc = file.read()\n",
    "        file.close()\n",
    "        if NLPmodel == 'small':\n",
    "            document = nlp(doc)\n",
    "        elif NLPmodel == 'medium':\n",
    "            document = nlp_md(doc)\n",
    "        elif NLPmodel == 'large':\n",
    "            document = nlp_lg(doc)\n",
    "            \n",
    "        df = findSpecies(document)\n",
    "        newfilename = re.sub('.txt', '-species.csv', filename)\n",
    "        df.to_csv(folder+'/'+ newfilename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-10T23:59:46.023699Z",
     "start_time": "2021-02-10T23:59:42.403154Z"
    }
   },
   "outputs": [],
   "source": [
    "extractSpeciesfromFolderofDocuments('SmallPDFs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T00:00:18.627722Z",
     "start_time": "2021-02-10T23:59:47.347452Z"
    }
   },
   "outputs": [],
   "source": [
    "extractSpeciesfromFolderofDocuments('PDFs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T00:00:28.325916Z",
     "start_time": "2021-02-11T00:00:19.888773Z"
    }
   },
   "outputs": [],
   "source": [
    "extractSpeciesfromFolderofDocuments('FraserSet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-27T01:28:06.980708Z",
     "start_time": "2021-02-27T01:27:58.855154Z"
    }
   },
   "outputs": [],
   "source": [
    "extractSpeciesfromFolderofDocuments('FraserSet/BodyOnly/Clean')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combining Species and Location into Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T00:02:37.558619Z",
     "start_time": "2021-02-11T00:02:37.553542Z"
    }
   },
   "outputs": [],
   "source": [
    "def extractSpeciesandLocations(tokenisedDoc):\n",
    "    df_species = findSpecies(tokenisedDoc)\n",
    "    df_locations = find_locations(tokenisedDoc)\n",
    "    df_locations = df_locations[df_locations.Found==True] #only include locations that were confirmed\n",
    "    df_speciesXlocation = pd.merge(df_species.assign(key=0), df_locations.assign(key=0), on='key', \n",
    "                               suffixes=('_species', '_location')).drop('key', axis=1)\n",
    "    return df_speciesXlocation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T00:02:39.079093Z",
     "start_time": "2021-02-11T00:02:39.072690Z"
    }
   },
   "outputs": [],
   "source": [
    "def extractSpeciesandLocationsfromFolderofDocuments(folder, NLPmodel='large'):\n",
    "    filelist = [x for x in os.listdir(folder) if x.endswith('txt')]\n",
    "    for filename in filelist:\n",
    "        file = open(folder+'/'+ filename)\n",
    "        doc = file.read()\n",
    "        file.close()\n",
    "        if NLPmodel == 'small':\n",
    "            document = nlp(doc)\n",
    "        elif NLPmodel == 'medium':\n",
    "            document = nlp_md(doc)\n",
    "        elif NLPmodel == 'large':\n",
    "            document = nlp_lg(doc)\n",
    "            \n",
    "        df = extractSpeciesandLocations(document)\n",
    "        newfilename = re.sub('.txt', '-matrix.csv', filename)\n",
    "        df.to_csv(folder+'/'+ newfilename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T01:38:12.314026Z",
     "start_time": "2021-02-11T01:37:42.100069Z"
    }
   },
   "outputs": [],
   "source": [
    "extractSpeciesandLocationsfromFolderofDocuments('SmallPDFs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T01:45:59.285796Z",
     "start_time": "2021-02-11T01:38:13.578851Z"
    }
   },
   "outputs": [],
   "source": [
    "extractSpeciesandLocationsfromFolderofDocuments('PDFs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-11T01:47:31.160094Z",
     "start_time": "2021-02-11T01:46:00.562342Z"
    }
   },
   "outputs": [],
   "source": [
    "extractSpeciesandLocationsfromFolderofDocuments('FraserSet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-26T08:24:27.970899Z",
     "start_time": "2021-02-26T08:23:04.692119Z"
    }
   },
   "outputs": [],
   "source": [
    "extractSpeciesandLocationsfromFolderofDocuments('FraserSet/BodyOnly/Clean')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sandbox"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are bits of code I did not end up using but were useful for testing aspects of strong matching"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### String Matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-20T02:17:49.778773Z",
     "start_time": "2021-01-20T02:17:49.775818Z"
    }
   },
   "outputs": [],
   "source": [
    "from difflib import SequenceMatcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T23:07:02.970479Z",
     "start_time": "2021-01-27T23:07:02.967989Z"
    }
   },
   "outputs": [],
   "source": [
    "import difflib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T23:24:07.132134Z",
     "start_time": "2021-01-27T23:24:06.976090Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Transantarctic Mountains', 'Merrick Mountains', 'Chester Mountains']"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "difflib.get_close_matches('the Transantarctic Mountains', SCARGlobalnames, cutoff=0.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T23:08:28.445859Z",
     "start_time": "2021-01-27T23:08:28.435446Z"
    }
   },
   "outputs": [],
   "source": [
    "def getCloseMatches(df, gazetteer, threshold=0.9):\n",
    "    for place in df[df.Found==False].Location.unique():\n",
    "        options = difflib.get_close_matches(place, gazetteer, cutoff=threshold, n=3)\n",
    "        if len(options)!=0:\n",
    "            print(place, options)\n",
    "#         else: print (place, 'None')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-27T23:08:32.425329Z",
     "start_time": "2021-01-27T23:08:30.232457Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mount Seuss ['Mount Suess']\n",
      "Dry Valleys ['Dry Valley']\n",
      "Mackay Glacier\n",
      " ['Mackay Glacier']\n",
      "Mcmurdo Dry Valleys ['McMurdo Dry Valleys']\n",
      "Western Antarctic Ice Sheet ['West Antarctic Ice Sheet']\n",
      "Granite Harbour B. ['Granite Harbour']\n",
      "Onyx river ['Onyx River']\n",
      "Transantarctic Mountain ['Transantarctic Mountains']\n",
      "Dry Valleys’ ['Dry Valley']\n",
      "Botany bay ['Botany Bay']\n",
      "Kar plateau ['Kar Plateau']\n",
      "McMurdo Dry Valley ['McMurdo Dry Valleys']\n",
      "Ross\n",
      "Island ['Ross Island']\n"
     ]
    }
   ],
   "source": [
    "getCloseMatches(locations_in_inital_set_lg2, nzgazAntartica, threshold=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-19T06:02:28.283705Z",
     "start_time": "2021-01-19T06:02:26.885534Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mcmurdo Dry Valleys ['McMurdo Dry Valleys']\n",
      "Western Antarctic Ice Sheet ['West Antarctic Ice Sheet']\n",
      "Mount Seuss ['Mount Suess']\n",
      "Granite Harbour B. ['Granite Harbour']\n",
      "Onyx river ['Onyx River']\n",
      "Transantarctic Mountain ['Transantarctic Mountains']\n",
      "Botany bay ['Botany Bay']\n"
     ]
    }
   ],
   "source": [
    "getCloseMatches(locations_in_inital_set_lg2, SCARNZnames, threshold=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-19T06:45:49.473699Z",
     "start_time": "2021-01-19T06:03:29.898226Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Antarctic ['Antarctica']\n",
      "Eastern Antarctica ['East Antarctica']\n",
      "East Antarctic ['East Antarctica']\n",
      "Antarctic Penisula ['Antarctic Peninsula']\n",
      "Antarctica Peninsula ['Antarctic Peninsula']\n",
      "Western Antarctica ['West Antarctica']\n",
      "Mcmurdo Dry Valleys ['McMurdo Dry Valleys']\n",
      "Western Antarctic Ice Sheet ['West Antarctic Ice Sheet']\n",
      "Sgr Rondane Mountains ['Sør Rondane Mountains']\n",
      "Mount Seuss ['Mount Suess']\n",
      "D. East Antarctica ['East Antarctica']\n",
      "Molodyezhnaya ['Molodezhnaya']\n",
      "South Shetland Island ['South Shetland Islands']\n",
      "West Antarctic ['West Antarctica']\n",
      "Granite Harbour B. ['Granite Harbour']\n",
      "Ongul Islands ['Ongul Island']\n",
      "Onyx river ['Onyx River']\n",
      "Transantarctic Mountain ['Transantarctic Mountains']\n",
      "Botany bay ['Botany Bay']\n"
     ]
    }
   ],
   "source": [
    "getCloseMatches(locations_in_inital_set_lg2, SCARGlobalnames, threshold=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-19T06:45:49.473699Z",
     "start_time": "2021-01-19T06:03:29.898226Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Antarctic ['Antarctica']\n",
      "Mcmurdo Dry Valleys ['McMurdo Dry Valleys']\n",
      "Western Antarctic Ice Sheet ['West Antarctic Ice Sheet']\n",
      "Sgr Rondane Mountains ['Sør Rondane Mountains']\n",
      "Mount Seuss ['Mount Suess']\n",
      "Granite Harbour B. ['Granite Harbour']\n",
      "Ongul Islands ['Ongul Island']\n",
      "Onyx river ['Onyx River']\n",
      "Transantarctic Mountain ['Transantarctic Mountains']\n",
      "Botany bay ['Botany Bay']\n"
     ]
    }
   ],
   "source": [
    "getCloseMatches(locations_in_inital_set_lg2, GeoNamesAnt, threshold=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-19T06:45:49.473699Z",
     "start_time": "2021-01-19T06:03:29.898226Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Antarctic ['Antarctica']\n",
      "Strandtmann ['Stratmann']\n",
      "Dry Valleys ['Dry Valley']\n",
      "North | Cape ['North Cape']\n",
      "Chathams Islands ['Chatham Islands', 'Chatham Island']\n",
      "Macquarie Islands ['Macquarie Island']\n",
      "earth ['Dearth']\n",
      "Lakes Rotorua ['Lake Rotorua']\n",
      "Waikawa Harbour | ['Waikawa Harbour']\n",
      "Mcmurdo Dry Valleys ['McMurdo Dry Valleys']\n",
      "Western Antarctic Ice Sheet ['West Antarctic Ice Sheet']\n",
      "Sgr Rondane Mountains ['Sør Rondane Mountains']\n",
      "Mount Seuss ['Mount Suess']\n",
      "Western Australia ['Western Austrailia', 'Westan Australia']\n",
      "(South America ['South America']\n",
      "Granite Harbour B. ['Granite Harbour']\n",
      "Ongul Islands ['Ongul Island']\n",
      "Onyx river ['Onyx River']\n",
      "Transantarctic Mountain ['Transantarctic Mountains']\n",
      "Oceans ['Oceanus', 'Oceanis', 'Ocean']\n",
      "Frozen ['Froen', 'Fozen']\n",
      "Botany bay ['Botany Bay']\n",
      "Grimmia ['Grimma']\n"
     ]
    }
   ],
   "source": [
    "getCloseMatches(locations_in_inital_set_lg2, GeoNamesUnique, threshold=0.9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing String Matching Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-19T20:34:38.201714Z",
     "start_time": "2021-01-19T20:34:38.199781Z"
    }
   },
   "outputs": [],
   "source": [
    "# pip install textdistance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-20T02:18:26.064251Z",
     "start_time": "2021-01-20T02:18:26.025654Z"
    }
   },
   "outputs": [],
   "source": [
    "import textdistance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-18T02:40:13.913573Z",
     "start_time": "2021-01-18T02:40:13.890109Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "North and South Victoria Land v Victoria Land\n",
      "---------------------------------------------\n",
      "sequencematcher ratio 0.6190476190476191\n",
      "ratcliff_obershelp: 0.6190476190476191\n",
      "hamming: 0.03448275862068961\n",
      "mlipns: 0.0\n",
      "levenshtein: 0.4482758620689655\n",
      "damerau_levenshtein: 0.4482758620689655\n",
      "jaro_winkler: 0.5194026918164849\n",
      "strcmp95: 0.589861479516652\n",
      "needleman_wunsch: 0.4482758620689655\n",
      "gotoh: 0.7307692307692307\n",
      "smith_waterman: 1.0\n"
     ]
    }
   ],
   "source": [
    "stringC = \"North and South Victoria Land\"\n",
    "stringD = \"Victoria Land\"\n",
    "print(stringC, \"v\", stringD)\n",
    "print('---------------------------------------------')\n",
    "print ('sequencematcher ratio',SequenceMatcher(None, stringC, stringD).ratio() )\n",
    "print('ratcliff_obershelp:', textdistance.ratcliff_obershelp.normalized_similarity(stringC, stringD))\n",
    "print('hamming:', textdistance.hamming.normalized_similarity(stringC, stringD))\n",
    "print('mlipns:', textdistance.mlipns.normalized_similarity(stringC, stringD))\n",
    "print('levenshtein:', textdistance.levenshtein.normalized_similarity(stringC, stringD))\n",
    "print('damerau_levenshtein:', textdistance.damerau_levenshtein.normalized_similarity(stringC, stringD))\n",
    "print('jaro_winkler:', textdistance.jaro_winkler.normalized_similarity(stringC, stringD))\n",
    "print('strcmp95:', textdistance.strcmp95.normalized_similarity(stringC, stringD))\n",
    "print('needleman_wunsch:', textdistance.needleman_wunsch.normalized_similarity(stringC, stringD))\n",
    "print('gotoh:', textdistance.gotoh.normalized_similarity(stringC, stringD))\n",
    "print('smith_waterman:', textdistance.smith_waterman.normalized_similarity(stringC, stringD))\n",
    "# print('jaccard:', textdistance.jaccard.normalized_similarity(stringC, stringD))\n",
    "# print('jaccard:', textdistance.jaccard.normalized_similarity(stringC, stringD))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-22T01:12:51.484458Z",
     "start_time": "2021-01-22T01:12:51.439375Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Western Antarctic Peninsula v Antarctic Peninsula\n",
      "---------------------------------------------\n",
      "sequencematcher ratio 0.8260869565217391\n",
      "ratcliff_obershelp: 0.8260869565217391\n",
      "hamming: 0.0\n",
      "mlipns: 0.0\n",
      "levenshtein: 0.7037037037037037\n",
      "damerau_levenshtein: 0.7037037037037037\n",
      "jaro_winkler: 0.7784275503573749\n",
      "strcmp95: 0.7784275503573749\n",
      "needleman_wunsch: 0.7037037037037037\n",
      "gotoh: 0.9\n",
      "smith_waterman: 1.0\n"
     ]
    }
   ],
   "source": [
    "stringC = \"Western Antarctic Peninsula\"\n",
    "stringD = \"Antarctic Peninsula\"\n",
    "print(stringC, \"v\", stringD)\n",
    "print('---------------------------------------------')\n",
    "print ('sequencematcher ratio',SequenceMatcher(None, stringC, stringD).ratio() )\n",
    "print('ratcliff_obershelp:', textdistance.ratcliff_obershelp.normalized_similarity(stringC, stringD))\n",
    "print('hamming:', textdistance.hamming.normalized_similarity(stringC, stringD))\n",
    "print('mlipns:', textdistance.mlipns.normalized_similarity(stringC, stringD))\n",
    "print('levenshtein:', textdistance.levenshtein.normalized_similarity(stringC, stringD))\n",
    "print('damerau_levenshtein:', textdistance.damerau_levenshtein.normalized_similarity(stringC, stringD))\n",
    "print('jaro_winkler:', textdistance.jaro_winkler.normalized_similarity(stringC, stringD))\n",
    "print('strcmp95:', textdistance.strcmp95.normalized_similarity(stringC, stringD))\n",
    "print('needleman_wunsch:', textdistance.needleman_wunsch.normalized_similarity(stringC, stringD))\n",
    "print('gotoh:', textdistance.gotoh.normalized_similarity(stringC, stringD))\n",
    "print('smith_waterman:', textdistance.smith_waterman.normalized_similarity(stringC, stringD))\n",
    "# print('jaccard:', textdistance.jaccard.normalized_similarity(stringC, stringD))\n",
    "# print('jaccard:', textdistance.jaccard.normalized_similarity(stringC, stringD))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-22T01:12:57.710423Z",
     "start_time": "2021-01-22T01:12:57.698622Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8260869565217391"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textdistance.ratcliff_obershelp.normalized_similarity(stringC, stringD)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-22T01:12:59.108273Z",
     "start_time": "2021-01-22T01:12:59.104260Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8260869565217391"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SequenceMatcher(None, stringC, stringD).ratio()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-18T02:43:27.337605Z",
     "start_time": "2021-01-18T02:43:27.305165Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Western Antarctic Ice Sheet v West Antarctic Ice Sheet\n",
      "---------------------------------------------\n",
      "sequencematcher ratio 0.9411764705882353\n",
      "ratcliff_obershelp: 0.9411764705882353\n",
      "hamming: 0.2592592592592593\n",
      "mlipns: 0.0\n",
      "levenshtein: 0.8888888888888888\n",
      "damerau_levenshtein: 0.8888888888888888\n",
      "jaro_winkler: 0.9527777777777777\n",
      "strcmp95: 0.9527777777777777\n",
      "needleman_wunsch: 0.8888888888888888\n",
      "gotoh: 0.9625\n",
      "smith_waterman: 0.875\n"
     ]
    }
   ],
   "source": [
    "stringC = \"Western Antarctic Ice Sheet\"\n",
    "stringD = \"West Antarctic Ice Sheet\"\n",
    "print(stringC, \"v\", stringD)\n",
    "print('---------------------------------------------')\n",
    "print ('sequencematcher ratio',SequenceMatcher(None, stringC, stringD).ratio() )\n",
    "print('ratcliff_obershelp:', textdistance.ratcliff_obershelp.normalized_similarity(stringC, stringD))\n",
    "print('hamming:', textdistance.hamming.normalized_similarity(stringC, stringD))\n",
    "print('mlipns:', textdistance.mlipns.normalized_similarity(stringC, stringD))\n",
    "print('levenshtein:', textdistance.levenshtein.normalized_similarity(stringC, stringD))\n",
    "print('damerau_levenshtein:', textdistance.damerau_levenshtein.normalized_similarity(stringC, stringD))\n",
    "print('jaro_winkler:', textdistance.jaro_winkler.normalized_similarity(stringC, stringD))\n",
    "print('strcmp95:', textdistance.strcmp95.normalized_similarity(stringC, stringD))\n",
    "print('needleman_wunsch:', textdistance.needleman_wunsch.normalized_similarity(stringC, stringD))\n",
    "print('gotoh:', textdistance.gotoh.normalized_similarity(stringC, stringD))\n",
    "print('smith_waterman:', textdistance.smith_waterman.normalized_similarity(stringC, stringD))\n",
    "# print('jaccard:', textdistance.jaccard.normalized_similarity(stringC, stringD))\n",
    "# print('jaccard:', textdistance.jaccard.normalized_similarity(stringC, stringD))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-18T03:19:45.501689Z",
     "start_time": "2021-01-18T03:19:45.477361Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ross Sea Region of Antarctica  v Ross Sea\n",
      "---------------------------------------------\n",
      "sequencematcher ratio 0.42105263157894735\n",
      "ratcliff_obershelp: 0.42105263157894735\n",
      "hamming: 0.2666666666666667\n",
      "mlipns: 0.0\n",
      "levenshtein: 0.2666666666666667\n",
      "damerau_levenshtein: 0.2666666666666667\n",
      "jaro_winkler: 0.8533333333333333\n",
      "strcmp95: 0.8551724137931035\n",
      "needleman_wunsch: 0.26666666666666666\n",
      "gotoh: 0.41249999999999987\n",
      "smith_waterman: 0.0\n"
     ]
    }
   ],
   "source": [
    "stringC = \"Ross Sea Region of Antarctica \"\n",
    "stringD = \"Ross Sea\"\n",
    "print(stringC, \"v\", stringD)\n",
    "print('---------------------------------------------')\n",
    "print ('sequencematcher ratio',SequenceMatcher(None, stringC, stringD).ratio() )\n",
    "print('ratcliff_obershelp:', textdistance.ratcliff_obershelp.normalized_similarity(stringC, stringD))\n",
    "print('hamming:', textdistance.hamming.normalized_similarity(stringC, stringD))\n",
    "print('mlipns:', textdistance.mlipns.normalized_similarity(stringC, stringD))\n",
    "print('levenshtein:', textdistance.levenshtein.normalized_similarity(stringC, stringD))\n",
    "print('damerau_levenshtein:', textdistance.damerau_levenshtein.normalized_similarity(stringC, stringD))\n",
    "print('jaro_winkler:', textdistance.jaro_winkler.normalized_similarity(stringC, stringD))\n",
    "print('strcmp95:', textdistance.strcmp95.normalized_similarity(stringC, stringD))\n",
    "print('needleman_wunsch:', textdistance.needleman_wunsch.normalized_similarity(stringC, stringD))\n",
    "print('gotoh:', textdistance.gotoh.normalized_similarity(stringC, stringD))\n",
    "print('smith_waterman:', textdistance.smith_waterman.normalized_similarity(stringC, stringD))\n",
    "# print('jaccard:', textdistance.jaccard.normalized_similarity(stringC, stringD))\n",
    "# print('jaccard:', textdistance.jaccard.normalized_similarity(stringC, stringD))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-20T02:18:33.567176Z",
     "start_time": "2021-01-20T02:18:33.562628Z"
    }
   },
   "outputs": [],
   "source": [
    "def compareString (string, possibilities, threshold=0.8):\n",
    "    options = []\n",
    "    metrics = ['SequenceMatcher', 'Jaro', 'Gotoh', 'Smith-Waterman']\n",
    "    for candidate in possibilities:\n",
    "        scoreSM = SequenceMatcher(None, string, candidate).ratio()\n",
    "        scoreJ = textdistance.jaro_winkler.normalized_similarity(string, candidate)\n",
    "        scoreG = textdistance.gotoh.normalized_similarity(string, candidate) \n",
    "        scoreSW = textdistance.smith_waterman.normalized_similarity(string, candidate) \n",
    "        for i, score in enumerate([scoreSM,scoreJ,scoreG,scoreSW]):\n",
    "            if score > threshold:\n",
    "                options.append([candidate, score, metrics[i]])\n",
    "    return options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-01-21T00:34:24.896999Z",
     "start_time": "2021-01-21T00:23:33.210890Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Ross', 0.8275862068965518, 'Jaro'],\n",
       " ['Ross Sea', 0.8551724137931035, 'Jaro']]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compareString (\"Ross Sea Region of Antarctica\", nzgaz)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "287.390625px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "144px",
    "left": "1350px",
    "right": "20px",
    "top": "118px",
    "width": "350px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}