webScraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import httpx\n",
    "import time\n",
    "import json\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "scrapedData = {}\n",
    "\n",
    "visitedURLs = []\n",
    "\n",
    "visiteddomains=[]\n",
    "\n",
    "rss_feeds=[]\n",
    "\n",
    "emails=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('scrapedDatabase/scrapedData.json', 'r') as file:\n",
    "    scrapedData = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('scrapedDatabase/visitedURLs.json', 'r') as file:\n",
    "    visitedURLs = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('mixed types', 3.666666666666667), ('natural numbers', 4.0), ('strict inequations', 4.0), ('nonstrict inequations', 4.0), ('upper bounds', 4.0), ('linear constraints', 4.5), ('minimal set', 4.666666666666666), ('minimal supporting set', 7.666666666666666), ('linear diophantine equations', 8.5), ('minimal generating sets', 8.666666666666666)]\n"
     ]
    }
   ],
   "source": [
    "import RAKE\n",
    "\n",
    "text_en = (\n",
    "    'Compatibility of systems of linear constraints over the set of '\n",
    "    'natural numbers. Criteria of compatibility of a system of linear '\n",
    "    'Diophantine equations, strict inequations, and nonstrict inequations '\n",
    "    'are considered. Upper bounds for components of a minimal set of '\n",
    "    'solutions and algorithms of construction of minimal generating sets '\n",
    "    'of solutions for all types of systems are given. These criteria and '\n",
    "    'the corresponding algorithms for constructing a minimal supporting '\n",
    "    'set of solutions can be used in solving all the considered types of '\n",
    "    'systems and systems of mixed types.'\n",
    ")\n",
    "\n",
    "rake = RAKE.Rake('SmartStoplist.txt')\n",
    "\n",
    "\n",
    "def srt(lst):\n",
    "    lst.sort(key=lambda x:x[1])\n",
    "    return lst\n",
    "\n",
    "print(srt(rake.run(text_en))[-10:])\n",
    "\n",
    "def get_top_kwds(text):\n",
    "    return srt(rake.run(text.replace('\\n',' ')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "strictness=3.5\n",
    "def find_topic(sentence):\n",
    "    '''\n",
    "    # Process the sentence\n",
    "    doc = nlp(sentence)\n",
    "    \n",
    "    # Extract the most relevant noun or noun chunk\n",
    "    for chunk in doc.noun_chunks:\n",
    "        return chunk.text\n",
    "        '''\n",
    "    top_kwds= get_top_kwds(sentence)\n",
    "    return top_kwds[-len(top_kwds)//10:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4.element import Comment\n",
    "\n",
    "\n",
    "def tag_visible(element):\n",
    "    if element.parent.name in ['style', 'script', 'head', 'title', 'meta', '[document]']:\n",
    "        return False\n",
    "    if isinstance(element, Comment):\n",
    "        return False\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def find_document_topic(soup):\n",
    "    paragraphs = soup.findAll(string=True)\n",
    "    visible_texts = filter(tag_visible, paragraphs)  \n",
    "\n",
    "    sentence = u\" \".join(t.strip() for t in visible_texts)\n",
    "\n",
    "    topic = find_topic(sentence)\n",
    "\n",
    "    return topic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def extract_links(soup):\n",
    "    # Find all the anchor tags with href attributes\n",
    "    links = [a['href'] for a in soup.find_all('a', href=True)]\n",
    "\n",
    "    return links\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addURL_kwd(kwd,url,queries={},fragment=''):\n",
    "    if kwd not in scrapedData:\n",
    "        scrapedData[kwd]={\n",
    "            url: queries\n",
    "        }\n",
    "        return\n",
    "    if url not in scrapedData[kwd]:\n",
    "        scrapedData[kwd][url]=queries\n",
    "    scrapedData[kwd][url].update(queries)\n",
    "    return\n",
    "\n",
    "def deconstruct_url(url):\n",
    "    #assuming URL is not malformed.\n",
    "    scheme,*url=url.split('://')\n",
    "    url='://'.join(url)\n",
    "    url,*fragment=url.split('#')\n",
    "    fragment=(fragment+[''])[0]\n",
    "    url=url.replace('&','?').strip('?')\n",
    "    auth_n_path,*strqueries=url.split('?')\n",
    "    queries=dict([tuple(query.split('=')) for query in strqueries])\n",
    "    if auth_n_path[-1]=='/':\n",
    "        auth_n_path=auth_n_path[:-1]\n",
    "    auth,*path=auth_n_path.split('/')\n",
    "    return scheme,auth,path,queries,fragment\n",
    "\n",
    "def reconstruct_url(scheme='',auth='',path=[],queries={},fragment=''):\n",
    "    return f'{scheme}://{auth}/'+ '/'.join(path)+ '?'*(len(queries)>0)+ '&'.join([f'{a}={b}' for a,b in queries.items()])+ '#'*(fragment!='')+fragment\n",
    "\n",
    "def hasScheme(url):\n",
    "    return '://' in url\n",
    "\n",
    "\n",
    "def contextify_url(previous_url,current_url):\n",
    "    if current_url=='':\n",
    "        return previous_url\n",
    "    if hasScheme(current_url):\n",
    "        return current_url\n",
    "    pscheme,pauth,ppath,pqueries,pfragment = deconstruct_url(previous_url)\n",
    "\n",
    "    current_url,*cfragment=current_url.split('#')\n",
    "    cfragment=(cfragment+[''])[0]\n",
    "    current_url=current_url.replace('&','?').strip('?')\n",
    "    current_url,*cstrqueries=current_url.split('?')\n",
    "    cqueries=dict([tuple(query.split('=')) for query in cstrqueries])\n",
    "    cauth_or_pathswitcher,*cpath=current_url.split('/')\n",
    "\n",
    "    if '.' in cauth_or_pathswitcher: # balls.com/blah?he=he#haha\n",
    "        return reconstruct_url(pscheme,cauth_or_pathswitcher,cpath,cqueries,cfragment)\n",
    "    cpath=[cauth_or_pathswitcher]+cpath\n",
    "    if cauth_or_pathswitcher=='':   # /blah?he=he#haha\n",
    "        return reconstruct_url(pscheme,pauth,cpath,cqueries,cfragment)\n",
    "    return reconstruct_url(pscheme,pauth,ppath+cpath,cqueries,cfragment)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('https', 'balls.com', [], {}, '')\n",
      "https://balls.com/\n"
     ]
    }
   ],
   "source": [
    "ex_url='https://balls.com'\n",
    "a=deconstruct_url(ex_url)\n",
    "print(a)\n",
    "b=reconstruct_url(*a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://balls.com/secret_location/tower'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ex_goto='secret_location/tower'\n",
    "contextify_url(ex_url,ex_goto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "scrapestack=[]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pagehashes=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "blacklist=[\n",
    "    '.pdf',\n",
    "    '.jpeg'\n",
    "]\n",
    "def scrape(url):\n",
    "    global scrapestack\n",
    "    if url in visitedURLs:\n",
    "        print('URL was already finished.')\n",
    "        return\n",
    "    \n",
    "    if 'mailto' in url and url not in emails:\n",
    "        emails.append(url)\n",
    "        print('That was an email!')\n",
    "        return\n",
    "    \n",
    "    if 'feed.xml' in url and url not in rss_feeds:\n",
    "        rss_feeds.append(url)\n",
    "        print('What was an RSS FEED!')\n",
    "        return\n",
    "\n",
    "    visitedURLs.append(url)\n",
    "\n",
    "    with open('scrapedDatabase/visitedURLs.json', 'w') as file:\n",
    "        json.dump(visitedURLs, file, indent=4)\n",
    "\n",
    "    \n",
    "    scheme,auth,path,queries,fragment=deconstruct_url(url)\n",
    "    print(scheme,auth,path,queries,fragment)\n",
    "\n",
    "    if (dom:=f'{scheme}://{auth}') not in visiteddomains:\n",
    "        visiteddomains.append(dom)\n",
    "    \n",
    "    try:\n",
    "        response = httpx.get(url, timeout=2)\n",
    "    except Exception as e:\n",
    "        print('Error encountered in response for url '+url+': '+str(e))\n",
    "        return\n",
    "    try:\n",
    "        response.raise_for_status() # Raise an exception for HTTP errors\n",
    "    except Exception as e:\n",
    "        print(f\"Error fetching the URL: {e}\")\n",
    "        for i in str(e).replace('\"',' ').replace('\\'',' ').split(' ')[1:]:\n",
    "            if hasScheme(i):\n",
    "                scrapestack.append(i)\n",
    "        return\n",
    "    if (hsh:=hash(response.content)) in pagehashes:\n",
    "        print('This page has been encountered before!')\n",
    "        return\n",
    "    pagehashes.append(hsh)\n",
    "    # Parse the HTML content\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "    links = extract_links(soup)\n",
    "\n",
    "    topics = find_document_topic(soup)\n",
    "    try:\n",
    "        topics+=(''.join(map(lambda x: x if x.isalnum() or x==' ' else '', soup.title.text))).split(' ')\n",
    "    except:\n",
    "        print(\"Document had no title.\")\n",
    "\n",
    "    for topic in topics:\n",
    "        if len(topic)==2:\n",
    "            topic=topic[0].lower().replace('-',' ')\n",
    "\n",
    "            if ('  ' not in ''.join([a if a.isalpha() else ' ' for a in topic])):\n",
    "                print(topic)\n",
    "                for t in topic.split(' '):\n",
    "                    addURL_kwd(t,reconstruct_url(scheme,auth,path),queries=queries)\n",
    "\n",
    "    for link in links:\n",
    "        try:\n",
    "            scrapestack.append(contextify_url(url,link))\n",
    "        except:\n",
    "            print(f'There was a problem while contextifying link {link} with url {url}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https neocities.org [] {} \n",
      "fast site performance\n",
      "favorite desktop editor\n",
      "command line tool\n",
      "drag-n-drop\n",
      "caching anycast cdn\n",
      "routinely out-perform\n",
      "pricey cloud services\n",
      "static web hosting\n",
      "fast static hosting\n",
      "in-browser html editor\n",
      "offer free static web hosting\n",
      "great in-browser html editor\n"
     ]
    }
   ],
   "source": [
    "\n",
    "startingURL = 'https://neocities.org/'\n",
    "scrape(startingURL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save():\n",
    "    with open('scrapedDatabase/scrapedData.json', 'w') as file:\n",
    "        json.dump(scrapedData, file, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0/48 remaining!)\n",
      "https://neocities.org/#!\n",
      "https neocities.org [] {} !\n",
      "fast site performance\n",
      "favorite desktop editor\n",
      "command line tool\n",
      "drag-n-drop\n",
      "caching anycast cdn\n",
      "routinely out-perform\n",
      "pricey cloud services\n",
      "static web hosting\n",
      "fast static hosting\n",
      "in-browser html editor\n",
      "offer free static web hosting\n",
      "great in-browser html editor\n",
      "\n",
      "\n",
      "(1/95 remaining!)\n",
      "https://neocities.org//\n",
      "https neocities.org [''] {} \n",
      "fast site performance\n",
      "favorite desktop editor\n",
      "command line tool\n",
      "drag-n-drop\n",
      "caching anycast cdn\n",
      "routinely out-perform\n",
      "pricey cloud services\n",
      "static web hosting\n",
      "fast static hosting\n",
      "in-browser html editor\n",
      "offer free static web hosting\n",
      "great in-browser html editor\n",
      "\n",
      "\n",
      "Saving...\n",
      "Saved!\n",
      "(2/142 remaining!)\n",
      "https://neocities.org//browse\n",
      "https neocities.org ['', 'browse'] {} \n",
      "\n",
      "\n",
      "(3/1092 remaining!)\n",
      "https://neocities.org//activity\n",
      "https neocities.org ['', 'activity'] {} \n",
      "Error encountered in response for url https://neocities.org//activity: The read operation timed out\n",
      "\n",
      "\n",
      "(4/1091 remaining!)\n",
      "https://neocities.org//tutorials\n",
      "https neocities.org ['', 'tutorials'] {} \n",
      "great beginners language\n",
      "fun interactive tutorial\n",
      "complicated server backends\n",
      "default webgl renderer\n",
      "basic building block\n",
      "pre-built css styles\n",
      "building virtual reality experiences\n",
      "allowing gpu-accelerated usage\n",
      "\n",
      "\n",
      "(5/1136 remaining!)\n",
      "https://neocities.org//supporter\n",
      "https neocities.org ['', 'supporter'] {} \n",
      "email support costs\n",
      "web building experience\n",
      "appeasing ad companies\n",
      "cache purging/reloading\n",
      "create web sites\n",
      "credit card information directly\n",
      "bring back creative web sites\n",
      "\n",
      "\n",
      "(6/1156 remaining!)\n",
      "https://neocities.org//signin\n",
      "https neocities.org ['', 'signin'] {} \n",
      "open source\n",
      "i\n",
      "\n",
      "\n",
      "(7/1179 remaining!)\n",
      "https://neocities.org//\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(8/1178 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(9/1177 remaining!)\n",
      "https://neocities.org//cli\n",
      "https neocities.org ['', 'cli'] {} \n",
      "site api key\n",
      "sudo gem install neocities\n",
      "git/hooks/pre-push\n",
      "\n",
      "\n",
      "(10/1200 remaining!)\n",
      "https://distantskies.neocities.org\n",
      "https distantskies.neocities.org [] {} \n",
      "nes emulator\n",
      "nintendo switch\n",
      "favorite nes game\n",
      "nintendo switch online\n",
      "great fan sites\n",
      "nintendo entertainment system\n",
      "called ゴッド・スレイヤー はるか天空のソナタ\n",
      "\n",
      "\n",
      "(11/1214 remaining!)\n",
      "https://kekkonen.neocities.org\n",
      "https kekkonen.neocities.org [] {} \n",
      "ei vai vähemmistöjen haukkuminen\n",
      "älä vedä herneitä nenään\n",
      "\n",
      "\n",
      "(12/1214 remaining!)\n",
      "https://gifypet.neocities.org\n",
      "https gifypet.neocities.org [] {} \n",
      "gifypet adoption center\n",
      "iframe auto updates\n",
      "melo-phonic hyper systems\n",
      "\n",
      "\n",
      "(13/1229 remaining!)\n",
      "https://civilservant.io\n",
      "https civilservant.io [] {} \n",
      "design award finalist\n",
      "civilservant helps online communities test\n",
      "conduct industry-independent evaluations\n",
      "remaking large-scale behavioral research\n",
      "\n",
      "\n",
      "(14/1272 remaining!)\n",
      "https://that1archive.neocities.org\n",
      "https that1archive.neocities.org [] {} \n",
      "fbi file george christopher\n",
      "dorothy hunt died onboard\n",
      "forensic linux distribution geared\n",
      "fbi file bill campbell\n",
      "top officials counterterrorism exercise\n",
      "fbi file bremer kidnapping\n",
      "fbi file daniel schorr\n",
      "fbi file gareth williams\n",
      "fbi file patricia ramsey\n",
      "fbi file percy sutton\n",
      "fbi file andrew walker\n",
      "fbi file daniel aaron\n",
      "fbi file marshall perlin\n",
      "fbi file bennett cerf\n",
      "fbi file art linkletter\n",
      "fbi social networking sites\n",
      "katyn forest massacre hearings\n",
      "fbi file david petraeus\n",
      "full cia crest database upload\n",
      "fbi file counterterrorism policy directive\n",
      "government ethics social media policies\n",
      "foia request rendition planes cia\n",
      "fbi file tattoo recognition database\n",
      "fbi file rosenberg-sobell committee\n",
      "fbi file thomas dalesandro jr\n",
      "trump private uk intelligence allegations\n",
      "fbi file trump management company\n",
      "fbi file richard charles holbrooke\n",
      "fbi file national socialist movement\n",
      "fbi file ghetto informant program\n",
      "foia request daniel sullivan fbi\n",
      "fbi file national wrestling alliance\n",
      "dhs email system documentation/manual\n",
      "fbi file project megiddo supplementals\n",
      "fbi file westboro baptist church\n",
      "fbi file family research council\n",
      "leaked memo reveals trump admin\n",
      "foia request ed reinecke fbi\n",
      "foia request rendition planes fbi\n",
      "fbi file john marshall richman\n",
      "fbi file trump manamgent company\n",
      "fbi file trump taj mahal\n",
      "fbi file senator jesse helms\n",
      "fbi file patricio fernandez flores\n",
      "fbi file ralph andrew cobb\n",
      "fbi file white supremacist infiltration\n",
      "fbi file daniel david rostenkowski\n",
      "katyn forest massacre final report\n",
      "katyn forest massacre interim report\n",
      "fbi file james sloss ackerman\n",
      "fbi file fbi recreation association\n",
      "fbi file john marks templeton\n",
      "spy chief reveals secret strategies\n",
      "fbi file david howell petraeus\n",
      "fbi file patricia hearst kidnapping\n",
      "fbi file henry steele commager\n",
      "fbi file charles wright mills\n",
      "fbi file william bradford shockley\n",
      "fbi file edward bennett williams\n",
      "fbi file william king harvey\n",
      "fbi file george hoben estabrooks\n",
      "generation identification monthly fact sheets\n",
      "fbi insider threat training program\n",
      "foia requset rudolf abel fbi\n",
      "fbi file fred leroy wineland\n",
      "fbi file art students league\n",
      "fbi file mary ritter beard\n",
      "fbi file huey percy newton\n",
      "army counterintelligence investigations evidence handling\n",
      "fbi file european-american unity\n",
      "fbi file american family association\n",
      "fbi file eddie lee long\n",
      "fbi file chiang kai-shek\n",
      "fbi file peter carey nowell\n",
      "fbi file frederick jackson turner\n",
      "fbi file willie roy evans\n",
      "fbi file pulse nightclub shooting\n",
      "fbi file gloria agrin josephson\n",
      "fbi file gregory scarpa sr\n",
      "fbi file joseph medicine crow\n",
      "fbi file gay liberation front\n",
      "fbi file guido rafael st\n",
      "fbi file francis parker yockey\n",
      "fbi file chi chi quintero\n",
      "laramie county sheriff nodapl materials\n",
      "elizebeth smith friedman oral histories\n",
      "hhs secretary nominee tom price\n",
      "fbi technology based tagging tracking\n",
      "leaked mass surveillance/immigration bill\n",
      "nikki haley hatch act violation\n",
      "fbi file german american federation/bund\n",
      "\n",
      "\n",
      "(15/1810 remaining!)\n",
      "https://lainzine.org\n",
      "https lainzine.org [] {} \n",
      "real\n",
      "\n",
      "\n",
      "(16/1818 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(17/1817 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(18/1816 remaining!)\n",
      "https://neocities.org//supporter\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(19/1815 remaining!)\n",
      "https://neocities.org//donate\n",
      "https neocities.org ['', 'donate'] {} \n",
      "open source\n",
      "awesome web site\n",
      "independent internet alive\n",
      "\n",
      "\n",
      "(20/1840 remaining!)\n",
      "https://neocities.org//cli\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(21/1839 remaining!)\n",
      "https://neocities.org//api\n",
      "https neocities.org ['', 'api'] {} \n",
      "common http libraries\n",
      "making http requests\n",
      "conduct regular backups\n",
      "command line usage\n",
      "command line interface\n",
      "command-line tool\n",
      "entire request stays\n",
      "madam frp manual\n",
      "disk space limit\n",
      "limit recurring site updates\n",
      "client-side http auth\n",
      "provide updated air quality information\n",
      "\n",
      "\n",
      "(22/1881 remaining!)\n",
      "https://github.com/neocities\n",
      "https github.com ['neocities'] {} \n",
      "\n",
      "\n",
      "(23/2035 remaining!)\n",
      "http://www.wired.com/2013/07/neocities/\n",
      "http www.wired.com ['2013', '07', 'neocities'] {} \n",
      "Error fetching the URL: Redirect response '301 Moved Permanently' for url 'http://www.wired.com/2013/07/neocities/'\n",
      "Redirect location: 'https://www.wired.com/2013/07/neocities/'\n",
      "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/301\n",
      "\n",
      "\n",
      "(24/2037 remaining!)\n",
      "http://www.fastcodesign.com/3037028/why-the-internet-is-time-traveling-back-to-1994\n",
      "http www.fastcodesign.com ['3037028', 'why-the-internet-is-time-traveling-back-to-1994'] {} \n",
      "Error fetching the URL: Redirect response '301 Moved Permanently' for url 'http://www.fastcodesign.com/3037028/why-the-internet-is-time-traveling-back-to-1994'\n",
      "Redirect location: 'https://www.fastcodesign.com/3037028/why-the-internet-is-time-traveling-back-to-1994'\n",
      "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/301\n",
      "\n",
      "\n",
      "(25/2039 remaining!)\n",
      "http://motherboard.vice.com/blog/neocities-is-recreating-the-garish-web-10-creativity-of-geocities\n",
      "http motherboard.vice.com ['blog', 'neocities-is-recreating-the-garish-web-10-creativity-of-geocities'] {} \n",
      "Error fetching the URL: Redirect response '301 Moved Permanently' for url 'http://motherboard.vice.com/blog/neocities-is-recreating-the-garish-web-10-creativity-of-geocities'\n",
      "Redirect location: 'https://motherboard.vice.com/blog/neocities-is-recreating-the-garish-web-10-creativity-of-geocities'\n",
      "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/301\n",
      "\n",
      "\n",
      "(26/2041 remaining!)\n",
      "http://arstechnica.com/tech-policy/2014/05/web-host-gives-fcc-a-28-8kbps-slow-lane-in-net-neutrality-protest/\n",
      "http arstechnica.com ['tech-policy', '2014', '05', 'web-host-gives-fcc-a-28-8kbps-slow-lane-in-net-neutrality-protest'] {} \n",
      "Error fetching the URL: Redirect response '301 Moved Permanently' for url 'http://arstechnica.com/tech-policy/2014/05/web-host-gives-fcc-a-28-8kbps-slow-lane-in-net-neutrality-protest/'\n",
      "Redirect location: 'https://arstechnica.com:443/tech-policy/2014/05/web-host-gives-fcc-a-28-8kbps-slow-lane-in-net-neutrality-protest/'\n",
      "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/301\n",
      "\n",
      "\n",
      "(27/2043 remaining!)\n",
      "http://web.appstorm.net/reviews/web-dev/neocities-the-free-place-to-code-your-own-site-from-scratch\n",
      "http web.appstorm.net ['reviews', 'web-dev', 'neocities-the-free-place-to-code-your-own-site-from-scratch'] {} \n",
      "Error encountered in response for url http://web.appstorm.net/reviews/web-dev/neocities-the-free-place-to-code-your-own-site-from-scratch: [Errno 11001] getaddrinfo failed\n",
      "\n",
      "\n",
      "(28/2042 remaining!)\n",
      "https://neocities.org//press\n",
      "https neocities.org ['', 'press'] {} \n",
      "independent web movement\n",
      "lost individual creativity\n",
      "8kbps slow lane\n",
      "partial websites online\n",
      "proficient software developer\n",
      "early personal websites\n",
      "junior web wizards\n",
      "pure nostalgia trips\n",
      "retro websites inspired\n",
      "online sandbox world\n",
      "cost-free hosting\n",
      "constantly rushing river\n",
      "command line prompt\n",
      "turned en masse\n",
      "current pre-formatted\n",
      "lowly html website\n",
      "fcc chairman tom wheeler\n",
      "well-defined design policies\n",
      "free web-hosting system neocities\n",
      "in-browser html editor\n",
      "web hosting service called neocities\n",
      "continent-spanning roller coasters\n",
      "proven financially self-sustaining\n",
      "internal ip address range\n",
      "\n",
      "\n",
      "(29/2080 remaining!)\n",
      "https://neocities.org/#new\n",
      "https neocities.org [] {} new\n",
      "fast site performance\n",
      "favorite desktop editor\n",
      "command line tool\n",
      "drag-n-drop\n",
      "caching anycast cdn\n",
      "routinely out-perform\n",
      "pricey cloud services\n",
      "static web hosting\n",
      "fast static hosting\n",
      "in-browser html editor\n",
      "offer free static web hosting\n",
      "great in-browser html editor\n",
      "\n",
      "\n",
      "(30/2127 remaining!)\n",
      "https://neocities.org//supporter\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(31/2126 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(32/2125 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(33/2124 remaining!)\n",
      "https://neocities.org//about\n",
      "https neocities.org ['', 'about'] {} \n",
      "including bitcoin\n",
      "web site\n",
      "share code back\n",
      "free web page\n",
      "easy site downloading\n",
      "generic social networks\n",
      "monitored addiction machines\n",
      "completely customized profile\n",
      "free web site option\n",
      "\n",
      "\n",
      "(34/2148 remaining!)\n",
      "https://neocities.org///blog.neocities.org\n",
      "https neocities.org ['', '', 'blog.neocities.org'] {} \n",
      "Error fetching the URL: Client error '404 Not Found' for url 'https://neocities.org///blog.neocities.org'\n",
      "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404\n",
      "\n",
      "\n",
      "(35/2149 remaining!)\n",
      "https://neocities.org\n",
      "https neocities.org [] {} \n",
      "fast site performance\n",
      "favorite desktop editor\n",
      "command line tool\n",
      "drag-n-drop\n",
      "caching anycast cdn\n",
      "routinely out-perform\n",
      "pricey cloud services\n",
      "static web hosting\n",
      "fast static hosting\n",
      "in-browser html editor\n",
      "offer free static web hosting\n",
      "great in-browser html editor\n",
      "\n",
      "\n",
      "(36/2196 remaining!)\n",
      "https://github.com/neocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(37/2195 remaining!)\n",
      "https://twitter.com/neocities\n",
      "https twitter.com ['neocities'] {} \n",
      "Error fetching the URL: Redirect response '302 Found' for url 'https://twitter.com/neocities'\n",
      "Redirect location: 'https://x.com/neocities'\n",
      "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/302\n",
      "\n",
      "\n",
      "(38/2197 remaining!)\n",
      "https://www.facebook.com/neocities\n",
      "https www.facebook.com ['neocities'] {} \n",
      "\n",
      "\n",
      "(39/2196 remaining!)\n",
      "https://neocities.org//about\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(40/2195 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(41/2194 remaining!)\n",
      "https://neocities.org//cli\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(42/2193 remaining!)\n",
      "https://neocities.org///blog.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(43/2192 remaining!)\n",
      "https://neocities.org//api\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(44/2191 remaining!)\n",
      "https://neocities.org//press\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(45/2190 remaining!)\n",
      "http://status.neocitiesops.net/\n",
      "http status.neocitiesops.net [] {} \n",
      "Error fetching the URL: Redirect response '308 Permanent Redirect' for url 'http://status.neocitiesops.net/'\n",
      "Redirect location: 'https://status.neocitiesops.net/'\n",
      "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/308\n",
      "\n",
      "\n",
      "(46/2192 remaining!)\n",
      "https://neocities.org//terms\n",
      "https neocities.org ['', 'terms'] {} \n",
      "e-mail\n",
      "mobile license agreement\n",
      "create seo spam\n",
      "similar materials commonly\n",
      "reporting copyright infringement\n",
      "personal information includes\n",
      "search engines\n",
      "client uploads seo\n",
      "policies grants neocities\n",
      "notifier including address\n",
      "security violations include\n",
      "actively monitor\n",
      "limitation internet outages\n",
      "material encouraging conduct\n",
      "unsolicited commercial information\n",
      "service specific aup\n",
      "designated agent listed\n",
      "service marks/trademarks\n",
      "neocities service records\n",
      "legal terms prior\n",
      "transaction risk services\n",
      "obtain unauthorized access\n",
      "gain unauthorized access\n",
      "party collection agency\n",
      "service specific acceptable\n",
      "stored content related\n",
      "make fraudulent offers\n",
      "electronic mail transmission\n",
      "electronic communications system\n",
      "suspected unlawful activity\n",
      "law enforcement authorities\n",
      "alternative account names\n",
      "personal nonpublic information\n",
      "neocities expressly disclaims\n",
      "notify neocities support\n",
      "bear full responsibility\n",
      "anti-spam organizations\n",
      "seo cleanup fees\n",
      "credit card numbers\n",
      "hold harmless neocities\n",
      "encourages bodily harm\n",
      "e-mail address\n",
      "spam cleanup fees\n",
      "report seo abuse\n",
      "social security number\n",
      "positive internet experience\n",
      "exercise editorial control\n",
      "report spam abuse\n",
      "service specific aups\n",
      "liquidated damage charges\n",
      "blacklist delisting fees\n",
      "law enforcement officials\n",
      "ip address space\n",
      "ensuring full compliance\n",
      "network probing tools\n",
      "prior written permission\n",
      "past due charges\n",
      "then-current acceptable\n",
      "undeliverable electronic mail\n",
      "bulk e-mail\n",
      "e-mail messages\n",
      "subject matter contained\n",
      "online transactions involves\n",
      "nonprofit educational purposes\n",
      "material potentially deemed illegal\n",
      "auto-generating sites\n",
      "web page represents\n",
      "pre-fabricated templates\n",
      "good faith belief\n",
      "service mark/trademark owner\n",
      "neocities then-current acceptable\n",
      "upload web sites intended\n",
      "additional service specific aup\n",
      "actively monitor communications provided\n",
      "search engine optimization\n",
      "limitation reasonable attorneys fees\n",
      "outgoing e-mail message\n",
      "day neocities receives notification\n",
      "online privacy protection act\n",
      "digital millennium copyright act\n",
      "make-money-fast schemes\n",
      "neocities strongly recommends keeping\n",
      "neocities supports unrestricted transfer\n",
      "save web sites edited\n",
      "gov/copyright/legislation/dmca\n",
      "mutant space cat invasion\n",
      "compromising security including cracking tools\n",
      "services providing html editing capabilities\n",
      "reduced search engine page rankings\n",
      "artifically boost search engine rankings\n",
      "\n",
      "\n",
      "(47/2214 remaining!)\n",
      "https://neocities.org//contact\n",
      "https neocities.org ['', 'contact'] {} \n",
      "password reset email\n",
      "http servers\n",
      "including supporter sites\n",
      "password reset form\n",
      "back-end\n",
      "make neocities unsustainable\n",
      "large files tend\n",
      "making static sites\n",
      "org/site/yourusername\n",
      "allowing programming languages\n",
      "manage site settings\n",
      "potential spam/abuse\n",
      "file size limit\n",
      "controlled access site\n",
      "site backup downloading\n",
      "large file distribution\n",
      "webapp hosting platform\n",
      "unique ip address\n",
      "perfectly accurate stats\n",
      "make manual exceptions\n",
      "make web pages\n",
      "transferring inactive sites\n",
      "content delivery network\n",
      "front-end programming\n",
      "newer free plans\n",
      "send arbitary requests\n",
      "complicated web applications\n",
      "internet connectivity issues\n",
      "randomized strong passwords\n",
      "individual visitor records\n",
      "dmca takedown notice\n",
      "party downloading tool\n",
      "prevent phishing attacks\n",
      "excessive bandwidth usage\n",
      "enforce secrecy zones\n",
      "anti-virus scan\n",
      "showing organic traffic\n",
      "in-depth explanation\n",
      "frequently asked questions\n",
      "popular options include bitwarden\n",
      "bringing back personal home pages\n",
      "sources block http referer headers\n",
      "\n",
      "\n",
      "(48/2273 remaining!)\n",
      "https://neocities.org/#!\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(49/2272 remaining!)\n",
      "https://neocities.org//\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(50/2271 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(51/2270 remaining!)\n",
      "https://neocities.org//activity\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(52/2269 remaining!)\n",
      "https://neocities.org//tutorials\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(53/2268 remaining!)\n",
      "https://neocities.org//supporter\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(54/2267 remaining!)\n",
      "https://neocities.org//signin\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(55/2266 remaining!)\n",
      "https://neocities.org//\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(56/2265 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(57/2264 remaining!)\n",
      "https://neocities.org//cli\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(58/2263 remaining!)\n",
      "https://distantskies.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(59/2262 remaining!)\n",
      "https://kekkonen.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(60/2261 remaining!)\n",
      "https://gifypet.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(61/2260 remaining!)\n",
      "https://civilservant.io\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(62/2259 remaining!)\n",
      "https://that1archive.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(63/2258 remaining!)\n",
      "https://lainzine.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(64/2257 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(65/2256 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(66/2255 remaining!)\n",
      "https://neocities.org//supporter\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(67/2254 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(68/2253 remaining!)\n",
      "https://neocities.org//cli\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(69/2252 remaining!)\n",
      "https://neocities.org//api\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(70/2251 remaining!)\n",
      "https://github.com/neocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(71/2250 remaining!)\n",
      "http://www.wired.com/2013/07/neocities/\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(72/2249 remaining!)\n",
      "http://www.fastcodesign.com/3037028/why-the-internet-is-time-traveling-back-to-1994\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(73/2248 remaining!)\n",
      "http://motherboard.vice.com/blog/neocities-is-recreating-the-garish-web-10-creativity-of-geocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(74/2247 remaining!)\n",
      "http://arstechnica.com/tech-policy/2014/05/web-host-gives-fcc-a-28-8kbps-slow-lane-in-net-neutrality-protest/\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(75/2246 remaining!)\n",
      "http://web.appstorm.net/reviews/web-dev/neocities-the-free-place-to-code-your-own-site-from-scratch\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(76/2245 remaining!)\n",
      "https://neocities.org//press\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(77/2244 remaining!)\n",
      "https://neocities.org/#new\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(78/2243 remaining!)\n",
      "https://neocities.org//supporter\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(79/2242 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(80/2241 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(81/2240 remaining!)\n",
      "https://neocities.org//about\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(82/2239 remaining!)\n",
      "https://neocities.org///blog.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(83/2238 remaining!)\n",
      "https://neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(84/2237 remaining!)\n",
      "https://github.com/neocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(85/2236 remaining!)\n",
      "https://twitter.com/neocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(86/2235 remaining!)\n",
      "https://www.facebook.com/neocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(87/2234 remaining!)\n",
      "https://neocities.org//about\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(88/2233 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(89/2232 remaining!)\n",
      "https://neocities.org//cli\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(90/2231 remaining!)\n",
      "https://neocities.org///blog.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(91/2230 remaining!)\n",
      "https://neocities.org//api\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(92/2229 remaining!)\n",
      "https://neocities.org//press\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(93/2228 remaining!)\n",
      "http://status.neocitiesops.net/\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(94/2227 remaining!)\n",
      "https://neocities.org//terms\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(95/2226 remaining!)\n",
      "https://neocities.org//contact\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(96/2225 remaining!)\n",
      "https://neocities.org/#!\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(97/2224 remaining!)\n",
      "https://neocities.org//\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(98/2223 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(99/2222 remaining!)\n",
      "https://neocities.org//activity\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(100/2221 remaining!)\n",
      "https://neocities.org//tutorials\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(101/2220 remaining!)\n",
      "https://neocities.org//supporter\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "Saving...\n",
      "Saved!\n",
      "(102/2219 remaining!)\n",
      "https://neocities.org//signin\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(103/2218 remaining!)\n",
      "https://neocities.org//\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(104/2217 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(105/2216 remaining!)\n",
      "https://neocities.org//cli\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(106/2215 remaining!)\n",
      "https://distantskies.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(107/2214 remaining!)\n",
      "https://kekkonen.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(108/2213 remaining!)\n",
      "https://gifypet.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(109/2212 remaining!)\n",
      "https://civilservant.io\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(110/2211 remaining!)\n",
      "https://that1archive.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(111/2210 remaining!)\n",
      "https://lainzine.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(112/2209 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(113/2208 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(114/2207 remaining!)\n",
      "https://neocities.org//supporter\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(115/2206 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(116/2205 remaining!)\n",
      "https://neocities.org//cli\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(117/2204 remaining!)\n",
      "https://neocities.org//api\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(118/2203 remaining!)\n",
      "https://github.com/neocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(119/2202 remaining!)\n",
      "http://www.wired.com/2013/07/neocities/\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(120/2201 remaining!)\n",
      "http://www.fastcodesign.com/3037028/why-the-internet-is-time-traveling-back-to-1994\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(121/2200 remaining!)\n",
      "http://motherboard.vice.com/blog/neocities-is-recreating-the-garish-web-10-creativity-of-geocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(122/2199 remaining!)\n",
      "http://arstechnica.com/tech-policy/2014/05/web-host-gives-fcc-a-28-8kbps-slow-lane-in-net-neutrality-protest/\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(123/2198 remaining!)\n",
      "http://web.appstorm.net/reviews/web-dev/neocities-the-free-place-to-code-your-own-site-from-scratch\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(124/2197 remaining!)\n",
      "https://neocities.org//press\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(125/2196 remaining!)\n",
      "https://neocities.org/#new\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(126/2195 remaining!)\n",
      "https://neocities.org//supporter\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(127/2194 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(128/2193 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(129/2192 remaining!)\n",
      "https://neocities.org//about\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(130/2191 remaining!)\n",
      "https://neocities.org///blog.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(131/2190 remaining!)\n",
      "https://neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(132/2189 remaining!)\n",
      "https://github.com/neocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(133/2188 remaining!)\n",
      "https://twitter.com/neocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(134/2187 remaining!)\n",
      "https://www.facebook.com/neocities\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(135/2186 remaining!)\n",
      "https://neocities.org//about\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(136/2185 remaining!)\n",
      "https://neocities.org//donate\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(137/2184 remaining!)\n",
      "https://neocities.org//cli\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(138/2183 remaining!)\n",
      "https://neocities.org///blog.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(139/2182 remaining!)\n",
      "https://neocities.org//api\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(140/2181 remaining!)\n",
      "https://neocities.org//press\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(141/2180 remaining!)\n",
      "http://status.neocitiesops.net/\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(142/2179 remaining!)\n",
      "https://neocities.org//terms\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(143/2178 remaining!)\n",
      "https://neocities.org//contact\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(144/2177 remaining!)\n",
      "https://neocities.org/#!\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(145/2176 remaining!)\n",
      "https://neocities.org//browse\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(146/2175 remaining!)\n",
      "https://neocities.org//activity\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(147/2174 remaining!)\n",
      "https://neocities.org//tutorials\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(148/2173 remaining!)\n",
      "https://neocities.org//supporter\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(149/2172 remaining!)\n",
      "https://neocities.org//#new\n",
      "https neocities.org [''] {} new\n",
      "fast site performance\n",
      "favorite desktop editor\n",
      "command line tool\n",
      "drag-n-drop\n",
      "caching anycast cdn\n",
      "routinely out-perform\n",
      "pricey cloud services\n",
      "static web hosting\n",
      "fast static hosting\n",
      "in-browser html editor\n",
      "offer free static web hosting\n",
      "great in-browser html editor\n",
      "\n",
      "\n",
      "(150/2219 remaining!)\n",
      "https://neocities.org//signin\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(151/2218 remaining!)\n",
      "https://neocities.org//\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(152/2217 remaining!)\n",
      "https://goblin-heart.net\n",
      "https goblin-heart.net [] {} \n",
      "theme dropdown\n",
      "blog section\n",
      "personal essay\n",
      "theme switcher code\n",
      "yellow light streaming\n",
      "warm feeling spreads\n",
      "essay/blog-type-thing\n",
      "\n",
      "\n",
      "(153/2221 remaining!)\n",
      "https://goblin-heart.net\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(154/2220 remaining!)\n",
      "https://neocities.org//site/goblin-heart\n",
      "https neocities.org ['', 'site', 'goblin-heart'] {} \n",
      "extensive html resources\n",
      "ainda estou aprendendo\n",
      "polarized political discussions\n",
      "\n",
      "\n",
      "(155/2750 remaining!)\n",
      "https://neocities.org//site/goblin-heart\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(156/2749 remaining!)\n",
      "https://neocities.org//browse?tag=internet\n",
      "https neocities.org ['', 'browse'] {'tag': 'internet'} \n",
      "\n",
      "\n",
      "(157/3730 remaining!)\n",
      "https://neocities.org//browse?tag=blog\n",
      "https neocities.org ['', 'browse'] {'tag': 'blog'} \n",
      "\n",
      "\n",
      "(158/4706 remaining!)\n",
      "https://neocities.org//browse?tag=nostalgia\n",
      "https neocities.org ['', 'browse'] {'tag': 'nostalgia'} \n",
      "strawberry gashes al\n",
      "\n",
      "\n",
      "(159/5697 remaining!)\n",
      "https://neocities.org//browse?tag=writing\n",
      "https neocities.org ['', 'browse'] {'tag': 'writing'} \n",
      "\n",
      "\n",
      "(160/6696 remaining!)\n",
      "https://neocities.org//browse?tag=graphics\n",
      "https neocities.org ['', 'browse'] {'tag': 'graphics'} \n",
      "\n",
      "\n",
      "(161/7688 remaining!)\n",
      "https://cinni.net\n",
      "https cinni.net [] {} \n",
      "change theme\n",
      "\n",
      "\n",
      "(162/7717 remaining!)\n",
      "https://cinni.net\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(163/7716 remaining!)\n",
      "https://neocities.org//site/cinni\n",
      "https neocities.org ['', 'site', 'cinni'] {} \n",
      "presentday-presenttime-band\n",
      "on-bambis-mind\n",
      "baggebo-the-goober\n",
      "dev/dither-me-\n",
      "-themed virtual pet\n",
      "ayla-is-a-nerd\n",
      "\n",
      "\n",
      "(164/8252 remaining!)\n",
      "https://neocities.org//site/cinni\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(165/8251 remaining!)\n",
      "https://neocities.org//browse?tag=pixels\n",
      "https neocities.org ['', 'browse'] {'tag': 'pixels'} \n",
      "strawberry gashes al\n",
      "\n",
      "\n",
      "(166/9030 remaining!)\n",
      "https://neocities.org//browse?tag=art\n",
      "https neocities.org ['', 'browse'] {'tag': 'art'} \n",
      "\n",
      "\n",
      "(167/10009 remaining!)\n",
      "https://neocities.org//browse?tag=personal\n",
      "https neocities.org ['', 'browse'] {'tag': 'personal'} \n",
      "\n",
      "\n",
      "(168/10975 remaining!)\n",
      "https://fauux.neocities.org\n",
      "https fauux.neocities.org [] {} \n",
      "html pt\n",
      "3d lain iwakura\n",
      "pixel-perfect people\n",
      "empty menu test\n",
      "transparent lain pictures\n",
      "player run poorly\n",
      "high performance computers\n",
      "years ago youtube removed\n",
      "ナツメロ、説明 youtube channel\n",
      "mini scenario thingy made\n",
      "online web player tests\n",
      "hawaiian music show ran\n",
      "\n",
      "\n",
      "(169/11054 remaining!)\n",
      "https://fauux.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(170/11053 remaining!)\n",
      "https://neocities.org//site/fauux\n",
      "https neocities.org ['', 'site', 'fauux'] {} \n",
      "average-oxygen-enjoyer\n",
      "week-ish ago\n",
      "\n",
      "\n",
      "(171/11417 remaining!)\n",
      "https://neocities.org//site/fauux\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(172/11416 remaining!)\n",
      "https://neocities.org//browse?tag=anime\n",
      "https neocities.org ['', 'browse'] {'tag': 'anime'} \n",
      "⌈ｗｉｒｅｄ　ｓｏｕｎｄ　ｆｏｒ　ｗｉｒｅｄ　ｐｅｏｐｌ\n",
      "\n",
      "\n",
      "(173/12412 remaining!)\n",
      "https://neocities.org//browse?tag=art\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(174/12411 remaining!)\n",
      "https://neocities.org//browse?tag=music\n",
      "https neocities.org ['', 'browse'] {'tag': 'music'} \n",
      "⌈ｗｉｒｅｄ　ｓｏｕｎｄ　ｆｏｒ　ｗｉｒｅｄ　ｐｅｏｐｌ\n",
      "\n",
      "\n",
      "(175/13402 remaining!)\n",
      "https://neocities.org//browse?tag=pixel\n",
      "https neocities.org ['', 'browse'] {'tag': 'pixel'} \n",
      "⌈ｗｉｒｅｄ　ｓｏｕｎｄ　ｆｏｒ　ｗｉｒｅｄ　ｐｅｏｐｌ\n",
      "\n",
      "\n",
      "(176/14037 remaining!)\n",
      "https://neocities.org//browse?tag=lain\n",
      "https neocities.org ['', 'browse'] {'tag': 'lain'} \n",
      "⌈ｗｉｒｅｄ　ｓｏｕｎｄ　ｆｏｒ　ｗｉｒｅｄ　ｐｅｏｐｌ\n",
      "\n",
      "\n",
      "(177/14844 remaining!)\n",
      "https://sugarforbrains.neocities.org\n",
      "https sugarforbrains.neocities.org [] {} \n",
      "sound effects\n",
      "mobile unfriendly\n",
      "\n",
      "\n",
      "(178/14847 remaining!)\n",
      "https://sugarforbrains.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(179/14846 remaining!)\n",
      "https://neocities.org//site/sugarforbrains\n",
      "https neocities.org ['', 'site', 'sugarforbrains'] {} \n",
      "remade cupcakes hd\n",
      "is-it-noid\n",
      "artfight dis year https\n",
      "ayla-is-a-nerd\n",
      "\n",
      "\n",
      "(180/15296 remaining!)\n",
      "https://neocities.org//site/sugarforbrains\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(181/15295 remaining!)\n",
      "https://neocities.org//browse?tag=art\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(182/15294 remaining!)\n",
      "https://neocities.org//browse?tag=rainbow\n",
      "https neocities.org ['', 'browse'] {'tag': 'rainbow'} \n",
      "haz chezburger plz\n",
      "\n",
      "\n",
      "(183/15730 remaining!)\n",
      "https://neocities.org//browse?tag=personal\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(184/15729 remaining!)\n",
      "https://districts.neocities.org\n",
      "https districts.neocities.org [] {} \n",
      "\n",
      "\n",
      "(185/15728 remaining!)\n",
      "https://districts.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(186/15727 remaining!)\n",
      "https://neocities.org//site/districts\n",
      "https neocities.org ['', 'site', 'districts'] {} \n",
      "Error fetching the URL: Redirect response '302 Found' for url 'https://neocities.org//site/districts'\n",
      "Redirect location: 'https://districts.neocities.org'\n",
      "For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/302\n",
      "\n",
      "\n",
      "(187/15729 remaining!)\n",
      "https://neocities.org//site/districts\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(188/15728 remaining!)\n",
      "https://neocities.org//browse?tag=index\n",
      "https neocities.org ['', 'browse'] {'tag': 'index'} \n",
      "trust neocities websi\n",
      "\n",
      "\n",
      "(189/15958 remaining!)\n",
      "https://neocities.org//browse?tag=neocities\n",
      "https neocities.org ['', 'browse'] {'tag': 'neocities'} \n",
      "\n",
      "\n",
      "(190/16358 remaining!)\n",
      "https://neocities.org//browse?tag=directory\n",
      "https neocities.org ['', 'browse'] {'tag': 'directory'} \n",
      "\n",
      "\n",
      "(191/16728 remaining!)\n",
      "https://melonking.net\n",
      "https melonking.net [] {} \n",
      "enable auto-play audio\n",
      "\n",
      "\n",
      "(192/16728 remaining!)\n",
      "https://melonking.net\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(193/16727 remaining!)\n",
      "https://neocities.org//site/melonking\n",
      "https neocities.org ['', 'site', 'melonking'] {} \n",
      "eness-goofy-site\n",
      "gifypet iframe bigger\n",
      "windows xp computer\n",
      "sons-of-arn\n",
      "sireingtons-cool-place\n",
      "gazing-at-saturn\n",
      "average-oxygen-enjoyer\n",
      "baggebo-the-goober\n",
      "just-a-rat\n",
      "acid trip xd\n",
      "\n",
      "\n",
      "(194/17253 remaining!)\n",
      "https://neocities.org//site/melonking\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(195/17252 remaining!)\n",
      "https://neocities.org//browse?tag=videogames\n",
      "https neocities.org ['', 'browse'] {'tag': 'videogames'} \n",
      "game boy abyss\n",
      "\n",
      "\n",
      "(196/18262 remaining!)\n",
      "https://neocities.org//browse?tag=art\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(197/18261 remaining!)\n",
      "https://neocities.org//browse?tag=music\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(198/18260 remaining!)\n",
      "https://neocities.org//browse?tag=90s\n",
      "https neocities.org ['', 'browse'] {'tag': '90s'} \n",
      "\n",
      "\n",
      "(199/19262 remaining!)\n",
      "https://neocities.org//browse?tag=3d\n",
      "https neocities.org ['', 'browse'] {'tag': '3d'} \n",
      "dopest virtual bl\n",
      "3\n",
      "\n",
      "\n",
      "(200/20102 remaining!)\n",
      "https://uncannyvalley.neocities.org\n",
      "https uncannyvalley.neocities.org [] {} \n",
      "blabla zoom\n",
      "\n",
      "\n",
      "(201/20103 remaining!)\n",
      "https://uncannyvalley.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "Saving...\n",
      "Saved!\n",
      "(202/20102 remaining!)\n",
      "https://neocities.org//site/uncannyvalley\n",
      "https neocities.org ['', 'site', 'uncannyvalley'] {} \n",
      "bio-virus-online\n",
      "gazing-at-saturn\n",
      "presentday-presenttime-band\n",
      "\n",
      "\n",
      "(203/20472 remaining!)\n",
      "https://neocities.org//site/uncannyvalley\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(204/20471 remaining!)\n",
      "https://neocities.org//browse?tag=ocs\n",
      "https neocities.org ['', 'browse'] {'tag': 'ocs'} \n",
      "\n",
      "\n",
      "(205/21465 remaining!)\n",
      "https://neocities.org//browse?tag=art\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(206/21464 remaining!)\n",
      "https://neocities.org//browse?tag=alternative\n",
      "https neocities.org ['', 'browse'] {'tag': 'alternative'} \n",
      "agenda des bas\n",
      "\n",
      "\n",
      "(207/21872 remaining!)\n",
      "https://neocities.org//browse?tag=comics\n",
      "https neocities.org ['', 'browse'] {'tag': 'comics'} \n",
      "\n",
      "\n",
      "(208/22856 remaining!)\n",
      "https://neocities.org//browse?tag=games\n",
      "https neocities.org ['', 'browse'] {'tag': 'games'} \n",
      "\n",
      "\n",
      "(209/23845 remaining!)\n",
      "https://odditycommoddity.neocities.org\n",
      "https odditycommoddity.neocities.org [] {} \n",
      "organizes vetted fundraisers\n",
      "\n",
      "\n",
      "(210/23861 remaining!)\n",
      "https://odditycommoddity.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(211/23860 remaining!)\n",
      "https://neocities.org//site/odditycommoddity\n",
      "https neocities.org ['', 'site', 'odditycommoddity'] {} \n",
      "sireingtons-cool-place\n",
      "on-bambis-mind\n",
      "baggebo-the-goober\n",
      "spiderweb-cotten-candy\n",
      "depth coding questions\n",
      "lot recently lol\n",
      "im gonna turn\n",
      "im gonna prioritize neoneighbors\n",
      "ayla-is-a-nerd\n",
      "guest book letting ppl leave\n",
      "\n",
      "\n",
      "(212/24338 remaining!)\n",
      "https://neocities.org//site/odditycommoddity\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(213/24337 remaining!)\n",
      "https://neocities.org//browse?tag=art\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(214/24336 remaining!)\n",
      "https://neocities.org//browse?tag=personal\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(215/24335 remaining!)\n",
      "https://neocities.org//browse?tag=groovy\n",
      "https neocities.org ['', 'browse'] {'tag': 'groovy'} \n",
      "\n",
      "\n",
      "(216/24465 remaining!)\n",
      "https://neocities.org//browse?tag=kidcore\n",
      "https neocities.org ['', 'browse'] {'tag': 'kidcore'} \n",
      "\n",
      "\n",
      "(217/24778 remaining!)\n",
      "https://ita.toys\n",
      "https ita.toys [] {} \n",
      "\n",
      "\n",
      "(218/24785 remaining!)\n",
      "https://ita.toys\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(219/24784 remaining!)\n",
      "https://neocities.org//site/ita\n",
      "https neocities.org ['', 'site', 'ita'] {} \n",
      "make slide shows\n",
      "present comic books\n",
      "\n",
      "\n",
      "(220/25373 remaining!)\n",
      "https://neocities.org//site/ita\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(221/25372 remaining!)\n",
      "https://neocities.org//browse?tag=music\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(222/25371 remaining!)\n",
      "https://neocities.org//browse?tag=lolitafashion\n",
      "https neocities.org ['', 'browse'] {'tag': 'lolitafashion'} \n",
      "\n",
      "\n",
      "(223/25787 remaining!)\n",
      "https://neocities.org//browse?tag=fashion\n",
      "https neocities.org ['', 'browse'] {'tag': 'fashion'} \n",
      "totally sickening web\n",
      "orchid gals gya\n",
      "\n",
      "\n",
      "(224/26761 remaining!)\n",
      "https://neocities.org//browse?tag=jfashion\n",
      "https neocities.org ['', 'browse'] {'tag': 'jfashion'} \n",
      "\n",
      "\n",
      "(225/27118 remaining!)\n",
      "https://neocities.org//browse?tag=90s\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(226/27117 remaining!)\n",
      "https://anlucas.neocities.org\n",
      "https anlucas.neocities.org [] {} \n",
      "stuff including\n",
      "interesting stuff\n",
      "a\n",
      "\n",
      "\n",
      "(227/27133 remaining!)\n",
      "https://anlucas.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(228/27132 remaining!)\n",
      "https://neocities.org//site/anlucas\n",
      "https neocities.org ['', 'site', 'anlucas'] {} \n",
      "Error encountered in response for url https://neocities.org//site/anlucas: The read operation timed out\n",
      "\n",
      "\n",
      "(229/27131 remaining!)\n",
      "https://neocities.org//site/anlucas\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(230/27130 remaining!)\n",
      "https://neocities.org//browse?tag=html\n",
      "https neocities.org ['', 'browse'] {'tag': 'html'} \n",
      "\n",
      "\n",
      "(231/28087 remaining!)\n",
      "https://neocities.org//browse?tag=90s\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(232/28086 remaining!)\n",
      "https://neocities.org//browse?tag=2000s\n",
      "https neocities.org ['', 'browse'] {'tag': '2000s'} \n",
      "\n",
      "\n",
      "(233/29089 remaining!)\n",
      "https://neocities.org//browse?tag=fireplace\n",
      "https neocities.org ['', 'browse'] {'tag': 'fireplace'} \n",
      "\n",
      "\n",
      "(234/29229 remaining!)\n",
      "https://neocities.org//browse?tag=art\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(235/29228 remaining!)\n",
      "https://mebious.neocities.org\n",
      "https mebious.neocities.org [] {} \n",
      "terrifying thought\n",
      "seek truth\n",
      "extraordinary claim requires\n",
      "seek truth find mebious\n",
      "make logical sense exist\n",
      "\n",
      "\n",
      "(236/29230 remaining!)\n",
      "https://mebious.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(237/29229 remaining!)\n",
      "https://neocities.org//site/mebious\n",
      "https neocities.org ['', 'site', 'mebious'] {} \n",
      "patch-of-mosserie\n",
      "the-red-and-the-blue\n",
      "\n",
      "\n",
      "(238/29475 remaining!)\n",
      "https://neocities.org//site/mebious\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(239/29474 remaining!)\n",
      "https://neocities.org//browse?tag=lain\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(240/29473 remaining!)\n",
      "https://neocities.org//browse?tag=serial\n",
      "https neocities.org ['', 'browse'] {'tag': 'serial'} \n",
      "\n",
      "\n",
      "(241/29631 remaining!)\n",
      "https://neocities.org//browse?tag=experiments\n",
      "https neocities.org ['', 'browse'] {'tag': 'experiments'} \n",
      "\n",
      "\n",
      "(242/29872 remaining!)\n",
      "https://neocities.org//browse?tag=mebious\n",
      "https neocities.org ['', 'browse'] {'tag': 'mebious'} \n",
      "\n",
      "\n",
      "(243/30003 remaining!)\n",
      "https://neocities.org//browse?tag=hacking\n",
      "https neocities.org ['', 'browse'] {'tag': 'hacking'} \n",
      "esoteric perl prog\n",
      "dulwich prep londo\n",
      "complain mgmt system\n",
      "\n",
      "\n",
      "(244/30630 remaining!)\n",
      "https://shishka.neocities.org\n",
      "https shishka.neocities.org [] {} \n",
      "\n",
      "\n",
      "(245/30642 remaining!)\n",
      "https://shishka.neocities.org\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(246/30641 remaining!)\n",
      "https://neocities.org//site/shishka\n",
      "https neocities.org ['', 'site', 'shishka'] {} \n",
      "\n",
      "\n",
      "(247/30934 remaining!)\n",
      "https://neocities.org//site/shishka\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(248/30933 remaining!)\n",
      "https://neocities.org//browse?tag=videogames\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(249/30932 remaining!)\n",
      "https://neocities.org//browse?tag=blinkies\n",
      "https neocities.org ['', 'browse'] {'tag': 'blinkies'} \n",
      "candice aka mool\n",
      "\n",
      "\n",
      "(250/31502 remaining!)\n",
      "https://neocities.org//browse?tag=graphics\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(251/31501 remaining!)\n",
      "https://neocities.org//browse?tag=art\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(252/31500 remaining!)\n",
      "https://neocities.org//browse?tag=anime\n",
      "URL was already finished.\n",
      "\n",
      "\n",
      "(253/31499 remaining!)\n",
      "https://horrorgifs.neocities.org\n",
      "https horrorgifs.neocities.org [] {} \n"
     ]
    }
   ],
   "source": [
    "for i in range(50000):\n",
    "    print(f'({i}/{len(scrapestack)} remaining!)')\n",
    "    #try:\n",
    "    print(scrapestack[0])\n",
    "    try:\n",
    "        scrape(scrapestack.pop(0))\n",
    "    except:\n",
    "        print('Big error')\n",
    "    print('\\n')\n",
    "    if i%100==1:\n",
    "        print(\"Saving...\")\n",
    "        save()\n",
    "        print(\"Saved!\")\n",
    "    #except Exception as e:\n",
    "    #    print(f'Failed: {e}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://neocities.org',\n",
       " 'https://distantskies.neocities.org',\n",
       " 'https://kekkonen.neocities.org',\n",
       " 'https://gifypet.neocities.org',\n",
       " 'https://civilservant.io',\n",
       " 'https://that1archive.neocities.org',\n",
       " 'https://lainzine.org',\n",
       " 'https://github.com',\n",
       " 'http://www.wired.com',\n",
       " 'http://www.fastcodesign.com',\n",
       " 'http://motherboard.vice.com',\n",
       " 'http://arstechnica.com',\n",
       " 'http://web.appstorm.net',\n",
       " 'https://twitter.com',\n",
       " 'https://www.facebook.com',\n",
       " 'http://status.neocitiesops.net']"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "visiteddomains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 299,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 298,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rss_feeds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}