TripadvisorListingInfo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1796fdad",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "import json\n",
    "import base64\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import os.path\n",
    "from datetime import date\n",
    "import re \n",
    "import random\n",
    "import csv\n",
    "from selenium import webdriver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5bac25b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',\n",
    "    'Cookie': \"\"\"TAUnique=%1%enc%3AL0wfTQZk3h9db908sCUm8GLEGz1AGZSeZ0ApmQ39cCaxlTHXi1NYRbOCf2lhG48QNox8JbUSTxk%3D; TASameSite=1; TASSK=enc%3AAF9GF37b558AgyR80pw085GBSO8jbmSZn4s9etRR3PpPQUzljtxX8fuQM%2BgRoT98eBHux606Ur1M7FGUsal91dY1dF8A7M07dvReI%2FhzrVdV4doZjnBPI71eTDciEh3NHw%3D%3D; TATravelInfo=V2*A.2*MG.-1*HP.2*FL.3*RS.1; TATrkConsent=eyJvdXQiOiIiLCJpbiI6IiJ9; TADCID=YpDT17MT-WHwyrfhABQCmq6heh9ZSU2yA8SXn9Wv5H07_H-5BZSJWcGWlTr_wpSRAla9-73wQyfmWzcPg5X0N6XsnSySrhmS4j4; ServerPool=B; PMC=V2*MS.40*MD.20240422*LD.20240430; TART=%1%enc%3AnrNG06l8K71g5En6cEMMPUVbJ4k83N0Txlbl70VubCG8YkTXr2xLzmRtrnIFOZmKqXvbBVFjSaA%3D; TAReturnTo=%1%%2FTravelersChoice-Hotels-cTop-g1; __vt=MGtstV-EOJo-GDy7ABQCwRB1grfcRZKTnW7buAoPsS0ZyetE6h4CtFbVfbGKU9o_pCv04v-98LtOvTZqqRqj2xVVmwZuXte34LZS3dnXwaVayTLnvSMb04O8d7dlNBRKupBSkuejikK-G77mX7jw0lMMiA; SRT=TART_SYNC; TAUD=LA-1713768563159-1*RDD-1-2024_04_22*LG-689450113-2.1.F.*LD-689450114-.....; TASID=55A76BF0F54E4179B4993678049CCC33; datadome=tllb0ybzrT90gFxZBwQLkmBja0WGVRuDpx69_3l32T6g58JSM7ifTQHbJmtpFG6SB21ViT3RgxdGpYzyiOH4zMeV_0zGC7AgOY2Os~pBvccLkAAFeohwC4E3zhzNenIu; TASession=V2ID.55A76BF0F54E4179B4993678049CCC33*SQ.10*LS.Hotel_Review*HS.recommended*ES.popularity*DS.5*SAS.popularity*FPS.oldFirst*FA.1*DF.0*TRA.true*LD.478381*EAU._; PAC=AEoCSFo4m-Xr8Hl4QZ6S9bUWy_BMkNnSu1LGBze5Qm3GOqLLMC4v0RVFuqRimjgREmeC6wkjtEk2XRhG-DBjQIDoJvu6z9399MbRY6N3agIEklzjnS_JJcHn47j_xKFGAFPtbndq6cF3W8egVJZjVWC0bTcwqmH5lZEWIWdp6ruUvf4mppDvrcJrH73YHqiqxkZ89YKk5_-i1thbc_3RLre-camYYepIgy9IbBixJpx_FfBx1gCyKDwycnyN_-m0jQ%3D%3D\"\"\"\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6b14dd76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html><head><title>tripadvisor.com</title><style>#cmsg{animation: A 1.5s;}@keyframes A{0%{opacity:0;}99%{opacity:0;}100%{opacity:1;}}</style></head><body style=\"margin:0\"><p id=\"cmsg\">Please enable JS and disable any ad blocker</p><script data-cfasync=\"false\">var dd={'rt':'c','cid':'AHrlqAAAAAMA7GAR87cxHoQAatdUqw==','hsh':'2F05D671381DB06BEE4CC52C7A6FD3','t':'fe','s':46694,'e':'489a52cc81ad3281bab7b1e4c81ab04efa7b0899c2cffa6e13fba9de600b3e6b','host':'geo.captcha-delivery.com'}</script><script data-cfasync=\"false\" src=\"https://ct.captcha-delivery.com/c.js\"></script></body></html>\n"
     ]
    }
   ],
   "source": [
    "url=\"https://www.tripadvisor.com/Hotel_Review-g6695203-d478381-Reviews-Sheraton_Maldives_Full_Moon_Resort_Spa-Furanafushi_Island.html\"\n",
    "r = requests.get(url, headers=headers)\n",
    "soup = BeautifulSoup(r.text, 'html.parser')\n",
    "print(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3352cb6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html><head><title>tripadvisor.com</title><style>#cmsg{animation: A 1.5s;}@keyframes A{0%{opacity:0;}99%{opacity:0;}100%{opacity:1;}}</style><meta content=\"width=device-width, initial-scale=1.0\" name=\"viewport\"/></head><body style=\"margin:0\"><script data-cfasync=\"false\">var dd={'rt':'c','cid':'AHrlqAAAAAMA6gZDjW_9kc0AatdUqw==','hsh':'2F05D671381DB06BEE4CC52C7A6FD3','t':'fe','s':46694,'e':'c6871edaddd0d93fea7afea0a0d03f202298b983b614d71174aa7098031d38e9','host':'geo.captcha-delivery.com'}</script><script data-cfasync=\"false\" src=\"https://ct.captcha-delivery.com/c.js\"></script><iframe border=\"0\" frameborder=\"0\" height=\"100%\" scrolling=\"yes\" src=\"https://geo.captcha-delivery.com/captcha/?initialCid=AHrlqAAAAAMA6gZDjW_9kc0AatdUqw%3D%3D&amp;hash=2F05D671381DB06BEE4CC52C7A6FD3&amp;cid=EoXYz74FZct8LMzPnp6wAw3fg1L2RkroD~~88_2hCwfQRgtyqkFwiogNxbKpcI5BPs~Bjv3xKT4t_fCkAvv2~SreeFROO~r5Ld6NUgjpaOWgyeahR1OHy8eq9zWzvSHV&amp;t=fe&amp;referer=https%3A%2F%2Fwww.tripadvisor.com%2FHotel_Review-g6695203-d478381-Reviews-Sheraton_Maldives_Full_Moon_Resort_Spa-Furanafushi_Island.html&amp;s=46694&amp;e=c6871edaddd0d93fea7afea0a0d03f202298b983b614d71174aa7098031d38e9&amp;dm=cd\" style=\"height:100vh;\" width=\"100%\"></iframe></body></html>\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver import ActionChains\n",
    "import time\n",
    "import selenium\n",
    "\n",
    "#Incognito Mode\n",
    "option = webdriver.ChromeOptions()\n",
    "option.add_argument(\"--incognito\")\n",
    "\n",
    "#Open Chrome\n",
    "driver = webdriver.Chrome(executable_path=r\"C:\\Users\\saura\\Downloads\\chromedriver-win64 (3)\\chromedriver-win64\\chromedriver.exe\",chrome_options=option)\n",
    "\n",
    "lists=[\"https://www.tripadvisor.com/Hotel_Review-g6695203-d478381-Reviews-Sheraton_Maldives_Full_Moon_Resort_Spa-Furanafushi_Island.html\"]\n",
    "\n",
    "for url in lists:\n",
    "\n",
    "    driver.get(url)\n",
    "    time.sleep(3)\n",
    "    soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
    "    print(soup)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "308374a0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d52949ae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8d01685",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da701be7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fd09d255",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing: https://www.tripadvisor.in/Restaurants-g188553-The_Netherlands.html\n",
      "<html><head><title>tripadvisor.in</title><style>#cmsg{animation: A 1.5s;}@keyframes A{0%{opacity:0;}99%{opacity:0;}100%{opacity:1;}}</style></head><body style=\"margin:0\"><p id=\"cmsg\">Please enable JS and disable any ad blocker</p><script data-cfasync=\"false\">var dd={'rt':'c','cid':'AHrlqAAAAAMAbprtDwzyRTYAerFgBw==','hsh':'2F05D671381DB06BEE4CC52C7A6FD3','t':'fe','s':46694,'e':'5d71e99cb82388a10d6b79309c2e0650ed89b5025e305b73d83b35e5a7edee4c','host':'geo.captcha-delivery.com'}</script><script data-cfasync=\"false\" src=\"https://ct.captcha-delivery.com/c.js\"></script></body></html>\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'group'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[4], line 22\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[38;5;28mprint\u001b[39m(soup)\n\u001b[0;32m     21\u001b[0m \u001b[38;5;66;03m# Extract restaurant ID from URL\u001b[39;00m\n\u001b[1;32m---> 22\u001b[0m restaurant_id \u001b[38;5;241m=\u001b[39m \u001b[43mre\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43md(\u001b[39;49m\u001b[38;5;124;43m\\\u001b[39;49m\u001b[38;5;124;43md+)-Reviews\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroup\u001b[49m(\u001b[38;5;241m1\u001b[39m)\n\u001b[0;32m     24\u001b[0m \u001b[38;5;66;03m# Get restaurant name\u001b[39;00m\n\u001b[0;32m     25\u001b[0m name_div \u001b[38;5;241m=\u001b[39m soup\u001b[38;5;241m.\u001b[39mfind(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdiv\u001b[39m\u001b[38;5;124m'\u001b[39m, class_\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTzFxL w u\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'group'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import pandas as pd\n",
    "from datetime import date\n",
    "\n",
    "##you have to read the csv in which you have restaurants links of philippines\n",
    "#df = pd.read_csv(r\"C:\\Users\\saura\\Downloads\\tripadvisorriyaadh (6).csv\")[0:2]\n",
    "urls=['https://www.tripadvisor.in/Restaurants-g188553-The_Netherlands.html']\n",
    "data_list = []\n",
    "\n",
    "# Loop through each URL\n",
    "for url in urls:\n",
    "    print(\"Processing:\", url)\n",
    "\n",
    "    # Make a GET request to the URL\n",
    "    r = requests.get(url, headers=headers)\n",
    "    soup = BeautifulSoup(r.text, 'html.parser')\n",
    "    print(soup)\n",
    "    # Extract restaurant ID from URL\n",
    "    restaurant_id = re.search(r'd(\\d+)-Reviews', url).group(1)\n",
    "\n",
    "    # Get restaurant name\n",
    "    name_div = soup.find('div', class_='TzFxL w u')\n",
    "    name = name_div.find('h1', class_='biGQs _P egaXP rRtyp').text.strip() if name_div else \"\"\n",
    "\n",
    "    # Get menu URL\n",
    "    menu_url = None\n",
    "    menu_a = soup.find_all('a', class_='BMQDV _F Gv wSSLS SwZTJ')\n",
    "    for na in menu_a:\n",
    "        if na.text.strip() == \"Menu\":\n",
    "            menu_url = na[\"href\"]\n",
    "            break\n",
    "\n",
    "    # Get latitude and longitude\n",
    "    map_div = soup.find('div', class_='hpxwy e j')\n",
    "    if map_div:\n",
    "        map_link = map_div.find('a', class_='BMQDV _F Gv wSSLS SwZTJ FGwzt ukgoS')['href']\n",
    "        lat_long = map_link.split('@')[1].split(',')\n",
    "        lat = float(lat_long[0])\n",
    "        lon = float(lat_long[1])\n",
    "    else:\n",
    "        lat, lon = \"\", \"\"\n",
    "\n",
    "    # Get reviews and rating\n",
    "    review_div = soup.find('div', class_='CsAqy u Ci Ph w')\n",
    "    if review_div:\n",
    "        review_text = review_div.get_text(strip=True)\n",
    "        rating = review_text.split('of')[0].replace(' ', '')\n",
    "        reviews = review_text.split('reviews')[0].split('bubbles')[-1]\n",
    "    else:\n",
    "        rating, reviews = \"\", \"\"\n",
    "        \n",
    "    email_link = soup.find('a', href=True, text='Email')\n",
    "    if email_link:\n",
    "        email_address = email_link['href'].split(':')[1]\n",
    "    else:\n",
    "        email_address = \"\"\n",
    "\n",
    "\n",
    "    # Get address, price level, price, cuisines, special diets, and meals\n",
    "    address, price_level, price, cuisines, special, meals = \"\", None, None, None, None, None\n",
    "    more = soup.find_all('div', class_='MJ')\n",
    "    for j in more:\n",
    "        text = j.get_text(strip=True)\n",
    "        if 'Address' in text:\n",
    "            address = text.replace('Address', '').strip()\n",
    "        elif 'PRICE RANGE' in text:\n",
    "            price = text.replace('PRICE RANGE', '').strip()\n",
    "        elif 'CUISINES' in text:\n",
    "            cuisines = text.replace('CUISINES', '').strip()\n",
    "        elif 'Special Diets' in text:\n",
    "            special = text.replace('Special Diets', '').strip()\n",
    "        elif 'Meals' in text:\n",
    "            meals = text.replace('Meals', '').strip()\n",
    "\n",
    "    # Append data to the data_list\n",
    "    data_list.append({\n",
    "        'restaurant_id': restaurant_id,\n",
    "        'CrawleDate': date.today().strftime('%Y-%m-%d'),\n",
    "        'url': url,\n",
    "        'name': name,\n",
    "        'lat': lat,\n",
    "        'lon': lon,\n",
    "        'reviews': reviews,\n",
    "        'rating': rating,\n",
    "        'address': address,\n",
    "        'price_level': price_level,\n",
    "        'price': price,\n",
    "        'cuisines': cuisines,\n",
    "        'special': special,\n",
    "        'meals': meals,\n",
    "        'city': \"Riyadh\",\n",
    "        'country': \"Saudi\",\n",
    "        'menu': menu_url,\n",
    "        'Email':email_address,\n",
    "    })\n",
    "\n",
    "# Convert data_list to DataFrame\n",
    "df_output = pd.DataFrame(data_list)\n",
    "print(df_output)\n",
    "# Write DataFrame to CSV file\n",
    "if not os.path.isfile('RiyaadhMenu11.csv'):\n",
    "    df_output.to_csv('RiyaadhMenu11.csv', mode=\"a\", header=True, index=False)\n",
    "else:\n",
    "    df_output.to_csv('RiyaadhMenu11.csv', mode=\"a\", header=False, index=False)\n",
    "\n",
    "print(\"Scraping and data saving completed.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "373af5df",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_output' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[1;32mIn [5]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mdf_output\u001b[49m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'df_output' is not defined"
     ]
    }
   ],
   "source": [
    "df_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5c75701",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "main_language": "python",
   "notebook_metadata_filter": "-all",
   "text_representation": {
    "extension": ".py",
    "format_name": "light"
   }
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}