scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_price(price):\n",
    "    \"\"\"\n",
    "    Convert crore, lakhs, millions and Thousand into numbers\n",
    "\n",
    "    :param price: str\n",
    "    :return: float\n",
    "    \"\"\"\n",
    "    if price.endswith('Crore'):\n",
    "        return round(float(price[:-5]) * 10000000)\n",
    "    elif price.endswith('Lakh'):\n",
    "        return round(float(price[:-4]) * 100000)\n",
    "    elif price.endswith('Million'):\n",
    "        return round(float(price[:-7]) * 1000000)\n",
    "    elif price.endswith('Arab'):\n",
    "        return round(float(price[:-4]) * 1000000000)\n",
    "    elif price.endswith('Thousand'):\n",
    "        return round(float(price[:-8]) * 1000)\n",
    "    else:\n",
    "        return round(float(price))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_size(size):\n",
    "    \"\"\"\n",
    "    Convert kanal merla into sqft\n",
    "\n",
    "    :param size: str\n",
    "    :return: float\n",
    "    \"\"\"\n",
    "    if size.endswith('Marla'):\n",
    "        return round(float(size[:-5].replace(\",\", \"\")) * 225)\n",
    "    elif size.endswith('Kanal'):\n",
    "        return round(float(size[:-5].replace(\",\", \"\")) * 4500)\n",
    "    elif size.endswith('Sq. Yd.'):\n",
    "        return round(float(size[:-7].replace(\",\", \"\")) * 9)\n",
    "    else:\n",
    "        return round(float(size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text(tag, datatype=\"str\"):\n",
    "    \"\"\"\n",
    "    This function will return the text of the tag.\n",
    "\n",
    "    :param tag: tag object\n",
    "    :param datatype: num or str or price, size\n",
    "    :return: price in number or string\n",
    "    \"\"\"\n",
    "    if tag is None and datatype == \"num\":\n",
    "        return 0\n",
    "    if datatype == \"num\":\n",
    "        try:\n",
    "            return int(tag.text.strip())\n",
    "        except ValueError:\n",
    "            return 0\n",
    "    if tag is None and datatype == \"str\":\n",
    "        return \"\"\n",
    "    if datatype == \"str\":\n",
    "        return tag.text.strip()\n",
    "    if tag is None and datatype == \"price\":\n",
    "        return 0.0\n",
    "    if datatype == \"price\":\n",
    "        return convert_price(tag.text.strip())\n",
    "    if tag is None and datatype == \"size\":\n",
    "        return 0.0\n",
    "    if datatype == \"size\":\n",
    "        return convert_size(tag.text.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrap(city, pages_range):\n",
    "    \"\"\"\n",
    "    This function will scrap the zameen.com website and\n",
    "    return the list of houses information\n",
    "\n",
    "    :param city: str\n",
    "    :param pages_range: int\n",
    "    :return: list\n",
    "    \"\"\"\n",
    "    house_info = []\n",
    "\n",
    "    for page_number in range(1, pages_range+1):\n",
    "        url = f'https://www.zameen.com/Homes/{city}-{page_number}.html'\n",
    "        print(url)\n",
    "        response = requests.get(url)\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "\n",
    "        house_list = soup.select(\"main > div > div > div > div > ul > li\")\n",
    "\n",
    "        # store lenght of previous length of house info list\n",
    "        prev_len = len(house_info)\n",
    "\n",
    "        for house in house_list:\n",
    "            baths = house.select_one(\"span[aria-label='Baths']\")\n",
    "            beds = house.select_one(\"span[aria-label='Beds']\")\n",
    "            location = house.select_one(\"div[aria-label='Location']\")\n",
    "            price = house.select_one(\"span[aria-label='Price']\")\n",
    "            size = house.select_one(\"div[title]>div > div > span:nth-child(1)\")\n",
    "\n",
    "            if price:\n",
    "                if size is None:\n",
    "                    size = location.parent.select_one(\n",
    "                        \"div:nth-child(2) > div > span:nth-child(3)\")\n",
    "                house_info.append(\n",
    "                    {\n",
    "                        \"location\": text(location),\n",
    "                        \"price\":    text(price, datatype=\"price\"),\n",
    "                        \"bedrooms\":     text(beds, datatype=\"num\"),\n",
    "                        \"baths\":    text(baths, datatype=\"num\"),\n",
    "                        \"size\":     text(size, datatype=\"size\")\n",
    "                    }\n",
    "                )\n",
    "\n",
    "        # get out of the loop if the last accessed page\n",
    "        # doesnot exist to avoid useless requests because\n",
    "        # next pages will not exist as well\n",
    "        if len(house_info) == prev_len:\n",
    "            break\n",
    "\n",
    "    return house_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.zameen.com/Homes/Lahore-1-1.html\n",
      "https://www.zameen.com/Homes/Lahore-1-2.html\n",
      "https://www.zameen.com/Homes/Lahore-1-3.html\n",
      "https://www.zameen.com/Homes/Lahore-1-4.html\n",
      "https://www.zameen.com/Homes/Lahore-1-5.html\n",
      "https://www.zameen.com/Homes/Lahore-1-6.html\n",
      "https://www.zameen.com/Homes/Lahore-1-7.html\n",
      "https://www.zameen.com/Homes/Lahore-1-8.html\n",
      "https://www.zameen.com/Homes/Lahore-1-9.html\n",
      "https://www.zameen.com/Homes/Lahore-1-10.html\n",
      "https://www.zameen.com/Homes/Lahore-1-11.html\n",
      "https://www.zameen.com/Homes/Lahore-1-12.html\n",
      "https://www.zameen.com/Homes/Lahore-1-13.html\n",
      "https://www.zameen.com/Homes/Lahore-1-14.html\n",
      "https://www.zameen.com/Homes/Lahore-1-15.html\n",
      "https://www.zameen.com/Homes/Lahore-1-16.html\n",
      "https://www.zameen.com/Homes/Lahore-1-17.html\n",
      "https://www.zameen.com/Homes/Lahore-1-18.html\n",
      "https://www.zameen.com/Homes/Lahore-1-19.html\n",
      "https://www.zameen.com/Homes/Lahore-1-20.html\n",
      "https://www.zameen.com/Homes/Karachi-2-1.html\n",
      "https://www.zameen.com/Homes/Karachi-2-2.html\n",
      "https://www.zameen.com/Homes/Karachi-2-3.html\n",
      "https://www.zameen.com/Homes/Karachi-2-4.html\n",
      "https://www.zameen.com/Homes/Karachi-2-5.html\n",
      "https://www.zameen.com/Homes/Karachi-2-6.html\n",
      "https://www.zameen.com/Homes/Karachi-2-7.html\n",
      "https://www.zameen.com/Homes/Karachi-2-8.html\n",
      "https://www.zameen.com/Homes/Karachi-2-9.html\n",
      "https://www.zameen.com/Homes/Karachi-2-10.html\n",
      "https://www.zameen.com/Homes/Karachi-2-11.html\n",
      "https://www.zameen.com/Homes/Karachi-2-12.html\n",
      "https://www.zameen.com/Homes/Karachi-2-13.html\n",
      "https://www.zameen.com/Homes/Karachi-2-14.html\n",
      "https://www.zameen.com/Homes/Karachi-2-15.html\n",
      "https://www.zameen.com/Homes/Karachi-2-16.html\n",
      "https://www.zameen.com/Homes/Karachi-2-17.html\n",
      "https://www.zameen.com/Homes/Karachi-2-18.html\n",
      "https://www.zameen.com/Homes/Karachi-2-19.html\n",
      "https://www.zameen.com/Homes/Karachi-2-20.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-1.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-2.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-3.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-4.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-5.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-6.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-7.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-8.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-9.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-10.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-11.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-12.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-13.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-14.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-15.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-16.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-17.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-18.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-19.html\n",
      "https://www.zameen.com/Homes/Islamabad-3-20.html\n",
      "https://www.zameen.com/Homes/Multan-15-1.html\n",
      "https://www.zameen.com/Homes/Multan-15-2.html\n",
      "https://www.zameen.com/Homes/Multan-15-3.html\n",
      "https://www.zameen.com/Homes/Multan-15-4.html\n",
      "https://www.zameen.com/Homes/Multan-15-5.html\n",
      "https://www.zameen.com/Homes/Multan-15-6.html\n",
      "https://www.zameen.com/Homes/Multan-15-7.html\n",
      "https://www.zameen.com/Homes/Multan-15-8.html\n",
      "https://www.zameen.com/Homes/Multan-15-9.html\n",
      "https://www.zameen.com/Homes/Multan-15-10.html\n",
      "https://www.zameen.com/Homes/Multan-15-11.html\n",
      "https://www.zameen.com/Homes/Multan-15-12.html\n",
      "https://www.zameen.com/Homes/Multan-15-13.html\n",
      "https://www.zameen.com/Homes/Multan-15-14.html\n",
      "https://www.zameen.com/Homes/Multan-15-15.html\n",
      "https://www.zameen.com/Homes/Multan-15-16.html\n",
      "https://www.zameen.com/Homes/Multan-15-17.html\n",
      "https://www.zameen.com/Homes/Multan-15-18.html\n",
      "https://www.zameen.com/Homes/Multan-15-19.html\n",
      "https://www.zameen.com/Homes/Multan-15-20.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-1.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-2.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-3.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-4.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-5.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-6.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-7.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-8.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-9.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-10.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-11.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-12.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-13.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-14.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-15.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-16.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-17.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-18.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-19.html\n",
      "https://www.zameen.com/Homes/Faisalabad-16-20.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-1.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-2.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-3.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-4.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-5.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-6.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-7.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-8.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-9.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-10.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-11.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-12.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-13.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-14.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-15.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-16.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-17.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-18.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-19.html\n",
      "https://www.zameen.com/Homes/Peshawar-17-20.html\n",
      "https://www.zameen.com/Homes/Quetta-18-1.html\n",
      "https://www.zameen.com/Homes/Quetta-18-2.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-1.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-2.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-3.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-4.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-5.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-6.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-7.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-8.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-9.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-10.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-11.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-12.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-13.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-14.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-15.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-16.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-17.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-18.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-19.html\n",
      "https://www.zameen.com/Homes/Rawalpindi-41-20.html\n",
      "https://www.zameen.com/Homes/Murree-36-1.html\n",
      "https://www.zameen.com/Homes/Murree-36-2.html\n",
      "https://www.zameen.com/Homes/Murree-36-3.html\n",
      "https://www.zameen.com/Homes/Murree-36-4.html\n",
      "https://www.zameen.com/Homes/Murree-36-5.html\n",
      "https://www.zameen.com/Homes/Murree-36-6.html\n",
      "https://www.zameen.com/Homes/Murree-36-7.html\n",
      "https://www.zameen.com/Homes/Murree-36-8.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-1.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-2.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-3.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-4.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-5.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-6.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-7.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-8.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-9.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-10.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-11.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-12.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-13.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-14.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-15.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-16.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-17.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-18.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-19.html\n",
      "https://www.zameen.com/Homes/Gujranwala-327-20.html\n",
      "https://www.zameen.com/Homes/Attock-1233-1.html\n",
      "https://www.zameen.com/Homes/Attock-1233-2.html\n",
      "https://www.zameen.com/Homes/2_FECHS-3234-1.html\n",
      "https://www.zameen.com/Homes/2_FECHS-3234-2.html\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    house_info = []\n",
    "\n",
    "    cities = [\n",
    "        {'id': 1, 'name': 'Lahore'},\n",
    "        {'id': 2, 'name': 'Karachi'},\n",
    "        {'id': 3, 'name': 'Islamabad'},\n",
    "        {'id': 15, 'name': 'Multan'},\n",
    "        {'id': 16, 'name': 'Faisalabad'},\n",
    "        {'id': 17, 'name': 'Peshawar'},\n",
    "        {'id': 18, 'name': 'Quetta'},\n",
    "        {'id': 41, 'name': 'Rawalpindi'},\n",
    "        {'id': 36, 'name': 'Murree'},\n",
    "        {'id': 327, 'name': 'Gujranwala'},\n",
    "        {'id': 1233, 'name': 'Attock'},\n",
    "        {'id': 3234, 'name': '2_FECHS'},\n",
    "\n",
    "    ]\n",
    "\n",
    "    for city in cities:\n",
    "\n",
    "        # change 20 to any number of pages you want to scrap\n",
    "        house_info.append(\n",
    "            {\n",
    "                \"city\": city.get('name'),\n",
    "                \"info\": scrap(f\"{city.get('name')}-{city.get('id')}\", 100)\n",
    "            }\n",
    "        )\n",
    "\n",
    "    with open(\"zameen.csv\", \"w\") as f:\n",
    "        # write csv header\n",
    "        f.write(\"city|location|price|bedrooms|baths|size\\n\")\n",
    "        for house in house_info:\n",
    "            for info in house.get('info'):\n",
    "                f.write(\n",
    "                    f\"{house.get('city')}|{info.get('location')}|{info.get('price')}|{info.get('bedrooms')}|{info.get('baths')}|{info.get('size')}\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}