Scrap.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver #pip install -U selenium #-U Upgrade all specified packages to the newest available version.\n",
    "from selenium.webdriver.common.keys import Keys #pip install -U selenium\n",
    "from selenium.common.exceptions import TimeoutException #slenesse timeout exception\n",
    "from selenium.webdriver.common.by import By #used wih functions that have by\n",
    "from selenium.webdriver.support.ui import WebDriverWait #selenesse wait (like the pause in python)\n",
    "from fake_useragent import UserAgent\n",
    "\n",
    "from selenium import webdriver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome('C:/Users/user/Desktop/chromedriver.exe')\n",
    "driver.get('https://www.douane.gov.tn/ventes-aux-encheres-publiques/')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "oads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-23_AV_OP_-RADES_47_N%C2%B03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-22_AV_OP_-rades-16-03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_RAS-JDIR_N%C2%B001-2021_3%C3%A8me.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_TUNIS-PORT_N_03-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-12_AV_OP_Tunis-port-metaux_N%C2%B004-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-10_AV_OP_KAIROUAN_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-05_AV_OP_SILIANA_N%C2%B001-2021.pdf\n",
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-01_AV_OP_SOUSSE_N%C2%B001-2021.pdf\n"
     ]
    }
   ],
   "source": [
    "link=0\n",
    "PDFS_LINK=[]\n",
    "while(link<=322):\n",
    "    urls = driver.find_elements_by_xpath(\"//td[@class='column-5']/a\")\n",
    "    for url in urls:\n",
    "        href= url.get_attribute('href')\n",
    "        PDFS_LINK.append(href)\n",
    "        print(href)\n",
    "        link=link+1\n",
    "driver.find_element_by_xpath(\"/html/body/div[1]/div[3]/div/div/div/div/div/div/div/div/div/div/div/div/div[1]/div/div/div[3]/div/div/div[3]/a[2]\").click()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ScrapPage \n",
    "from ScrapPage import *\n",
    "from pathlib import Path\n",
    "import json\n",
    "from django.contrib.admin.utils import flatten\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "330\n"
     ]
    }
   ],
   "source": [
    "print(link)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "https://www.douane.gov.tn/wp-content/uploads/2021/03/2021-03-30_AV_OP_-RAS-JDIR_N%C2%B001-2021_4%C3%A8me-fois.pdf\n",
      "yes byte\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'Done'"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "print(PDFS_LINK[0])\n",
    "scrap_url=PDFS_LINK[0]\n",
    "name_url=\"PDF_DOUANE\"+\".pdf\"\n",
    "html_content=GetPageContent(scrap_url)\n",
    "WriteToFile(html_content ,name_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "330\n"
     ]
    }
   ],
   "source": [
    "print(len(PDFS_LINK))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "error",
     "ename": "NameError",
     "evalue": "name 'pathitem' is not defined",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-27305e16f708>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[0mname_url\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"PDF_DOUANE\"\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m\".pdf\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mnouveau\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0mpath_scrap\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpathitem\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m\"/\"\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mname_url\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m     \u001b[0mhtml_content\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mGetPageContent\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnouveau\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'pathitem' is not defined"
     ]
    }
   ],
   "source": [
    "i=0\n",
    "for url in PDFS_LINK:\n",
    "    name_url=\"PDF_DOUANE\"+str(i)+\".pdf\"\n",
    "    nouveau=url\n",
    "   \n",
    "    html_content=GetPageContent(nouveau)\n",
    "   \n",
    "    WriteToFile(html_content ,name_url)\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Collecting tabula-py\n  Downloading tabula_py-2.2.0-py3-none-any.whl (11.7 MB)\nRequirement already satisfied: numpy in c:\\users\\user\\anaconda3\\lib\\site-packages (from tabula-py) (1.18.5)\nRequirement already satisfied: pandas>=0.25.3 in c:\\users\\user\\anaconda3\\lib\\site-packages (from tabula-py) (1.0.5)\nCollecting distro\n  Using cached distro-1.5.0-py2.py3-none-any.whl (18 kB)\nRequirement already satisfied: pytz>=2017.2 in c:\\users\\user\\anaconda3\\lib\\site-packages (from pandas>=0.25.3->tabula-py) (2020.1)\nRequirement already satisfied: python-dateutil>=2.6.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from pandas>=0.25.3->tabula-py) (2.8.1)\nRequirement already satisfied: six>=1.5 in c:\\users\\user\\anaconda3\\lib\\site-packages (from python-dateutil>=2.6.1->pandas>=0.25.3->tabula-py) (1.15.0)\nInstalling collected packages: distro, tabula-py\nSuccessfully installed distro-1.5.0 tabula-py-2.2.0\n"
     ]
    }
   ],
   "source": [
    "#PDF _EXTRACTION_TABLES\n",
    "!pip install tabula-py"
   ]
  }
 ]
}