트위터 크롤링.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import selenium\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 웹사이트 열기\n",
    "browser = webdriver.Chrome()\n",
    "browser.maximize_window() # 브라우저 화면 최대화\n",
    "browser.get('https://twitter.com/?lang=ko')\n",
    "browser.implicitly_wait(10)  # 로딩이 끝날 때까지 10초까지는 기다려\n",
    "\n",
    "\n",
    "# 로그인창 클릭\n",
    "# browser.find_element_by_css_selector('.css-4rbku5.css-18t94o4.css-1dbjc4n.r-1niwhzg').click()\n",
    "browser.find_element_by_css_selector('.css-901oao.r-1awozwy.r-1cvl2hr.r-6koalj.r-18u37iz').click()\n",
    "browser.implicitly_wait(5)\n",
    "browser.find_element_by_css_selector('.css-1dbjc4n.r-1ets6dv.r-z2wwpe.r-rs99b7').click()\n",
    "browser.implicitly_wait(5)\n",
    "\n",
    "# 아이디 입력\n",
    "search = browser.find_element_by_xpath('//*[@id=\"layers\"]/div/div/div/div/div/div/div[2]/div[2]/div/div/div[2]/div[2]/div[1]/div/div/div[5]/label/div/div[2]/div/input')\n",
    "browser.implicitly_wait(5)\n",
    "search.click()\n",
    "search.send_keys('wpghfj2@gmail.com')\n",
    "search.send_keys(Keys.ENTER)\n",
    "browser.implicitly_wait(10)\n",
    "\n",
    "# 추가 정보 요청시 핸폰 입력\n",
    "time.sleep(2)\n",
    "search2 = browser.find_element_by_xpath('//*[@id=\"layers\"]/div/div/div/div/div/div/div[2]/div[2]/div/div/div[2]/div[2]/div[1]/div/div[2]/label/div/div[2]/div/input')\n",
    "search2.send_keys('01084200399')\n",
    "search2.send_keys(Keys.ENTER)\n",
    "browser.implicitly_wait(10)\n",
    "\n",
    "\n",
    "# 비번 입력\n",
    "search3 = browser.find_element_by_xpath('//*[@id=\"layers\"]/div/div/div/div/div/div/div[2]/div[2]/div/div/div[2]/div[2]/div[1]/div/div/div[3]/div/label/div/div[2]/div[1]/input')\n",
    "search3.send_keys('dbghwns12!')\n",
    "search3.send_keys(Keys.ENTER)\n",
    "\n",
    "\n",
    "# 검색어 입력\n",
    "browser.find_element_by_xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div[2]/div/div[2]/div/div/div/div[1]/div/div/div/form/div[1]/div/div/label/div[2]/div/input').click()\n",
    "browser.find_element_by_xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div[2]/div/div[2]/div/div/div/div[1]/div/div/div/form/div[1]/div/div/label/div[2]/div/input').send_keys('치아미백')\n",
    "browser.find_element_by_xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div[2]/div/div[2]/div/div/div/div[1]/div/div/div/form/div[1]/div/div/label/div[2]/div/input').send_keys(Keys.ENTER)\n",
    "\n",
    "\n",
    "# 최근 글로 이동\n",
    "browser.find_element_by_xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div[1]/div/div[1]/div[2]/nav/div/div[2]/div/div[2]/a').click()\n",
    "time.sleep(5)\n",
    "\n",
    "\n",
    "\n",
    "# # 크롤링 시작\n",
    "# contexts = []\n",
    "# dates = []\n",
    "# retweets = []\n",
    "# hearts = []\n",
    "\n",
    "\n",
    "# # 스크롤 내리기\n",
    "# for i in range(1, 4000):\n",
    "#     for j in range(7):\n",
    "#         browser.find_element_by_tag_name('body').send_keys(Keys.PAGE_DOWN)\n",
    "        \n",
    "\n",
    "#     try:\n",
    "#         for z in range(1,21):\n",
    "#             path = f'/html/body/div[1]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/section/div/div/div[{z}]/div/div/article/div/div/div/div[2]/div[2]/div[2]'\n",
    "#             path2 = f'/html/body/div[1]/div/div/div[2]/main/div/div/div/div[1]/div/div[2]/div/section/div/div/div[{z}]'\n",
    "#             box = browser.find_element_by_xpath(path)\n",
    "#             box2 = browser.find_element_by_xpath(path2)\n",
    "            \n",
    "#             context = box.find_element_by_class_name('css-901oao')\n",
    "#             date = box2.find_element_by_css_selector(\".css-4rbku5 > time\")\n",
    "#             retweet = box.find_element_by_css_selector(\"div[data-testid='retweet']\")\n",
    "#             heart = box.find_element_by_css_selector(\"div[data-testid='like']\")\n",
    "#             contexts.append(context.text)\n",
    "#             dates.append(date.text)\n",
    "#             retweets.append(retweet.text)\n",
    "#             hearts.append(heart.text)\n",
    "#     except:\n",
    "#         pass\n",
    "\n",
    "\n",
    "\n",
    "# #Dataframe 만들기\n",
    "# df = pd.DataFrame({'본문':contexts, '날짜':dates, '리트윗':retweets,'하트':hearts})\n",
    "\n",
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#중간에 끊겻을 경우에 스크롤 내리는 용으로 사용\n",
    "for j in range(4000):\n",
    "        browser.find_element_by_tag_name('body').send_keys(Keys.PAGE_DOWN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 스크롤 내리기\n",
    "# 크롤링 시작\n",
    "contexts = []\n",
    "dates = []\n",
    "retweets = []\n",
    "hearts = []\n",
    "\n",
    "for i in range(1, 10000):\n",
    "    print(i)\n",
    "    for j in range(7):\n",
    "        browser.find_element_by_tag_name('body').send_keys(Keys.PAGE_DOWN)\n",
    "        \n",
    "\n",
    "    try:\n",
    "        for z in range(1,21):\n",
    "            path = f'/html/body/div[1]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/section/div/div/div[{z}]/div/div/article/div/div/div/div[2]/div[2]/div[2]'\n",
    "            path2 = f'/html/body/div[1]/div/div/div[2]/main/div/div/div/div[1]/div/div[2]/div/section/div/div/div[{z}]'\n",
    "            box = browser.find_element_by_xpath(path)\n",
    "            box2 = browser.find_element_by_xpath(path2)\n",
    "            \n",
    "            #본문, 날짜, 리트위수, 하트수\n",
    "            try:\n",
    "                context = box.find_element_by_class_name('css-901oao')\n",
    "                contexts.append(context.text)\n",
    "            except:\n",
    "                contexts.append('')\n",
    "\n",
    "            try:\n",
    "                date = box2.find_element_by_css_selector(\".css-4rbku5 > time\")\n",
    "                dates.append(date.text)\n",
    "            except:\n",
    "                dates.append('')\n",
    "\n",
    "            try:\n",
    "                retweet = box.find_element_by_css_selector(\"div[data-testid='retweet']\")\n",
    "                retweets.append(retweet.text)\n",
    "            except:\n",
    "                retweets.append('')\n",
    "\n",
    "            try:\n",
    "                heart = box.find_element_by_css_selector(\"div[data-testid='like']\")\n",
    "                hearts.append(heart.text)\n",
    "            except:\n",
    "                hearts.append('')                \n",
    "\n",
    "            # context = box.find_element_by_class_name('css-901oao')\n",
    "            # date = box2.find_element_by_css_selector(\".css-4rbku5 > time\")\n",
    "            # retweet = box.find_element_by_css_selector(\"div[data-testid='retweet']\")\n",
    "            # heart = box.find_element_by_css_selector(\"div[data-testid='like']\")\n",
    "\n",
    "            # #리스트에 추가\n",
    "            # contexts.append(context.text)\n",
    "            # dates.append(date.text)\n",
    "            # retweets.append(retweet.text)\n",
    "            # hearts.append(heart.text)\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(contexts))\n",
    "print(len(dates))\n",
    "print(len(retweets))\n",
    "print(len(hearts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic = {'본문':contexts, '날짜':dates, '리트윗':retweets,'하트':hearts}                                     #길이가 다른 리스트들을 데이터프레임으로 저장  \n",
    "res = pd.DataFrame.from_dict(dic, orient='index')\n",
    "res = res.transpose()\n",
    "res\n",
    "\n",
    "res.to_csv('C:/Users/lcr14/OneDrive/바탕 화면/코딩 연습/MZ-trend/치아미백2.csv',sep=',', encoding=\"utf-8-sig\") #csv 파일로 저장"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}