-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy path01-getTennisExplorerPlayers.py
48 lines (41 loc) · 1.71 KB
/
01-getTennisExplorerPlayers.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
# -*- coding: utf-8 -*-
import sys
import requests
from bs4 import BeautifulSoup
sys.path.insert(1, 'models')
import db, objects
categories = [{'category': 'ATP', 'sex': 'M', 'URL_BASE': "https://www.tennisexplorer.com/ranking/atp-men/?page="},
{'category': 'WTA', 'sex': 'W', 'URL_BASE': "https://www.tennisexplorer.com/ranking/wta-women/?page="}]
dbConnection = db.Database()
breaksDB = dbConnection.connect()
playersObj = objects.Players(breaksDB)
playersObj.empty() # To delete
for category in categories:
page = 1
ranking = 1
end = False
while not end:
print("# Extracting {} players from the page {}".format(category['category'], page))
url = category['URL_BASE'] + str(page)
r = requests.get(url)
data = r.text
soup = BeautifulSoup(data, "lxml")
players = soup.select("tbody[class=flags] tr")
if len(players) == 0:
end = True
continue
else:
for player in players:
playerDB = {}
tennisExplorerKeyword = player.select("td[class=t-name] a")[0]['href'].split("/")[2]
tennisExplorerCountry = player.select("td[class=tl] a")[0]['href'].split("/?country=")[1]
playerDB['_id'] = tennisExplorerKeyword
playerDB['country'] = tennisExplorerCountry
playerDB['sex'] = category['sex']
playerDB['startingRanking'] = ranking
playerDB['tennisExplorerKeyword'] = tennisExplorerKeyword
playerDB['tennisExplorerName'] = player.select("td[class=t-name]")[0].text
playersObj.create(playerDB)
ranking += 1
page += 1
dbConnection.close()