Skip to content

Commit

Permalink
Add support for showing all maps
Browse files Browse the repository at this point in the history 
Closes #21

Signed-off-by: Tomas Slusny <[email protected]>
  • Loading branch information
@deathbeam
deathbeam committed Oct 30, 2023
1 parent ba07ab8 commit 29c0aa2
Show file tree
Hide file tree
Showing 10 changed files with 2,673 additions and 324 deletions.
52 changes: 52 additions & 0 deletions .github/ISSUE_TEMPLATE/map_data.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,30 @@ body:
- Acton's Nightmare
- Alleyways
- Altered Distant Memory
- Ancient City
- Arachnid Nest
- Arachnid Tomb
- Arcade
- Arena
- Arid Lake
- Armoury
- Arsenal
- Ashen Wood
- Atoll
- Augmented Distant Memory
- Barrows
- Basilica
- Bazaar
- Beach
- Belfry
- Bog
- Bone Crypt
- Bramble Valley
- Burial Chambers
- Caer Blaidd, Wolfpack's Den
- Cage
- Caldera
- Canyon
- Carcass
- Castle Ruins
- Cells
Expand All @@ -51,44 +60,68 @@ body:
- Cold River
- Colonnade
- Colosseum
- Conservatory
- Coral Ruins
- Core
- Cortex
- Courthouse
- Courtyard
- Coves
- Crater
- Crimson Temple
- Crimson Township
- Crystal Ore
- Cursed Crypt
- Dark Forest
- Death and Taxes
- Defiled Cathedral
- Desert
- Desert Spring
- Dig
- Doryani's Machinarium
- Dry Sea
- Dunes
- Dungeon
- Engraved Ultimatum
- Estuary
- Excavation
- Factory
- Fields
- Flooded Mine
- Forbidden Woods
- Forge of the Phoenix
- Forking River
- Foundry
- Frozen Cabins
- Fungal Hollow
- Gardens
- Geode
- Ghetto
- Glacier
- Grave Trough
- Graveyard
- Grotto
- Hall of Grandmasters
- Hallowed Ground
- Harbinger
- Haunted Mansion
- Iceberg
- Infested Valley
- Infused Beachhead
- Ivory Temple
- Jungle Valley
- Laboratory
- Lair
- Lair of the Hydra
- Lava Chamber
- Lava Lake
- Leyline
- Lighthouse
- Lookout
- "Maelstr\xF6m of Chaos"
- Malformation
- Mao Kun
- Marshes
- Mausoleum
- Maze
- Maze of the Minotaur
Expand All @@ -101,47 +134,64 @@ body:
- Oba's Cursed Trove
- Olmec's Sanctum
- Orchard
- Overgrown Ruin
- Overgrown Shrine
- Palace
- Park
- Pen
- Peninsula
- Phantasmagoria
- Pier
- Pillars of Arun
- Pit
- Pit of the Chimera
- Plateau
- Plaza
- Poorjoy's Asylum
- Port
- Precinct
- Primordial Blocks
- Primordial Pool
- Promenade
- Racecourse
- Ramparts
- Reef
- Relic Chambers
- Replica Cortex
- Replica Pillars of Arun
- Replica Poorjoy's Asylum
- Residence
- Rewritten Distant Memory
- Scriptorium
- Sepulchre
- Shipyard
- Shore
- Shrine
- Siege
- Silo
- Spider Forest
- Spider Lair
- Stagnation
- Strand
- Sulphur Vents
- Summit
- Sunken City
- Temple
- Terrace
- The Alluring Abyss
- The Apex of Sacrifice
- The Beachhead
- The Coward's Trial
- The Pale Court
- The Perandus Manor
- The Putrid Cloister
- The Shaper's Realm
- The Tower of Ordeals
- The Twilight Temple
- The Vinktar Square
- Thicket
- Tower
- Toxic Sewer
- Tropical Island
- Twisted Distant Memory
- Underground River
Expand All @@ -151,7 +201,9 @@ body:
- Vaal Temple
- Vault
- Vaults of Atziri
- Villa
- Volcano
- Waste Pool
- Wasteland
- Waterways
- Whakawairua Tuahu
Expand Down
157 changes: 88 additions & 69 deletions data/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ def get_map_wiki(config):
"join_on": "items._pageID=maps._pageID,maps.area_id=areas.id",
"fields": "items.name,maps.area_id,maps.area_level,areas.boss_monster_ids,maps.unique_area_id",
"group_by": "items.name",
"where": "items.class_id='Map' AND maps.area_id LIKE '%MapWorlds%'",
"where": "maps.area_id LIKE '%MapWorlds%' OR maps.series='Unique'",
},
).json()["cargoquery"]
return list(map(lambda x: x["title"], wiki_maps))
Expand All @@ -431,6 +431,26 @@ def get_maps(key, config):
map_wiki = get_map_wiki(config)

out = []

for m in map_wiki:
unique = not m["name"].endswith(" Map")
out_map = {
"name": m["name"],
"level": int(m["area level"]),
"poedb": "https://poedb.tw/us/" + m["name"].replace(" ", "_"),
"boss": {},
}

if unique and m.get("unique area id"):
out_map["id"] = m["unique area id"]
else:
out_map["id"] = m["area id"]
if m.get("boss monster ids"):
out_map["boss"]["ids"] = sorted(
list(set(filter(None, m["boss monster ids"].split(","))))
)
out.append(out_map)

url = config["poedb"]["list"]
print(f"Getting maps from url {url}")
r = requests.get(url)
Expand All @@ -446,13 +466,14 @@ def get_maps(key, config):

map_url = cols[3].find("a").attrs["href"]
map_url = config["poedb"]["base"] + map_url
out_map = {"name": name.strip(), "poedb": map_url}

out.append(out_map)
existing_map = next(filter(lambda x: x["name"] == name.strip(), out), None)
if existing_map:
existing_map["poedb"] = map_url
else:
out_map = {"name": name.strip(), "poedb": map_url}
out.append(out_map)

base_names = sorted(
list(map(lambda x: x["name"], out)) + ["Harbinger Map", "Engraved Ultimatum"]
)
base_names = sorted(list(map(lambda x: x["name"], out)))
mapslist = soup.find(id="MapsUnique").find("table").find("tbody").find_all("tr")

for row in mapslist:
Expand All @@ -467,7 +488,12 @@ def get_maps(key, config):
continue
name = name.replace(n, "")

out.append({"name": name.strip(), "poedb": map_url})
existing_map = next(filter(lambda x: x["name"] == name.strip(), out), None)
if existing_map:
existing_map["poedb"] = map_url
else:
out_map = {"name": name.strip(), "poedb": map_url}
out.append(out_map)

out = list(filter(lambda x: x["name"] not in config["ignored"], out))
out = deduplicate(sorted(out, key=lambda d: d["name"]), "name")
Expand All @@ -492,7 +518,7 @@ def get_maps(key, config):
m["name"] = name
unique = not name.endswith(" Map")
m["shorthand"] = find_shortest_substring(name.replace(" Map", ""), out_names)
m["boss"] = {}
m["boss"] = m.get("boss", {})
m["layout"] = {}
m["rating"] = {}
m["info"] = {}
Expand All @@ -513,79 +539,72 @@ def get_maps(key, config):
existing_rating.pop("name")
existing_rating.pop("density_unreliable")
m["rating"] = existing_rating
existing_wiki = next(filter(lambda x: x["name"] == name, map_wiki), None)
if existing_wiki:
if unique and existing_wiki.get("unique area id"):
m["id"] = existing_wiki["unique area id"]
else:
m["id"] = existing_wiki["area id"]
if existing_wiki.get("boss monster ids"):
m["boss"]["ids"] = sorted(
list(
set(filter(None, existing_wiki["boss monster ids"].split(",")))
)
)
existing_position = next(
filter(lambda x: x["name"] == name.replace(" Map", ""), map_positions), None
)
if existing_position:
m["atlas"] = True
m["x"] = existing_position["x"]
m["y"] = existing_position["y"]

return out


def get_map_data(map_data, extra_map_data):
url = map_data["poedb"]
map_data.pop("poedb")

# PoeDB map metadata
print(f"Getting map data for {map_data['name']} from url {url}")
s = requests.Session()
r = s.get(url)
soup = BeautifulSoup(r.content, "html.parser")
maptabs = soup.find("div", class_="tab-content").findChildren(
"div", recursive=False
)
url = map_data.get("poedb")

if url:
map_data.pop("poedb")

# PoeDB map metadata
print(f"Getting map data for {map_data['name']} from url {url}")
s = requests.Session()
r = s.get(url)
soup = BeautifulSoup(r.content, "html.parser")
maptabs = soup.find("div", class_="tab-content").findChildren(
"div", recursive=False
)

for data in maptabs:
table = data.find("table")
if not table:
continue
tbody = table.find("tbody")
if not tbody:
continue
rows = tbody.find_all("tr")
if not rows:
continue
level_found = False
for data in maptabs:
table = data.find("table")
if not table:
continue
tbody = table.find("tbody")
if not tbody:
continue
rows = tbody.find_all("tr")
if not rows:
continue

for row in rows:
cols = row.find_all("td")
name = cols[0].text.strip().lower()
value = cols[1]
if name == "monster level" and "level" not in map_data:
if map_data.get("tiers"):
continue
map_data["level"] = int(value.text.strip())
elif name == "boss":
map_data["boss"]["names"] = sorted(
list(set(map(lambda x: x.text.strip(), value.find_all("a"))))
)
elif name == "atlas linked":
map_data["connected"] = sorted(
list(set(map(lambda x: x.text.strip(), value.find_all("a"))))
)
elif name == "the pantheon":
map_data["pantheon"] = next(
map(lambda x: x.text.strip(), value.find_all("a"))
)
elif name == "tags":
if "cannot_be_twinned" in value.text.strip():
map_data["boss"]["not_twinnable"] = True
if "no_boss" in value.text.strip():
map_data["boss"].pop("ids", None)
elif name == "icon" and "icon" not in map_data:
map_data["icon"] = value.text.strip()
for row in rows:
cols = row.find_all("td")
name = cols[0].text.strip().lower()
value = cols[1]
if name == "monster level" and not level_found:
level = int(value.text.strip())
if level:
level_found = True
map_data["level"] = level
elif name == "boss":
map_data["boss"]["names"] = sorted(
list(set(map(lambda x: x.text.strip(), value.find_all("a"))))
)
elif name == "atlas linked":
map_data["connected"] = sorted(
list(set(map(lambda x: x.text.strip(), value.find_all("a"))))
)
elif name == "the pantheon":
map_data["pantheon"] = next(
map(lambda x: x.text.strip(), value.find_all("a"))
)
elif name == "tags":
if "cannot_be_twinned" in value.text.strip():
map_data["boss"]["not_twinnable"] = True
if "no_boss" in value.text.strip():
map_data["boss"].pop("ids", None)
elif name == "icon" and "icon" not in map_data:
map_data["icon"] = value.text.strip()

# Merge existing data
existing = next(
Expand Down
Loading

0 comments on commit 29c0aa2

Please sign in to comment.