113 lines
3.2 KiB
Python
113 lines
3.2 KiB
Python
#!/usr/bin/env python
|
|
# coding: utf-8
|
|
|
|
# # Scrape data from aon2e and generate csvs to import in to sqlite
|
|
|
|
# In[ ]:
|
|
|
|
|
|
# Dependencies
|
|
import pandas as pd
|
|
from bs4 import BeautifulSoup as bs, Tag, NavigableString
|
|
import os
|
|
from splinter import Browser
|
|
|
|
# Setting up Selenium
|
|
chrome_driver = os.path.join('..', 'chromedriver.exe')
|
|
executable_path = {'executable_path': chrome_driver}
|
|
browser = Browser('chrome', **executable_path, headless=False)
|
|
|
|
# Pandas config
|
|
pd.set_option('display.max_columns', None)
|
|
|
|
|
|
# In[ ]:
|
|
|
|
|
|
def scrape_description(url, id_number):
|
|
|
|
# Empty lists to store the scraped values
|
|
name_list = []
|
|
description_list = []
|
|
|
|
print(f'Beginning Data Retrieval')
|
|
print(f'------------------------')
|
|
|
|
# Loop from 1 to the value in weapon_number
|
|
for page in range(1, id_number+1):
|
|
|
|
browser.visit(url + str(page))
|
|
html = browser.html
|
|
soup = bs(html, 'html.parser')
|
|
|
|
# Select only the content section
|
|
content = soup.find(id='ctl00_MainContent_DetailedOutput')
|
|
|
|
try:
|
|
# Store the name
|
|
name = content.find('h1', class_='title').a.text.strip()
|
|
name
|
|
|
|
except:
|
|
name = f'name: {page}'
|
|
|
|
try:
|
|
# Start the loop after the link to the book
|
|
start = content.find('a', class_='external-link').next_sibling
|
|
description = ''
|
|
for e in start.next_siblings:
|
|
if isinstance(e, Tag):
|
|
if e.name == 'br':
|
|
if e.next_sibling.name == 'br':
|
|
# If the next 2 elements are br skip this
|
|
# loop it will be handled in the elif
|
|
continue
|
|
elif e.previous_sibling.name == 'br':
|
|
# If this element and the previous are br
|
|
# and the next is not append /n
|
|
description = description + ' /n/n'
|
|
else:
|
|
# If there is just one br append /n
|
|
description = description + ' /n'
|
|
else:
|
|
# Append the text inside the element
|
|
description = description + e.text.strip()
|
|
elif isinstance(e, NavigableString):
|
|
# Since it is just a text append it
|
|
description = description + e
|
|
|
|
except:
|
|
description = f'name: {page}'
|
|
|
|
print(f'{page} of {id_number} | {name}')
|
|
|
|
# Append values to our empty lists
|
|
name_list.append(name)
|
|
description_list.append(description)
|
|
|
|
print(f'------------------------')
|
|
print(f'Data Retrieval Complete')
|
|
|
|
# Create df with the scraped data
|
|
data = {'Name': name_list, 'description': description_list}
|
|
|
|
# Returns a data frame
|
|
return pd.DataFrame(data)
|
|
|
|
|
|
# In[ ]:
|
|
|
|
|
|
# scrape the descriptions
|
|
url_background = 'https://2e.aonprd.com/Backgrounds.aspx?ID='
|
|
number_background = 50 # number to scrape
|
|
|
|
description_background = scrape_description(url_background, number_background)
|
|
|
|
|
|
# In[ ]:
|
|
|
|
|
|
description_background.to_csv('background.csv', encoding='UTF-8', index=False)
|
|
|