-
Notifications
You must be signed in to change notification settings - Fork 23
/
wiki_toc.py
45 lines (35 loc) · 1.26 KB
/
wiki_toc.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import csv
import requests
from bs4 import BeautifulSoup
import requests
def get_data(url):
response = requests.get(url)
soup = BeautifulSoup(response.text, 'lxml')
table_of_contents = soup.find("div", id="toc")
headings = table_of_contents.find_all("li")
data = []
for heading in headings:
heading_text = heading.find("span", class_="toctext").text
heading_number = heading.find("span", class_="tocnumber").text
data.append({
'heading_number': heading_number,
'heading_text': heading_text,
})
return data
def export_data(data, file_name):
with open(file_name, "w", newline="") as file:
writer = csv.DictWriter(file, fieldnames=['heading_number', 'heading_text'])
writer.writeheader()
writer.writerows(data)
def main():
url_to_parse = "https://en.wikipedia.org/wiki/Python_(programming_language)"
file_name = "python_toc.csv"
data = get_data(url_to_parse)
export_data(data, file_name)
url_to_parse = "https://en.wikipedia.org/wiki/Web_scraping"
file_name = "web_scraping_toc.csv"
data = get_data(url_to_parse)
export_data(data, file_name)
print('Done')
if __name__ == '__main__':
main()