-
Notifications
You must be signed in to change notification settings - Fork 0
/
finder.py
42 lines (34 loc) · 1.02 KB
/
finder.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
SITE = 'https://ya-goloden.ru'
visited = []
def find404(url,parent):
if url in visited:
return
else:
visited.append(url)
print('url=',url)
try:
html = get_html(url)
soup = BeautifulSoup(html, 'html.parser')
elements = soup.select('a')
for element in elements:
if not element.has_attr('href'):
continue
href = element['href']
if href.startswith('.') or href.startswith('#'):
continue
href = urljoin(url,href)
print('href=',href)
if href.startswith(SITE):
find404(href,url)
except ResourceWarning:
print('Битая ссылка', url, 'на странице', parent)
def get_html(url):
response = requests.get(url)
if response.status_code == 404:
raise ResourceWarning()
return response.text
if __name__ == '__main__':
find404(SITE,SITE)