-
Notifications
You must be signed in to change notification settings - Fork 4
/
multithreaded_crawling.py
62 lines (48 loc) · 1.46 KB
/
multithreaded_crawling.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
# Mirko Mantovani
import threading
from queue import Queue
from crawler import Crawler
from domain_utils import *
from crawling_utils import *
FOLDER = 'uic'
HOMEPAGE = 'https://www.cs.uic.edu/'
DOMAIN_NAME = get_domain_name(HOMEPAGE)
QUEUE_PATH = FOLDER + '/queue.txt'
CRAWLED_PATH = FOLDER + '/crawled.txt'
THREAD_NUMBER = 20
# The queue module implements multi-producer, multi-consumer queues. It is especially useful in threaded programming
# when information must be exchanged safely between multiple threads.
queue = Queue()
counter = 100
Crawler(FOLDER, HOMEPAGE, DOMAIN_NAME)
def start_crawling():
create_workers()
crawl()
# Create worker threads (will die when main exits)
def create_workers():
for _ in range(THREAD_NUMBER):
t = threading.Thread(target=work)
t.daemon = True
t.start()
# Do the next job in the queue
def work():
# global counter
# while counter > 0:
# counter = counter-1
while True:
url = queue.get()
Crawler.crawl_page(threading.current_thread().name, url)
queue.task_done()
# Each queued link is a new job
def create_jobs():
for link in get_set_from_file(QUEUE_PATH):
queue.put(link)
queue.join()
crawl()
# Check if there are items in the queue, if so crawl them
def crawl():
queued_links = get_set_from_file(QUEUE_PATH)
if len(queued_links) > 0:
print(str(len(queued_links)) + ' links in the queue')
create_jobs()
start_crawling()