-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
94 lines (72 loc) · 3.14 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
from timeit import default_timer as timer
from Crawler import Crawler
import logging
import utils
import sys
import DebugPrinter
class UndefinedCommandError(Exception):
pass
class ArgsWrongTypeError(Exception):
pass
def printUsage():
print("Usage: python main.py -s <SEEDS> -n <LIMIT> [-d]")
exit(1)
def getConfigFromArgs(validCommands):
argsConfig = getConfigDictTemplate()
readAllCommands = False
posCommandExpected = 1
while not readAllCommands:
if sys.argv[posCommandExpected] in validCommands:
if sys.argv[posCommandExpected] == "-s":
argsConfig['seedPathFile'] = sys.argv[posCommandExpected+1]
posCommandExpected += 2
elif sys.argv[posCommandExpected] == "-n":
try:
argsConfig['LIMIT'] = int(sys.argv[posCommandExpected+1])
except ValueError as e:
raise ArgsWrongTypeError("Wrong value type for command '",
sys.argv[posCommandExpected],
"'. type() == int expected but '",
sys.argv[posCommandExpected+1],
"' value was given") from e
else:
posCommandExpected += 2
elif sys.argv[posCommandExpected] == "-d":
argsConfig['debugMode'] = True
posCommandExpected += 1
else:
raise UndefinedCommandError("Unsupported command: ",sys.argv[posCommandExpected])
if posCommandExpected >= len(sys.argv):
readAllCommands = True
return argsConfig
def getConfigDictTemplate():
templateConfig = dict()
templateConfig['seedPathFile'] = ""
templateConfig['LIMIT'] = 0
templateConfig['debugMode'] = False
return templateConfig
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format='%(thread)d-%(threadName)s-%(levelname)s-%(message)s',
filename="log.log", filemode="w")
MINNUMARGS = 5
if len(sys.argv) < MINNUMARGS:
printUsage()
else:
VALIDCOMMANDS = ["-s", "-n", "-d"]
configs = dict()
try:
configs = getConfigFromArgs(VALIDCOMMANDS)
except (UndefinedCommandError, ArgsWrongTypeError) as e:
utils.printErrorMessageAndExitWithErrorCode(e, 1)
else:
logging.info(f"Todos o comandos foram aceitos {configs}")
NUMWORKERS = 80
myCrawler = Crawler(configs['LIMIT'], NUMWORKERS, configs['debugMode'])
try:
start = timer()
myCrawler.startCrawlingFromSeedsFile(configs['seedPathFile'])
end = timer()
totalTime = end - start
logging.info(f"{DebugPrinter.JsonPrinter().getJsonOfDict(myCrawler.getResourcesNumPerHost())}")
except FileNotFoundError as e:
utils.printErrorMessageAndExitWithErrorCode(e, 1)