-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
110 lines (95 loc) · 4.26 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from time import sleep
import pandas as pd # manipulate the date
def get_currencies(currencies, start, end, export_csv=False):
frames = [] # store data for each currency
# Get the historic data between USD and other currencies
for currency in currencies:
while True:
try:
# Open the URL and maximize the window
my_url = f"https://investing.com/currencies/usd-{currency.lower()}-historical-data"
option = Options()
option.add_experimental_option("excludeSwitches", ["enable-logging"])
option.headless = False # Make the actions visible
driver = webdriver.Chrome(options=option)
driver.get(my_url)
print("Got the URL.")
driver.maximize_window()
print("Maximized window.")
sleep(5)
# Accept the cookies, otherwise the prompt does not go away...
cookies_button = WebDriverWait(driver, 20).until(
EC.element_to_be_clickable((By.ID, "onetrust-accept-btn-handler"))
)
cookies_button.click()
print("Accepted the cookies.")
sleep(5)
# Click on the date button to change the range
date_button = WebDriverWait(driver, 20).until(
EC.element_to_be_clickable((By.ID, "flatDatePickerCanvasHol"))
)
date_button.click()
print("Clicked the date button.")
sleep(5)
# Select Start and End Date, clear their contents and input our own
start_bar = WebDriverWait(driver, 20).until(
EC.element_to_be_clickable(
(By.XPATH, "/html/body/div[7]/div[1]/input[1]")
)
)
start_bar.clear()
start_bar.send_keys(start)
print("Entered the start date.")
sleep(5)
end_bar = WebDriverWait(driver, 20).until(
EC.element_to_be_clickable(
(By.XPATH, "/html/body/div[7]/div[1]/input[2]")
)
)
end_bar.clear()
end_bar.send_keys(end)
print("Entered the end date.")
sleep(5)
# Click the apply button and wait a bit
apply_button = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, "/html/body/div[7]/div[5]/a"))
)
apply_button.click()
print("Clicked 'Apply'.")
sleep(5)
# Get the source code of the page that appeared with pandas
# Use the default read_html flavor parameter (lxml)
dataframes = pd.read_html(driver.page_source)
# From the webpage source code we collected, keep only the table containing the historical data
for dataframe in dataframes:
if dataframe.columns.tolist() == [
"Date",
"Price",
"Open",
"High",
"Low",
"Change %",
]:
frames.append(dataframe)
df = dataframe
break
frames.append(df)
# Export to csv if asked by function argument
if export_csv:
df.to_csv("currency.csv", index=False)
print(f"{currency}.csv exported")
driver.quit()
print(f"{currency} scraped.")
break
except:
driver.quit()
print(f"Failed to scrape {currency}. Trying again in 10 seconds.")
sleep(10)
continue
return frames
print(get_currencies(["EUR"], "01/01/2020", "01/01/2021", True))