-
Notifications
You must be signed in to change notification settings - Fork 0
/
matching.py
168 lines (107 loc) · 5.02 KB
/
matching.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
from scrape_arxiv import *
from sending_email import *
import datetime # date and time manipulation
# The following function takes as input a list of scraped papers in the format of the output
# of get_arxiv_newsub() function defined in scrape_arxiv.py. It produces as output an html string with
# all papers that have a specific keyword in the title.
def match_keyword(keyword,list):
listmatch = []
for k in range(len(list[1])):
if keyword.lower() in list[1][k].lower():
listmatch.append(k)
cntk=''
if len(listmatch)!=0:
print(keyword + " found in the following " + list[3] + " submissions:\n")
cntk += '<b>'+keyword.upper()+'</b>' + " found in the following " + list[3] + " submissions:" + '<br><br>\n'
for i in listmatch:
print(str(list[0][i])+")",list[1][i])
cntk += '<b>'+str(list[0][i])+") "+list[1][i]+ '</b>' + '<br>\n'
if list[0][i] / 10 < 1:
print(' Authors: '+ list[2][i]+'\n')
cntk += ' Authors: '+ list[2][i]+ '<br><br>\n'
else:
print(' Authors: '+ list[2][i]+'\n')
cntk += ' Authors: '+ list[2][i]+ '<br><br>\n'
return cntk
if len(listmatch)==0:
pass
# The following function takes as input a list of scraped papers in the format of the output
# of get_arxiv_newsub() function defined in scrape_arxiv.py. It produces as output an html string with
# all papers written by a specific author name.
def match_author(name,list):
listmatch = []
for k in range(len(list[2])):
if name.lower() in list[2][k].lower():
listmatch.append(k)
cntk=''
if len(listmatch)!=0:
print(name + " found in the following " + list[3] + " submissions:\n")
cntk += '<b>'+name.upper()+'</b>' + " found in the following " + list[3] + " submissions:" + '<br><br>\n'
for i in listmatch:
print(str(list[0][i])+")",list[1][i])
cntk += '<b>'+str(list[0][i])+") "+list[1][i]+ '</b>' + '<br>\n'
if list[0][i] / 10 < 1:
print(' Authors: '+ list[2][i]+'\n')
cntk += ' Authors: '+ list[2][i]+ '<br><br>\n'
else:
print(' Authors: '+ list[2][i]+'\n')
cntk += ' Authors: '+ list[2][i]+ '<br><br>\n'
return cntk
if len(listmatch)==0:
pass
def search_keywords(klist,cat='hep-ph',type='new',send=False):
if not isinstance(klist,list):
raise Exception("The input is not a list!")
url = 'http://arxiv.org/list/'+cat+'/new'
soup = make_soup(url)
list_new, list_cross, list_repl = get_arxiv_newsub(soup)
if type == 'new':
lar = list_new
elif type == 'cross':
lar = list_cross
elif type == 'replace':
lar = list_repl
now = datetime.datetime.now()
date = str(now.day) + '-' + str(now.month) + '-' + str(now.year)
print(date +': Searching for user selected keywords in the titles of today\'s '+cat+' '+type+' submissions.' + ' ' + '\n')
cnt=''
for word in klist:
cntp = match_keyword(word,lar)
if cntp!=None:
cnt += cntp
if cnt =='':
cnt += '<b>'+"No matches from your keywords\' list!"+'</b>'+'<br><br>\n'
print("No matches from your keyword\' list!")
cnt += 'For more details visit <a href=\"'+url+'\"+>arxiv.org/list/'+cat+'/new</a>'+'<br><br>\n'
#print(cnt)
if send == True:
# create subject as plain text
subject='Scraping titles of '+cat+' arXiv new submissions - ' + type + ' - ' + date
sendemail(subject,cnt)
def search_authors(autlist,cat='hep-ph',type='new',send=False):
url = 'http://arxiv.org/list/'+cat+'/new'
soup = make_soup(url)
list_new, list_cross, list_repl = get_arxiv_newsub(soup)
if type == 'new':
lar = list_new
elif type == 'cross':
lar = list_cross
elif type == 'replace':
lar = list_repl
now = datetime.datetime.now()
date = str(now.day) + '-' + str(now.month) + '-' + str(now.year)
print(date +': Searching for user selected authors in today\'s '+cat+' '+type+' submissions.' + ' ' + '\n')
cnt=''
for name in autlist:
cntp = match_author(name,lar)
if cntp!=None:
cnt += cntp
if cnt =='':
cnt += '<b>'+"No matches from your authors\' list!"+'</b>'+'<br><br>\n'
print("No matches from your authors\' list!")
cnt += 'For more details visit <a href=\"'+url+'\"+>arxiv.org/list/'+cat+'/new</a>'+'<br><br>\n'
#print(cnt)
if send == True:
# create subject as plain text
subject='Scraping authors of '+cat+' arXiv new submissions - ' + type + ' - ' + date
sendemail(subject,cnt)