forked from CircuitVerse/recommender
-
Notifications
You must be signed in to change notification settings - Fork 0
/
clean.py
82 lines (65 loc) · 2.59 KB
/
clean.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
from io import StringIO
from html.parser import HTMLParser
import json
import joblib
# Opening JSON file
f = open(r'.\Results\dataset.json', encoding="ascii", errors= "ignore")
# returns JSON object as a dictionary
data = json.load(f)
# define punctuation to remove
punctuations = '''()[]{};:-",<>.?@#$_~'''
#function to remove punctuations
def removepunct(s):
no_punct = ""
for char in s:
if char in punctuations:
s = s.replace(char," ")
return s
#function to remove the HTML tags
class MLStripper(HTMLParser):
def __init__(self):
super().__init__()
self.reset()
self.strict = False
self.convert_charrefs = True
self.text = StringIO()
def handle_data(self, d):
self.text.write(d)
def get_data(self):
return self.text.getvalue()
def strip_tags(html):
s = MLStripper()
s.feed(html)
return s.get_data()
#opening the cleaned datset file to write
f = open(r'.\Results\datasetcleaned.json','w')
li = []
di = {}
#cleaning the dataset
for j in range(len(data)):
di[data[j]["id"]] = j
data[j]["name"] = data[j]["name"].lower().replace("untitled", "")
encoded = data[j]["name"].encode("ascii", "ignore")
data[j]["name"] = encoded.decode()
data[j]["name"] = data[j]["name"].replace("\r\n", "")
data[j]["name"] = removepunct(data[j]["name"])
if data[j]["description"] == "" or data[j]["description"] == None:
data[j]["combined"] = data[j]["name"] + " " + data[j]["name"]
encoded = data[j]["combined"].encode("ascii", "ignore")
data[j]["combined"] = encoded.decode()
data[j]["combined"] = data[j]["combined"].replace("\r\n", "")
data[j]["combined"] = removepunct(data[j]["combined"])
else:
data[j]["description"] = strip_tags(data[j]["description"])
data[j]["description"] = data[j]["description"].encode("ascii", "ignore")
data[j]["description"] = data[j]["description"].decode()
data[j]["description"] = removepunct(data[j]["description"])
data[j]["combined"] = data[j]["description"] + " " + data[j]["name"] + " " + data[j]["name"]
encoded = data[j]["combined"].encode("ascii", "ignore")
data[j]["combined"] = encoded.decode()
data[j]["combined"] = data[j]["combined"].replace("\r\n", "")
data[j]["combined"] = removepunct(data[j]["combined"])
li.append(data[j])
#storing the cleaned dataset
json.dump(li, f, ensure_ascii=True, indent=2)
joblib.dump(di, r'.\Models\mapping.pkl')