-
Notifications
You must be signed in to change notification settings - Fork 0
/
preprocessing.py
45 lines (31 loc) · 1.18 KB
/
preprocessing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
from gensim.utils import tokenize
class Preprocessor:
def __init__(self, mode: str = "test", padding_size=3):
self.mode = mode.lower()
self.padding_size = padding_size
@staticmethod
def delete_punct(sentence: str) -> str:
sentence = "".join([c for c in sentence if c == " " or c.isalpha()])
return sentence
@staticmethod
def tokenize(line: str) -> list:
tokenized_line = list(tokenize(line, lowercase=True, deacc=True))
return tokenized_line
def add_special_tokens(self, line: list) -> list:
for i in range(self.padding_size):
line.insert(0, "pad")
line.insert(2, "bos")
if self.mode == "train":
line.insert(len(line), "eos")
return line
def preprocess(self, sentences: list) -> list:
methods = [self.delete_punct, self.tokenize, self.add_special_tokens]
for method in methods:
sentences = list(map(method, sentences))
return sentences
def preprocess_bible(lines: list) -> list:
new_lines = []
for line in lines:
if line != "":
new_lines.append(" ".join(line.split()[1:]))
return new_lines