-
Notifications
You must be signed in to change notification settings - Fork 0
/
nlp_function.py
46 lines (31 loc) · 1.11 KB
/
nlp_function.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
import json
import nltk
import string
import numpy as np
from nltk.corpus import stopwords
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
with open("data/additional_words.json") as json_file:
additional_words = json.load(json_file)
nltk.download("punkt")
nltk.download("stopwords")
stopwords_list = stopwords.words("indonesian") + additional_words
factory = StemmerFactory()
stemmer = factory.create_stemmer()
def tokenization(raw_text):
return nltk.word_tokenize(raw_text.lower())
def remove_punctuation(token):
unPunctuation_token = [word for word in token if word not in string.punctuation]
return unPunctuation_token
def remove_stopWords(unPunctuation_token):
unStopWords_token = [
word for word in unPunctuation_token if word not in stopwords_list
]
return unStopWords_token
def stemming_token(unStopWords_token):
return stemmer.stem(unStopWords_token)
def vectorization(clean_token, all_token):
bag = np.zeros(len(all_token), dtype=np.float32)
for idx, word in enumerate(all_token):
if word in clean_token:
bag[idx] = 1.0
return bag