-
Notifications
You must be signed in to change notification settings - Fork 2
/
convert_to_text.py
83 lines (52 loc) · 1.57 KB
/
convert_to_text.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
# coding: utf-8
# In[ ]:
# Imports
import pdfplumber
import glob
import pandas as pd
import pathlib
# __Note:__ The parts below are seperated since the first step takes to much time and to be able to continue when terminated.
# Simply, for continuos processing of PDFs.
# In[3]:
# Converting PDFs and append them to texts array
path = "./data/full/"
texts_path = "./data/texts/"
pathlib.Path(texts_path).mkdir(parents=True, exist_ok=True)
files = glob.glob(path + "*")
for f in files:
f_name = f.split("\\")[-1]
txt_path = texts_path + f_name + ".txt"
file_path = pathlib.Path(txt_path)
if file_path.exists():
continue
pdf = pdfplumber.open(f)
text = u''
for page in pdf.pages:
try: # since pdfplumber tries to convert "P14" into decimal for some reason
et = page.extract_text()
except:
pass
if et is not None: # since pdfplumber returns None when an empty page occurs
text += et
else:
print("Extracted text is None in file " + f)
file = open(txt_path, "wb")
file.write(text.encode("utf-8", "ignore"))
file.close()
# In[15]:
# read texts
texts_path = "./data/texts/"
text_files = glob.glob(texts_path + "*")
texts = []
for f in text_files:
with open(f, 'rb') as file:
texts.append(file.read().decode("utf-8"))
# Create pandas dataframe for texts
df = pd.DataFrame({ "text": pd.Series(texts) })
df.head()
# In[16]:
# Number of writings
df.shape
# In[17]:
# Convert dataframe to csv
df.to_csv("data/texts.csv")