-
-
Notifications
You must be signed in to change notification settings - Fork 7
/
gutenberg-metadata.py
86 lines (69 loc) · 2.1 KB
/
gutenberg-metadata.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
#!/usr/bin/env python
"""
Extract the metadata from Project Gutenberg titles using Gutenberg:
https://github.com/c-w/Gutenberg
Tested on Python 3.10.
"""
import argparse
import json
import sys
# https://github.com/c-w/Gutenberg#installation
from gutenberg.query import get_metadata
# http://stackoverflow.com/a/652284/724176
class AutoVivification(dict):
"""Implementation of Perl's autovivification feature."""
def __getitem__(self, item: int):
try:
return dict.__getitem__(self, item)
except KeyError:
value = self[item] = type(self)()
return value
class SetEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, frozenset):
return list(obj)
return json.JSONEncoder.default(self, obj)
def get_all_metadata(last_ebook_id: int) -> None:
metadata = AutoVivification()
i = 1 # First ebook starts at 1
while i <= last_ebook_id:
if i % 100 == 0:
sys.stdout.write(str(i) + "\r")
for feature_name in [
"author",
"formaturi",
"language",
"rights",
"subject",
"title",
]:
data = get_metadata(feature_name, i)
metadata[i][feature_name] = data
i += 1
sys.stdout.write("\r\n")
# from pprint import pprint
# pprint(metadata)
with open("gutenberg-metadata.json", "w") as fp:
json.dump(
metadata,
fp,
cls=SetEncoder,
indent=0,
separators=(",", ":"),
sort_keys=True,
)
class Formatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawTextHelpFormatter):
pass
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Extract the metadata from Project Gutenberg",
formatter_class=Formatter,
)
parser.add_argument(
"id",
type=int,
help="ID of latest released book:\n"
"https://www.gutenberg.org/ebooks/search/?sort_order=release_date\n",
)
args = parser.parse_args()
get_all_metadata(args.id)