-
Notifications
You must be signed in to change notification settings - Fork 8
/
run.py
executable file
·36 lines (30 loc) · 1.54 KB
/
run.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#!/usr/bin/env python
import logging
import os
import sys
LEVEL = logging.INFO # alternative: logging.DEBUG
sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "allennlp"))
sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "transformers", "src"))
from allennlp.common.util import import_module_and_submodules
import_module_and_submodules("allennlp_lib")
sys.path.insert(0, os.path.dirname(os.path.abspath(os.path.join(__file__, os.pardir))))
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
level=LEVEL)
from allennlp.commands import main # pylint: disable=wrong-import-position
import transformers
# Following for handling this issue: https://github.com/allenai/allennlp/issues/4847
# The solution is taken from https://github.com/pytorch/pytorch/issues/973#issuecomment-346405667
# If the issue persists, increase 2048 and/or decrease max_instances_in_memory
import resource
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (20480, rlimit[1]))
if __name__ == "__main__":
if "setting" not in os.environ:
os.environ["setting"] = "full"
print(f'Running in {os.environ["setting"]} setting.')
os.environ["TOKENIZERS_PARALLELISM"] = "0"
os.environ["OMP_NUM_THREADS"] = "1"
logging.info(f"Transformers version: {transformers.__version__}")
logging.info(f"Transformers path: {transformers.__file__}")
main(prog="allennlp")