make compatible to transformers>4.28

.github/workflows/pull_request_test.yml

@@ -55,7 +55,7 @@ jobs:
         /opt/python/${{ matrix.cp }}/bin/python -m pytest --verbose test/test_kiwipiepy.py
     - name: Test transformers_addon
       run: |
-        for v in {12..28}
+        for v in {12..38}
         do
           echo "Test with transformers 4.$v ..."
           if /opt/python/${{ matrix.cp }}/bin/python -m pip install -U "transformers<4.$(($v+1))"; then

kiwipiepy/transformers_addon.py

@@ -66,11 +66,13 @@ class KiwiTokenizer(PreTrainedTokenizerBase):
     vocab_files_names = {"tokenizer_file": "tokenizer.json"}
 
     def __init__(self, tokenizer_file=None, **kwargs):
-        super().__init__(**kwargs)
         if tokenizer_file is None:
             raise ValueError(f"Cannot instantiate tokenizer from {tokenizer_file!r}")
 
         self._tokenizer = SwTokenizer(tokenizer_file)
+
+        super().__init__(**kwargs)
+
         self._post_processor = self._tokenizer.config.additional.get('post_processor') if isinstance(self._tokenizer.config.additional, dict) else None
         if self._post_processor not in (None, 'bert'):
             raise ValueError(f"Unknown post_processor `{self._post_processor!r}`")
@@ -503,4 +505,8 @@ def _save_pretrained(
 
         return file_names
 
+    @property
+    def added_tokens_decoder(self):
+        return {}
+
 AutoTokenizer.register('KiwiTokenizer', None, KiwiTokenizer)

test/sample_tokenizer/tokenizer_config.json

@@ -1,4 +1,5 @@
 {
     "model_max_length": 512,
-    "tokenizer_class": "KiwiTokenizer"
+    "tokenizer_class": "KiwiTokenizer",
+    "added_tokens_decoder": {}
 }

test/test_transformers_addon.py

@@ -1,8 +1,9 @@
+import tempfile
 
-def test_init():
-    from transformers import AutoTokenizer
-    import kiwipiepy.transformers_addon
+from transformers import AutoTokenizer
+import kiwipiepy.transformers_addon
 
+def test_init():
     global tokenizer
     tokenizer = AutoTokenizer.from_pretrained('test/sample_tokenizer')
 
@@ -91,6 +92,12 @@ def test_tokenize():
     t = tokenizer.tokenize("맞습니다요!")
     assert t == ["맞/V", "습니다/E", "요/J", "!"]
 
+def test_save_pretrained():
+    path = tempfile.gettempdir() + '/test_tokenizer'
+    tokenizer.save_pretrained(path)
+    new_tokenizer = AutoTokenizer.from_pretrained(path)
+    assert new_tokenizer.get_vocab() == tokenizer.get_vocab()
+
 if __name__ == '__main__':
     for k, v in locals().copy().items():
         if k.startswith('test'): v()