fix(linker): add lots of dashes to tokenizer

Sefaria · Feb 11, 2024 · b84aeeb · b84aeeb
1 parent df270cb
commit b84aeeb
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/sefaria/spacy_function_registry.py b/sefaria/spacy_function_registry.py
@@ -5,7 +5,7 @@
 def inner_punct_tokenizer_factory():
     def inner_punct_tokenizer(nlp):
         # infix_re = spacy.util.compile_infix_regex(nlp.Defaults.infixes)
-        infix_re = re.compile(r'''[.,?!:;…‘’`“”"'~–\-/()<>]''')
+        infix_re = re.compile(r'''[.,?!:;…‘’`“”"'~–—\-‐‑‒־―⸺⸻/()<>]''')
         prefix_re = spacy.util.compile_prefix_regex(nlp.Defaults.prefixes)
         suffix_re = spacy.util.compile_suffix_regex(nlp.Defaults.suffixes)