feat(linker): update tokenizer to include more punctuation.

Sefaria · Oct 26, 2023 · 873c496 · 873c496
1 parent 9d8dc2a
commit 873c496
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/sefaria/spacy_function_registry.py b/sefaria/spacy_function_registry.py
@@ -5,7 +5,7 @@
 def inner_punct_tokenizer_factory():
     def inner_punct_tokenizer(nlp):
         # infix_re = spacy.util.compile_infix_regex(nlp.Defaults.infixes)
-        infix_re = re.compile(r'''[\.\,\?\:\;…\‘\’\`\“\”\"\'~\–\-/\(\)]''')
+        infix_re = re.compile(r'''[.,?!:;…‘’`“”"'~–\-/()<>]''')
         prefix_re = spacy.util.compile_prefix_regex(nlp.Defaults.prefixes)
         suffix_re = spacy.util.compile_suffix_regex(nlp.Defaults.suffixes)