{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":219035799,"defaultBranch":"main","name":"tokenizers","ownerLogin":"huggingface","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2019-11-01T17:52:20.000Z","ownerAvatar":"https://github.com/avatars/u/25720743?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1723044393.0","currentOid":""},"activityList":{"items":[{"before":"f275e038c7422c62a610a5a82a93fbb7d34c9922","after":"0fe7b800a63b734fe374a96479dd2dba9ce3ea44","ref":"refs/heads/pre_tokenizer-serde","pushedAt":"2024-08-07T18:15:17.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"don't change the test","shortMessageHtmlLink":"don't change the test"}},{"before":null,"after":"07f09e6946d66664bb63b38f69fe70ec02897fb1","ref":"refs/heads/improved_deserialization_error_normalizers","pushedAt":"2024-08-07T15:26:33.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://github.com/avatars/u/204321?s=80&v=4"},"commit":{"message":"Tests + Deserialization improvement for normalizers.","shortMessageHtmlLink":"Tests + Deserialization improvement for normalizers."}},{"before":"dd6652b4de8ab40aab40e77a0e26417e93f667bf","after":"f275e038c7422c62a610a5a82a93fbb7d34c9922","ref":"refs/heads/pre_tokenizer-serde","pushedAt":"2024-08-07T15:24:18.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"fmt","shortMessageHtmlLink":"fmt"}},{"before":"e99daffb59011c19a271d1a27d5479da1c8bf96e","after":"dd6652b4de8ab40aab40e77a0e26417e93f667bf","ref":"refs/heads/pre_tokenizer-serde","pushedAt":"2024-08-07T15:21:50.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"fix rust tests","shortMessageHtmlLink":"fix rust tests"}},{"before":"794209adb25571b3389015cb0714fb3da9090f8a","after":"e99daffb59011c19a271d1a27d5479da1c8bf96e","ref":"refs/heads/pre_tokenizer-serde","pushedAt":"2024-08-07T15:12:02.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"clippy","shortMessageHtmlLink":"clippy"}},{"before":"04e8032a3b9ddf442664ce4536ce0d1d7ed0da98","after":"794209adb25571b3389015cb0714fb3da9090f8a","ref":"refs/heads/pre_tokenizer-serde","pushedAt":"2024-08-07T15:10:40.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"fmt","shortMessageHtmlLink":"fmt"}},{"before":"da394ee510248c0bf5b2408cdc2767263c21bb5c","after":"04e8032a3b9ddf442664ce4536ce0d1d7ed0da98","ref":"refs/heads/pre_tokenizer-serde","pushedAt":"2024-08-07T15:09:34.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"copy from the decoder","shortMessageHtmlLink":"copy from the decoder"}},{"before":null,"after":"da394ee510248c0bf5b2408cdc2767263c21bb5c","ref":"refs/heads/pre_tokenizer-serde","pushedAt":"2024-08-07T14:33:15.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"add deserialize","shortMessageHtmlLink":"add deserialize"}},{"before":"ad2b608dab0388ced11fa2f5f12a543547c659d7","after":null,"ref":"refs/heads/fix-strip","pushedAt":"2024-08-07T13:36:30.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"}},{"before":"bded212356c06e686391a81c6f28098bb20d817c","after":"49dafd707edd7e978dcc13ecfbaa4aaac214de58","ref":"refs/heads/main","pushedAt":"2024-08-07T13:36:28.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"Fix strip python type (#1602)\n\n* update\r\n\r\n* the fix\r\n\r\n* Revert \"update\"\r\n\r\nThis reverts commit 4c2f32f116479b0ec8ccd7c832f86cbc8787d8a9.\r\n\r\n* add a test and rebase\r\n\r\n* style\r\n\r\n* oups","shortMessageHtmlLink":"Fix strip python type (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2453478674\" data-permission-text=\"Title is private\" data-url=\"https://github.com/huggingface/tokenizers/issues/1602\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/huggingface/tokenizers/pull/1602/hovercard\" href=\"https://github.com/huggingface/tokenizers/pull/1602\">#1602</a>)"}},{"before":"724083d82b546ee17dea05135f7dc2904a1cc6f4","after":"ad2b608dab0388ced11fa2f5f12a543547c659d7","ref":"refs/heads/fix-strip","pushedAt":"2024-08-07T13:27:51.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"oups","shortMessageHtmlLink":"oups"}},{"before":"fd6fff5b1ad871d164782d7fe6f250d98f673cbe","after":"724083d82b546ee17dea05135f7dc2904a1cc6f4","ref":"refs/heads/fix-strip","pushedAt":"2024-08-07T13:27:05.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"style","shortMessageHtmlLink":"style"}},{"before":"16e6798518031165da2c1521f8be1fe46f88bbdd","after":"fd6fff5b1ad871d164782d7fe6f250d98f673cbe","ref":"refs/heads/fix-strip","pushedAt":"2024-08-07T13:26:27.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"add a test and rebase","shortMessageHtmlLink":"add a test and rebase"}},{"before":"944f6e559e7105f60025a0cb2a5715cb7fc8bf78","after":"16e6798518031165da2c1521f8be1fe46f88bbdd","ref":"refs/heads/fix-strip","pushedAt":"2024-08-07T13:22:44.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"Revert \"update\"\n\nThis reverts commit 4c2f32f116479b0ec8ccd7c832f86cbc8787d8a9.","shortMessageHtmlLink":"Revert \"update\""}},{"before":null,"after":"944f6e559e7105f60025a0cb2a5715cb7fc8bf78","ref":"refs/heads/fix-strip","pushedAt":"2024-08-07T13:21:46.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"the fix","shortMessageHtmlLink":"the fix"}},{"before":"2de36e33fd945e752aad84741bc9390f797d62af","after":null,"ref":"refs/heads/fix-sequences","pushedAt":"2024-08-07T10:52:37.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"}},{"before":"eea8e1ae6fc4a281513a536d8d4e156ef3b8fb2f","after":"bded212356c06e686391a81c6f28098bb20d817c","ref":"refs/heads/main","pushedAt":"2024-08-07T10:52:35.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"Support `None` to reset pre_tokenizers and normalizers, and index sequences (#1590)\n\n* initial commit\r\n\r\n* support None\r\n\r\n* fix clippy\r\n\r\n* cleanup\r\n\r\n* clean?\r\n\r\n* propagate to pre_tokenizer\r\n\r\n* fix test\r\n\r\n* fix rust tests\r\n\r\n* fix node\r\n\r\n* propagate to decoder and post processor\r\n\r\n* fix calls\r\n\r\n* lint\r\n\r\n* fmt\r\n\r\n* node be happy I am fixing you\r\n\r\n* initial commit\r\n\r\n* support None\r\n\r\n* fix clippy\r\n\r\n* cleanup\r\n\r\n* clean?\r\n\r\n* propagate to pre_tokenizer\r\n\r\n* fix test\r\n\r\n* fix rust tests\r\n\r\n* fix node\r\n\r\n* propagate to decoder and post processor\r\n\r\n* fix calls\r\n\r\n* lint\r\n\r\n* fmt\r\n\r\n* node be happy I am fixing you\r\n\r\n* add a small test\r\n\r\n* styling\r\n\r\n* style merge\r\n\r\n* fix merge test\r\n\r\n* fmt\r\n\r\n* nits\r\n\r\n* update tset","shortMessageHtmlLink":"Support <code>None</code> to reset pre_tokenizers and normalizers, and index seq…"}},{"before":"e7727079b5b81967ce31fe035512b32b6ef6a274","after":"2de36e33fd945e752aad84741bc9390f797d62af","ref":"refs/heads/fix-sequences","pushedAt":"2024-08-07T10:43:24.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"update tset","shortMessageHtmlLink":"update tset"}},{"before":"cf610ed8ab6e532447349a90a9357d1d805efe25","after":null,"ref":"refs/heads/doc-regex","pushedAt":"2024-08-07T10:35:02.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://github.com/avatars/u/204321?s=80&v=4"}},{"before":"6a5fce9fa094d1514a498419f86ac0916e98ef8a","after":"eea8e1ae6fc4a281513a536d8d4e156ef3b8fb2f","ref":"refs/heads/main","pushedAt":"2024-08-07T10:35:01.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://github.com/avatars/u/204321?s=80&v=4"},"commit":{"message":"Fix doc about split (#1591)\n\n* update doc\r\n\r\n* add example\r\n\r\n* Update bindings/python/src/pre_tokenizers.rs\r\n\r\nCo-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>\r\n\r\n* stub\r\n\r\n---------\r\n\r\nCo-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>","shortMessageHtmlLink":"Fix doc about split (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2447837369\" data-permission-text=\"Title is private\" data-url=\"https://github.com/huggingface/tokenizers/issues/1591\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/huggingface/tokenizers/pull/1591/hovercard\" href=\"https://github.com/huggingface/tokenizers/pull/1591\">#1591</a>)"}},{"before":"6fe1889b5e286ce675d293810ac8e2e7a5f052a3","after":null,"ref":"refs/heads/support_bpe_with_whitespace_tokens","pushedAt":"2024-08-07T10:34:55.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://github.com/avatars/u/204321?s=80&v=4"}},{"before":"ab9c7ded8bce66c6da0e94956ff6867cd36af30a","after":"6a5fce9fa094d1514a498419f86ac0916e98ef8a","ref":"refs/heads/main","pushedAt":"2024-08-07T10:34:53.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://github.com/avatars/u/204321?s=80&v=4"},"commit":{"message":"Merges cannot handle tokens containing spaces. (#909)\n\n* Merges cannot handle tokens containing spaces.\r\n\r\nThis fixes this while keeping backward support.\r\nWe don't want to merge that blindly.\r\n\r\n* Update the tests.\r\n\r\n* Fixing clippy.\r\n\r\n* Add a test with spaces in the token/merge.","shortMessageHtmlLink":"Merges cannot handle tokens containing spaces. (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"1140605392\" data-permission-text=\"Title is private\" data-url=\"https://github.com/huggingface/tokenizers/issues/909\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/huggingface/tokenizers/pull/909/hovercard\" href=\"https://github.com/huggingface/tokenizers/pull/909\">#909</a>)"}},{"before":"0321308b53c393556c2beda6731d99c36bb51c37","after":"cf610ed8ab6e532447349a90a9357d1d805efe25","ref":"refs/heads/doc-regex","pushedAt":"2024-08-07T10:26:10.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"stub","shortMessageHtmlLink":"stub"}},{"before":"d76c0cd623daeea0ea322e289d33406d42abb26a","after":"6fe1889b5e286ce675d293810ac8e2e7a5f052a3","ref":"refs/heads/support_bpe_with_whitespace_tokens","pushedAt":"2024-08-07T10:25:27.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://github.com/avatars/u/204321?s=80&v=4"},"commit":{"message":"Add a test with spaces in the token/merge.","shortMessageHtmlLink":"Add a test with spaces in the token/merge."}},{"before":"ceabef3de7324bf6512d0ed96b05c8796f3514c0","after":null,"ref":"refs/heads/add-display","pushedAt":"2024-08-07T10:20:03.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"}},{"before":"886be885c203bf065838d580e72851f533d5fddc","after":"e7727079b5b81967ce31fe035512b32b6ef6a274","ref":"refs/heads/fix-sequences","pushedAt":"2024-08-07T10:18:20.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"fmt","shortMessageHtmlLink":"fmt"}},{"before":"5026d82d7054f556fccadd8df2e733ab2aa6a730","after":"d76c0cd623daeea0ea322e289d33406d42abb26a","ref":"refs/heads/support_bpe_with_whitespace_tokens","pushedAt":"2024-08-07T10:17:35.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"Narsil","name":"Nicolas Patry","path":"/Narsil","primaryAvatarUrl":"https://github.com/avatars/u/204321?s=80&v=4"},"commit":{"message":"Fixing clippy.","shortMessageHtmlLink":"Fixing clippy."}},{"before":"ca1534b494742c3d7ff5ec225a5f2cb86d5ec9da","after":"886be885c203bf065838d580e72851f533d5fddc","ref":"refs/heads/fix-sequences","pushedAt":"2024-08-07T10:16:26.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"fix merge test","shortMessageHtmlLink":"fix merge test"}},{"before":"bce0545af306944765995b5856998b171b5a0b90","after":"0321308b53c393556c2beda6731d99c36bb51c37","ref":"refs/heads/doc-regex","pushedAt":"2024-08-07T10:13:45.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"Update bindings/python/src/pre_tokenizers.rs\n\nCo-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>","shortMessageHtmlLink":"Update bindings/python/src/pre_tokenizers.rs"}},{"before":"af12117959c3333f6023908216a094659a1c2a36","after":"ca1534b494742c3d7ff5ec225a5f2cb86d5ec9da","ref":"refs/heads/fix-sequences","pushedAt":"2024-08-07T10:13:03.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"ArthurZucker","name":"Arthur","path":"/ArthurZucker","primaryAvatarUrl":"https://github.com/avatars/u/48595927?s=80&v=4"},"commit":{"message":"style merge","shortMessageHtmlLink":"style merge"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAElHvoPAA","startCursor":null,"endCursor":null}},"title":"Activity · huggingface/tokenizers"}