From 4821e12de07034987a7deaf2dee88eeffa110a7b Mon Sep 17 00:00:00 2001 From: Sergey Shkarin Date: Sat, 31 Oct 2026 19:08:05 -0600 Subject: [PATCH 1/2] Add anySeg to the spaCy Universe Co-Authored-By: Claude Opus 5.5 --- website/meta/universe.json | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index 1f55d96160e..280c1a32786 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1,5 +1,40 @@ { "resources": [ + { + "id": "anyseg", + "title": "anySeg", + "slogan": "Fast tokenizer and sentence segmenter written in Rust, with the conventions of Universal Dependencies", + "description": "anySeg segments text into tokens and sentences with their positions in the source text, following the conventions of Universal Dependencies; the core is written in Rust and the text is never changed. Its spaCy integration adds the tokenizer `anyseg.Tokenizer.v1` and the component `anyseg_sentencizer`, registered through entry points: they are used by name in a config, and a saved pipeline loads with them. The component sets the sentence starts before the parser, which keeps them. With `ru_core_news_sm` on the UD Russian test sets, sentence F1 goes from 95.4% to 98.5% on SynTagRus and from 80.4% to 89.9% on Taiga, LAS from raw text rises by 1 to 3.5 points, and the tokenizer is about ten times faster than the one of spaCy. `anyseg.spacy.to_doc` and `from_doc` convert between a `Doc` and a document with the columns of CoNLL-U. Russian is supported.", + "github": "SergeyShk/anySeg", + "pip": "anyseg[spacy]", + "code_example": [ + "import spacy", + "", + "tokenizer = {\"@tokenizers\": \"anyseg.Tokenizer.v1\"}", + "nlp = spacy.load(\"ru_core_news_sm\", config={\"nlp\": {\"tokenizer\": tokenizer}})", + "nlp.add_pipe(\"anyseg_sentencizer\", before=\"parser\")", + "", + "doc = nlp(\"В 1914 г. цена была 3 000 руб. Затем выросла\")", + "print([token.text for token in doc])", + "print([sentence.text for sentence in doc.sents])" + ], + "code_language": "python", + "url": "https://sergeyshk.github.io/anySeg/spacy/", + "category": [ + "pipeline", + "standalone" + ], + "tags": [ + "russian", + "tokenizer", + "sentence segmentation", + "universal dependencies" + ], + "author": "Sergey Shkarin", + "author_links": { + "github": "SergeyShk" + } + }, { "id": "TeNs", "title": "Temporal Expressions Normalization spaCy", From b94fa4b632fa7480fc99d7839d3a37c942a7bc41 Mon Sep 17 00:00:00 2001 From: Sergey Shkarin Date: Fri, 6 Nov 2026 20:42:53 -0600 Subject: [PATCH 2/2] Add Spanish to the entry of anySeg Co-Authored-By: Claude Opus 5.5 --- website/meta/universe.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index 280c1a32786..545367e981a 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -4,7 +4,7 @@ "id": "anyseg", "title": "anySeg", "slogan": "Fast tokenizer and sentence segmenter written in Rust, with the conventions of Universal Dependencies", - "description": "anySeg segments text into tokens and sentences with their positions in the source text, following the conventions of Universal Dependencies; the core is written in Rust and the text is never changed. Its spaCy integration adds the tokenizer `anyseg.Tokenizer.v1` and the component `anyseg_sentencizer`, registered through entry points: they are used by name in a config, and a saved pipeline loads with them. The component sets the sentence starts before the parser, which keeps them. With `ru_core_news_sm` on the UD Russian test sets, sentence F1 goes from 95.4% to 98.5% on SynTagRus and from 80.4% to 89.9% on Taiga, LAS from raw text rises by 1 to 3.5 points, and the tokenizer is about ten times faster than the one of spaCy. `anyseg.spacy.to_doc` and `from_doc` convert between a `Doc` and a document with the columns of CoNLL-U. Russian is supported.", + "description": "anySeg segments text into tokens, words and sentences with their positions in the source text, following the conventions of Universal Dependencies; the core is written in Rust and the text is never changed. Its spaCy integration adds the tokenizer `anyseg.Tokenizer.v1` and the component `anyseg_sentencizer`, registered through entry points: they are used by name in a config, and a saved pipeline loads with them. The component sets the sentence starts before the parser, which keeps them. With `ru_core_news_sm` on the UD Russian test sets, sentence F1 goes from 95.4% to 98.9% on SynTagRus and from 80.4% to 89.9% on Taiga, LAS from raw text rises by 1 to 3.5 points, and the tokenizer is about ten times faster than the one of spaCy. With `es_core_news_sm` on the UD Spanish test sets, sentence F1 goes from 97.7% to 98.7% on AnCora and from 88.1% to 91.0% on GSD, and LAS rises by 1.2 to 1.7 points. The words of a multiword token, such as Spanish `del`, are in `token._.anyseg_words`. `anyseg.spacy.to_doc` and `from_doc` convert between a `Doc` and a document with the columns of CoNLL-U. Russian and Spanish are supported.", "github": "SergeyShk/anySeg", "pip": "anyseg[spacy]", "code_example": [ @@ -26,6 +26,7 @@ ], "tags": [ "russian", + "spanish", "tokenizer", "sentence segmentation", "universal dependencies"