diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts index ffd75600..b0986de7 100644 --- a/src/__tests__/parsers/papi/bookTokenizer.test.ts +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -309,6 +309,76 @@ describe('tokenizeBook', () => { expect(segments[0].tokens[0].surfaceText).toBe('a--b'); }); + it('tokenizes U+2010 (hyphen) between word characters as a single word token', () => { + const text = 'well‐known'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it('tokenizes U+2011 (non-breaking hyphen) between word characters as a single word token', () => { + const text = 'well‑known'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it('tokenizes an unspaced U+2014 (em dash) as punctuation between two words', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'sky—for' }])); + expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([ + ['sky', 'word'], + ['—', 'punctuation'], + ['for', 'word'], + ]); + }); + + it('tokenizes an unspaced U+2013 (en dash) as punctuation between two words', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'ground–man' }])); + expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([ + ['ground', 'word'], + ['–', 'punctuation'], + ['man', 'word'], + ]); + }); + + it('tokenizes an unspaced U+2012 (figure dash) as punctuation between two words', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'a‒b' }])); + expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([ + ['a', 'word'], + ['‒', 'punctuation'], + ['b', 'word'], + ]); + }); + + it('tokenizes an unspaced U+2015 (horizontal bar) as punctuation between two words', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'a―b' }])); + expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([ + ['a', 'word'], + ['―', 'punctuation'], + ['b', 'word'], + ]); + }); + + it('emits each character of a multi-character em dash run as its own punctuation token', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'a——b' }])); + expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([ + ['a', 'word'], + ['—', 'punctuation'], + ['—', 'punctuation'], + ['b', 'word'], + ]); + }); + + it('upholds the charStart/charEnd invariant for tokens around an unspaced em dash', () => { + const text = 'ground—man, and birds of the sky—for'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + segments[0].tokens.forEach((token) => + expect(text.slice(token.charStart, token.charEnd)).toBe(token.surfaceText), + ); + }); + it('upholds the charStart/charEnd invariant for joiner-containing tokens', () => { const text = "it's well-known, don’t you think?"; const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); diff --git a/src/parsers/papi/bookTokenizer.ts b/src/parsers/papi/bookTokenizer.ts index eae30dec..99d8a143 100644 --- a/src/parsers/papi/bookTokenizer.ts +++ b/src/parsers/papi/bookTokenizer.ts @@ -24,17 +24,13 @@ const CHAR_SET = String.raw`\p{L}\p{N}\p{M}\p{Join_Control}`; const GLOTTAL_SET = String.raw`\u0027\u2019`; /** - * Word-internal joiners: + * Characters that join two halves of one word: apostrophes and the shorter hyphens. * - * - \u0027 (Apostrophe) - * - \u002D (Hyphen-minus) - * - \u2010-\u2015 (Unicode hyphens/dashes) - * - \u2019 (Right single quote) + * Excludes \u2012-\u2015, which separate two items rather than joining one word. * - * `\uXXXX` escapes are used for joiner characters to prevent auto-formatters from converting them - * to typographic quotes or other Unicode variants. + * Written as escapes so auto-formatters cannot convert them to typographic variants. */ -const JOIN_SET = String.raw`\u0027\u002D\u2010-\u2015\u2019`; +const JOIN_SET = String.raw`\u0027\u002D\u2010\u2011\u2019`; /** * Matches word tokens and punctuation tokens. Whitespace is not tokenized. @@ -44,8 +40,8 @@ const JOIN_SET = String.raw`\u0027\u002D\u2010-\u2015\u2019`; * absorbed into the surrounding word only when it is both preceded and followed by word characters. * Trailing joiners that are not in GLOTTAL_SET are left as standalone punctuation tokens. * - * Multiple leading or trailing glottal characters are absorbed greedily. Leading hyphens/dashes are - * NOT absorbed. + * Multiple leading or trailing glottal characters are absorbed greedily. Leading hyphens are NOT + * absorbed. * * Multiple consecutive word-internal joiners between word characters are absorbed greedily (e.g. * `a--b` → one token `a--b`).