Treat an unspaced dash as punctuation, not a word joiner

src/__tests__/parsers/papi/bookTokenizer.test.ts

@@ -309,6 +309,76 @@ describe('tokenizeBook', () => {
       expect(segments[0].tokens[0].surfaceText).toBe('a--b');
     });
 
+    it('tokenizes U+2010 (hyphen) between word characters as a single word token', () => {
+      const text = 'well‐known';
+      const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }]));
+      expect(segments[0].tokens).toHaveLength(1);
+      expect(segments[0].tokens[0].type).toBe('word');
+      expect(segments[0].tokens[0].surfaceText).toBe(text);
+    });
+
+    it('tokenizes U+2011 (non-breaking hyphen) between word characters as a single word token', () => {
+      const text = 'well‑known';
+      const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }]));
+      expect(segments[0].tokens).toHaveLength(1);
+      expect(segments[0].tokens[0].type).toBe('word');
+      expect(segments[0].tokens[0].surfaceText).toBe(text);
+    });
+
+    it('tokenizes an unspaced U+2014 (em dash) as punctuation between two words', () => {
+      const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'sky—for' }]));
+      expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([
+        ['sky', 'word'],
+        ['—', 'punctuation'],
+        ['for', 'word'],
+      ]);
+    });
+
+    it('tokenizes an unspaced U+2013 (en dash) as punctuation between two words', () => {
+      const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'ground–man' }]));
+      expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([
+        ['ground', 'word'],
+        ['–', 'punctuation'],
+        ['man', 'word'],
+      ]);
+    });
+
+    it('tokenizes an unspaced U+2012 (figure dash) as punctuation between two words', () => {
+      const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'a‒b' }]));
+      expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([
+        ['a', 'word'],
+        ['‒', 'punctuation'],
+        ['b', 'word'],
+      ]);
+    });
+
+    it('tokenizes an unspaced U+2015 (horizontal bar) as punctuation between two words', () => {
+      const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'a―b' }]));
+      expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([
+        ['a', 'word'],
+        ['―', 'punctuation'],
+        ['b', 'word'],
+      ]);
+    });
+
+    it('emits each character of a multi-character em dash run as its own punctuation token', () => {
+      const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'a——b' }]));
+      expect(segments[0].tokens.map((t) => [t.surfaceText, t.type])).toEqual([
+        ['a', 'word'],
+        ['—', 'punctuation'],
+        ['—', 'punctuation'],
+        ['b', 'word'],
+      ]);
+    });
+
+    it('upholds the charStart/charEnd invariant for tokens around an unspaced em dash', () => {
+      const text = 'ground—man, and birds of the sky—for';
+      const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }]));
+      segments[0].tokens.forEach((token) =>
+        expect(text.slice(token.charStart, token.charEnd)).toBe(token.surfaceText),
+      );
+    });
+
     it('upholds the charStart/charEnd invariant for joiner-containing tokens', () => {
       const text = "it's well-known, don’t you think?";
       const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }]));

src/parsers/papi/bookTokenizer.ts

@@ -24,17 +24,13 @@ const CHAR_SET = String.raw`\p{L}\p{N}\p{M}\p{Join_Control}`;
 const GLOTTAL_SET = String.raw`\u0027\u2019`;
 
 /**
- * Word-internal joiners:
+ * Characters that join two halves of one word: apostrophes and the shorter hyphens.
  *
- * - \u0027 (Apostrophe)
- * - \u002D (Hyphen-minus)
- * - \u2010-\u2015 (Unicode hyphens/dashes)
- * - \u2019 (Right single quote)
+ * Excludes \u2012-\u2015, which separate two items rather than joining one word.
  *
- * `\uXXXX` escapes are used for joiner characters to prevent auto-formatters from converting them
- * to typographic quotes or other Unicode variants.
+ * Written as escapes so auto-formatters cannot convert them to typographic variants.
  */
-const JOIN_SET = String.raw`\u0027\u002D\u2010-\u2015\u2019`;
+const JOIN_SET = String.raw`\u0027\u002D\u2010\u2011\u2019`;
 
 /**
  * Matches word tokens and punctuation tokens. Whitespace is not tokenized.
@@ -44,8 +40,8 @@ const JOIN_SET = String.raw`\u0027\u002D\u2010-\u2015\u2019`;
  * absorbed into the surrounding word only when it is both preceded and followed by word characters.
  * Trailing joiners that are not in GLOTTAL_SET are left as standalone punctuation tokens.
  *
- * Multiple leading or trailing glottal characters are absorbed greedily. Leading hyphens/dashes are
- * NOT absorbed.
+ * Multiple leading or trailing glottal characters are absorbed greedily. Leading hyphens are NOT
+ * absorbed.
  *
  * Multiple consecutive word-internal joiners between word characters are absorbed greedily (e.g.
  * `a--b` → one token `a--b`).