[pull] develop from marktext:develop

packages/muya/src/block/base/__tests__/arrowNavigation.spec.ts

@@ -212,6 +212,32 @@ describe('content arrowHandler — trailing-paragraph creation at document end',
         expect(event.stopPropagation).toHaveBeenCalled();
     });
 
+    // #3520: pressing ArrowDown on an already-empty trailing paragraph must NOT
+    // keep appending new empty paragraphs on every keypress. A trailing
+    // paragraph is created only when the current (last) block has content.
+    it('does not append another paragraph when ArrowDown is pressed in an already-empty trailing paragraph (#3520)', async () => {
+        const muya = bootMuya('alpha\n\nbeta\n');
+        const beta = contentByText(muya, 'beta');
+
+        // First ArrowDown at the end of a non-empty last block appends one
+        // trailing empty paragraph (existing, desired behavior).
+        arrowAt(muya, beta, 'ArrowDown', 'beta'.length);
+        await flush();
+        expect(muya.getState().length).toBe(3);
+
+        const appended = muya.editor.scrollPage!.lastContentInDescendant() as Content;
+        expect(appended.text).toBe('');
+
+        // Pressing ArrowDown again, now inside the empty trailing paragraph,
+        // must NOT create a fourth block — the caret stays put.
+        const event = arrowAt(muya, appended, 'ArrowDown', 0);
+        await flush();
+
+        expect(muya.getState().length).toBe(3);
+        expect(appended.getCursor()).not.toBeNull();
+        expect(event.preventDefault).toHaveBeenCalled();
+    });
+
     it('shiftKey held suppresses cross-block navigation (selection extend, not move)', async () => {
         const muya = bootMuya('alpha\n\nbeta\n');
         const alpha = contentByText(muya, 'alpha');

packages/muya/src/block/base/__tests__/convertToListAfterStrong.spec.ts

@@ -0,0 +1,87 @@
+// @vitest-environment happy-dom
+
+import type { Muya } from '../../../muya';
+import type Format from '../format';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { Muya as MuyaClass } from '../../../muya';
+
+vi.mock('../../../utils/prism/index', () => ({
+    default: {},
+    walkTokens: () => null,
+    loadedLanguages: new Set(),
+    transformAliasToOrigin: (s: string) => s,
+    loadLanguage: () => Promise.resolve([]),
+    search: () => [],
+}));
+
+// #2429: typing a list marker on a new (soft-line-break) line after bold text
+// converted the wrong character. `_convertToList`'s regex used a lazy pre-group
+// that grabbed the `*` inside the closing `**` of the bold text as the bullet
+// marker (because two trailing spaces followed it), instead of the `-` the user
+// just typed on the next line — corrupting the bold syntax. The marker must be
+// taken from the start of a line.
+
+const bootedHosts: HTMLElement[] = [];
+
+beforeEach(() => {
+    window.MUYA_VERSION = 'test';
+});
+
+afterEach(() => {
+    while (bootedHosts.length)
+        bootedHosts.pop()!.remove();
+    delete (window as Partial<Window>).MUYA_VERSION;
+});
+
+function bootMuya(markdown: string): Muya {
+    const host = document.createElement('div');
+    document.body.appendChild(host);
+    const muya = new MuyaClass(host, { markdown } as ConstructorParameters<typeof MuyaClass>[1]);
+    muya.init();
+    bootedHosts.push(muya.domNode);
+    return muya;
+}
+
+interface ILiveBlock {
+    blockName?: string;
+    meta?: { marker?: string };
+    firstContentInDescendant: () => { text: string };
+    children?: { forEach: (cb: (b: ILiveBlock) => void) => void };
+}
+
+// Collect the top-level blocks of the live block tree after conversion.
+function convert(text: string): ILiveBlock[] {
+    const muya = bootMuya('seed\n');
+    const content = muya.editor.scrollPage!.firstContentInDescendant() as Format;
+    content.text = text;
+    content.checkInlineUpdate();
+
+    const top: ILiveBlock[] = [];
+    (muya.editor.scrollPage as unknown as ILiveBlock).children!.forEach(b => top.push(b));
+    return top;
+}
+
+describe('_convertToList after bold text + soft-line-break (#2429)', () => {
+    it('uses the `-` on the new line as the marker, not the `*` inside `**`', () => {
+        // `**foo:**` + two spaces + soft-line-break + `- `
+        const blocks = convert('**foo:**  \n- ');
+
+        const list = blocks.find(b => b.blockName === 'bullet-list');
+        expect(list).toBeDefined();
+        // marker is the dash typed on the new line, not a `*` from the bold run.
+        expect(list!.meta!.marker).toBe('-');
+
+        // the bold text is preserved intact as a leading paragraph, not corrupted
+        // into `**foo:*`.
+        const para = blocks.find(b => b.blockName === 'paragraph');
+        expect(para).toBeDefined();
+        expect(para!.firstContentInDescendant().text).toBe('**foo:**');
+    });
+
+    it('still converts a plain single-line `- ` to a bullet list', () => {
+        const blocks = convert('- ');
+        expect(blocks.some(b => b.blockName === 'bullet-list')).toBe(true);
+        // no spurious leading paragraph
+        expect(blocks.some(b => b.blockName === 'paragraph')).toBe(false);
+    });
+});

packages/muya/src/block/base/content.ts

@@ -420,7 +420,7 @@
             event.preventDefault();
     }
 
     arrowHandler(event: Event) {
         if (!isKeyboardEvent(event))
             return;
 
@@ -473,7 +473,10 @@
             if (nextContentBlock) {
                 cursorBlock = nextContentBlock;
             }
-            else {
+            // Only append a trailing paragraph when the last block has content.
+            // Otherwise ArrowDown in an already-empty last paragraph would keep
+            // creating empty paragraphs on every keypress (#3520).
+            else if (this.text.length > 0) {
                 const newNodeState = {
                     name: 'paragraph',
                     text: '',
@@ -485,7 +488,8 @@
                 this.scrollPage?.append(newNode, 'user');
                 cursorBlock = newNode.children.head;
             }
-            offset = adjustOffset(0, cursorBlock, event);
+            if (cursorBlock)
+                offset = adjustOffset(0, cursorBlock, event);
         }
 
         if (cursorBlock) {

packages/muya/src/block/base/format.ts

@@ -56,7 +56,7 @@
     '(?:^|\n) {0,3}((?:\\* *\\* *\\*|- *- *-|_ *_ *_)[ *_-]*)(?=\n|$)', // Thematic break
 ];
 
 const INLINE_UPDATE_REG = new RegExp(INLINE_UPDATE_FRAGMENTS.join('|'), 'i');
 
 // Offset of the cursor relative to a symmetric/asymmetric marker pair
 // (strong/em/code/math/html_tag). `open`/`close` are the opening/closing
@@ -819,8 +819,12 @@
     private _convertToList() {
         const { text, parent, muya, hasSelection } = this;
         const { preferLooseListItem } = muya.options;
+        // The marker must start a line: the pre-group captures whole lines up to
+        // (and including) the newline before the marker, so a `*` inside e.g.
+        // `**bold**` on an earlier soft-line is never mistaken for the bullet
+        // marker (#2429).
         const matches = text.match(
-            /^([\s\S]*?) {0,3}([*+-]|\d{1,9}(?:\.|\))) {1,4}([\s\S]*)$/,
+            /^([\s\S]*\n)? {0,3}([*+-]|\d{1,9}(?:\.|\))) {1,4}([\s\S]*)$/,
         );
         const isOrdered = /\d/.test(matches![2]);
 
@@ -1004,7 +1008,7 @@
         let atxLineHasPushed = false;
 
         for (const l of lines) {
             if (/^ {0,3}#{1,6}(?=\s+|$)/.test(l) && !atxLineHasPushed) {
                 atxLine = l;
                 atxLineHasPushed = true;
             }
@@ -1083,7 +1087,7 @@
         let setextLineHasPushed = false;
 
         for (const l of lines) {
             if (/^ {0,3}(?:={3,}|-{3,})(?= +|$)/.test(l) && !setextLineHasPushed)
                 setextLineHasPushed = true;
             else if (!setextLineHasPushed)
                 setextLines.push(l);

packages/muya/src/inlineRenderer/__tests__/autoLinkTrailingPunct.spec.ts

@@ -0,0 +1,89 @@
+// @vitest-environment happy-dom
+
+import type { Token } from '../types';
+import { describe, expect, it } from 'vitest';
+import { tokenizer } from '../lexer';
+
+// #2096: an extended (bare) autolink swallowed trailing punctuation because the
+// path component matched `\S+`. Per GFM §6.9, trailing punctuation
+// (?!.,:*_~) must not be part of the link.
+
+function autoLinkExt(src: string) {
+    const token = tokenizer(src).find(t => t.type === 'auto_link_extension') as
+        | (Token & { url?: string; www?: string; raw: string })
+        | undefined;
+    return token;
+}
+
+describe('extended autolink — trailing punctuation (#2096)', () => {
+    it('excludes a trailing colon from the link', () => {
+        const token = autoLinkExt('http://some.domain.name/path/to/resource: rest');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('http://some.domain.name/path/to/resource');
+        expect(token!.raw).toBe('http://some.domain.name/path/to/resource');
+    });
+
+    it('excludes a trailing period (sentence end)', () => {
+        const token = autoLinkExt('https://example.com/a/b. Next sentence.');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('https://example.com/a/b');
+    });
+
+    it('keeps interior punctuation, only trims the trailing run', () => {
+        const token = autoLinkExt('https://example.com/a:b:c! end');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('https://example.com/a:b:c');
+    });
+
+    it('leaves a clean URL untouched', () => {
+        const token = autoLinkExt('https://example.com/a/b end');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('https://example.com/a/b');
+    });
+});
+
+// GFM §6.9 also trims the link extent for three further cases. The match is
+// greedy (`\S+`) so these all need post-match trimming, not regex.
+describe('extended autolink — GFM §6.9 extent trimming', () => {
+    // Rule: an unmatched trailing `)` is excluded when the link has more `)`
+    // than `(`, so an autolink can sit inside parentheses.
+    it('excludes a trailing ) when parens are unbalanced', () => {
+        const token = autoLinkExt('(https://en.wikipedia.org/wiki/Foo_(bar)) end');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('https://en.wikipedia.org/wiki/Foo_(bar)');
+    });
+
+    it('keeps a trailing ) when parens are balanced', () => {
+        const token = autoLinkExt('https://example.com/foo(bar) end');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('https://example.com/foo(bar)');
+    });
+
+    // Rule: a trailing `;` closing an `&entity;`-looking reference is excluded.
+    it('excludes a trailing &entity; reference', () => {
+        const token = autoLinkExt('https://example.com/foo?bar=1& end');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('https://example.com/foo?bar=1');
+    });
+
+    it('keeps a bare trailing ; that is not an entity', () => {
+        const token = autoLinkExt('https://example.com/a;b; end');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('https://example.com/a;b;');
+    });
+
+    // Rule: a `<` ends the autolink.
+    it('ends the link at a < character', () => {
+        const token = autoLinkExt('https://example.com/a<b end');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('https://example.com/a');
+    });
+
+    // The rules interleave and apply repeatedly: ").": trim '.' then the now-
+    // unbalanced ')'.
+    it('applies the rules repeatedly (trailing ").")', () => {
+        const token = autoLinkExt('(see https://example.com/path). rest');
+        expect(token).toBeDefined();
+        expect(token!.url).toBe('https://example.com/path');
+    });
+});

packages/muya/src/inlineRenderer/lexer.ts

@@ -515,6 +515,60 @@ function tryHtmlEscape(state: ILexState): boolean {
     return true;
 }
 
+// GFM §6.9 (https://github.github.com/gfm/#autolinks-extension-): trim a
+// www/url autolink's extent to drop characters that are not part of the link.
+// The match is greedy (`\S+`), so these are applied after the regex, mirroring
+// cmark-gfm's `autolink_delim`:
+//   - a `<` ends the autolink;
+//   - trailing punctuation `?!.,:*_~` is excluded (interior is kept);
+//   - a trailing `)` is excluded when the link has more `)` than `(`, so an
+//     autolink can sit inside parentheses;
+//   - a trailing `;` closing an `&entity;`-looking reference is excluded.
+// The last three rules interleave and are applied repeatedly (e.g. `).`).
+function trimAutoLinkExtent(raw: string): string {
+    let end = raw.length;
+
+    const lt = raw.indexOf('<');
+    if (lt !== -1)
+        end = lt;
+
+    let changed = true;
+    while (changed && end > 0) {
+        changed = false;
+        const c = raw[end - 1];
+
+        if ('?!.,:*_~'.includes(c)) {
+            end -= 1;
+            changed = true;
+        }
+        else if (c === ')') {
+            let opening = 0;
+            let closing = 0;
+            for (let i = 0; i < end; i++) {
+                if (raw[i] === '(')
+                    opening += 1;
+                else if (raw[i] === ')')
+                    closing += 1;
+            }
+            if (closing > opening) {
+                end -= 1;
+                changed = true;
+            }
+        }
+        else if (c === ';') {
+            let entityStart = end - 2;
+            while (entityStart >= 0 && /[a-z0-9]/i.test(raw[entityStart]))
+                entityStart -= 1;
+            if (entityStart >= 0 && entityStart < end - 2 && raw[entityStart] === '&') {
+                end = entityStart;
+                changed = true;
+            }
+        }
+    }
+
+    return raw.slice(0, end);
+}
+
 // auto link extension
 function tryAutoLinkExtension(state: ILexState): boolean {
     const autoLinkExtTo = state.inlineRules.auto_link_extension.exec(state.src);
@@ -528,22 +582,41 @@ function tryAutoLinkExtension(state: ILexState): boolean {
         return false;
     }
 
+    let raw = autoLinkExtTo[0];
+    let www = autoLinkExtTo[1];
+    let url = autoLinkExtTo[2];
+    const email = autoLinkExtTo[3];
+
+    // GFM §6.9: trim characters that are not part of a www/url autolink so the
+    // leftover renders as plain text instead (#2096). Email autolinks are
+    // unaffected (their extent is fixed by the domain regex).
+    if (!email) {
+        const trimmed = trimAutoLinkExtent(raw);
+        if (trimmed.length !== raw.length) {
+            raw = trimmed;
+            if (www)
+                www = trimmed;
+            if (url)
+                url = trimmed;
+        }
+    }
+
     pushPending(state);
     state.tokens.push({
         type: 'auto_link_extension',
-        raw: autoLinkExtTo[0],
-        www: autoLinkExtTo[1],
-        url: autoLinkExtTo[2],
-        email: autoLinkExtTo[3],
-        linkType: autoLinkExtTo[1] ? 'www' : autoLinkExtTo[2] ? 'url' : 'email',
+        raw,
+        www,
+        url,
+        email,
+        linkType: www ? 'www' : url ? 'url' : 'email',
         parent: state.tokens,
         range: {
             start: state.pos,
-            end: state.pos + autoLinkExtTo[0].length,
+            end: state.pos + raw.length,
         },
     });
-    state.src = state.src.substring(autoLinkExtTo[0].length);
-    state.pos = state.pos + autoLinkExtTo[0].length;
+    state.src = state.src.substring(raw.length);
+    state.pos = state.pos + raw.length;
 
     return true;
 }

packages/muya/src/state/__tests__/mathTrailingSpace.spec.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from 'vitest';
+import { MarkdownToState } from '../markdownToState';
+
+// #1931: a display-math block whose closing `$$` has trailing whitespace
+// (e.g. `$$ `) was not recognized as math — the block regex required the
+// closing marker to be immediately followed by a newline or end-of-input, so
+// any trailing space made it fall through to plain text. Fenced code blocks
+// already tolerate trailing spaces; math should too.
+
+interface IBlock { name: string; text?: string }
+
+function parse(markdown: string): IBlock[] {
+    return new MarkdownToState({
+        footnote: false,
+        math: true,
+        isGitlabCompatibilityEnabled: false,
+        trimUnnecessaryCodeBlockEmptyLines: false,
+        frontMatter: false,
+    } as never).generate(markdown) as unknown as IBlock[];
+}
+
+describe('block math — closing $$ with trailing whitespace (#1931)', () => {
+    it('parses a math block whose closing $$ has a trailing space', () => {
+        const states = parse('$$\nx = 1\n$$ \n\nbar\n');
+        expect(states.some(s => s.name === 'math-block')).toBe(true);
+    });
+
+    it('parses a math block whose closing $$ has a trailing tab', () => {
+        const states = parse('$$\nx = 1\n$$\t\n\nbar\n');
+        expect(states.some(s => s.name === 'math-block')).toBe(true);
+    });
+
+    it('still parses a math block with no trailing space (regression)', () => {
+        const states = parse('$$\nx = 1\n$$\n\nbar\n');
+        expect(states.some(s => s.name === 'math-block')).toBe(true);
+    });
+});