From d48a48d1ff256fe99dd1f66180c1d2429020f6c9 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:27:55 +0300 Subject: [PATCH 01/17] fix: STORE accepts exactly the flags PERMANENTFLAGS lists In a mailbox with allowPermanentFlags false, SELECT listed the flags its messages have or had in PERMANENTFLAGS, but STORE and the control API checked only the permanentFlags of the mailbox and dropped the others. Both now use server.isPermanentFlag(), the same list SELECT sends, and APPEND and COPY leave out flags the mailbox can not store instead of turning them into new keywords (RFC 3501 section 7.1). Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/docs/extensions/overview.md | 2 +- docs/docs/guides/storage.md | 24 +++++------ src/commands/handlers/store.ts | 8 ++-- src/server.ts | 19 ++++++++- src/store-operations.ts | 2 +- test/store.test.ts | 71 ++++++++++++++++++++++++++++++++ 7 files changed, 107 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 4f7b0f0..603c1f1 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ All RFC 3501 commands are supported. Some choices that the RFCs leave to the ser - The subscription list holds names, not mailboxes (RFC 3501 section 6.3.6). DELETE does not unsubscribe, so LSUB and `LIST (SUBSCRIBED)` keep listing the name (as `\NonExistent` in extended LIST) until UNSUBSCRIBE, and a mailbox created again under that name is subscribed. RENAME leaves the subscription with the old name (RFC 9051 section 6.3.6). A mailbox from the storage object is subscribed unless it has `"subscribed": false`, a new mailbox is not. SUBSCRIBE refuses names that are not mailboxes, UNSUBSCRIBE accepts any name - CREATE `a/b` also creates `a` as a normal mailbox if it does not exist (RFC 3501 section 6.3.3, Dovecot creates a `\Noselect` level instead). An existing `\Noselect` level stays `\Noselect` - DELETE of a mailbox with children leaves a `\Noselect` level that keeps nothing but the children, CREATE of that name makes a new mailbox with a new UIDVALIDITY -- A keyword stays in the FLAGS and PERMANENTFLAGS of a mailbox once a message in it had the keyword, also after that message is expunged (RFC 3501 section 7.2.6, like Dovecot) +- A keyword stays in the FLAGS and PERMANENTFLAGS of a mailbox once a message in it had the keyword, also after that message is expunged (RFC 3501 section 7.2.6, like Dovecot). In a mailbox with `"allowPermanentFlags": false` STORE accepts exactly the flags PERMANENTFLAGS lists (`permanentFlags` and the flags its messages have or had) and ignores the others, APPEND and COPY leave them out (RFC 3501 section 7.1) ### Supported Plugins diff --git a/docs/docs/extensions/overview.md b/docs/docs/extensions/overview.md index e04f375..458beea 100644 --- a/docs/docs/extensions/overview.md +++ b/docs/docs/extensions/overview.md @@ -164,7 +164,7 @@ Some choices that the RFCs leave to the server: - The subscription list holds names, not mailboxes (RFC 3501 section 6.3.6). DELETE does not unsubscribe, so LSUB and `LIST (SUBSCRIBED)` keep listing the name until UNSUBSCRIBE, and a mailbox created again under that name is subscribed. RENAME leaves the subscription with the old name. A mailbox from the storage object is subscribed unless it has `"subscribed": false`, a new mailbox is not. SUBSCRIBE refuses names that are not mailboxes, UNSUBSCRIBE accepts any name. - CREATE `a/b` also creates `a` as a normal mailbox if it does not exist (RFC 3501 section 6.3.3). An existing `\Noselect` level stays `\Noselect`. - DELETE of a mailbox with children leaves a `\Noselect` level that keeps nothing but the children. CREATE of that name makes a new mailbox with a new UIDVALIDITY. -- A keyword stays in the FLAGS and PERMANENTFLAGS of a mailbox once a message in it had the keyword, also after that message is expunged (RFC 3501 section 7.2.6). +- A keyword stays in the FLAGS and PERMANENTFLAGS of a mailbox once a message in it had the keyword, also after that message is expunged (RFC 3501 section 7.2.6). In a mailbox with `"allowPermanentFlags": false` STORE accepts exactly the flags PERMANENTFLAGS lists (`permanentFlags` and the flags its messages have or had) and ignores the others, APPEND and COPY leave them out (RFC 3501 section 7.1). - SEARCH, SORT and THREAD support the `US-ASCII` and `UTF-8` charsets. Any other charset gets `NO [BADCHARSET (US-ASCII UTF-8)]`. The [Mailboxes](./mailboxes.md#core-list-lsub-and-subscriptions) page shows these in transcripts, and [Strict by design](../guides/strict-by-design.md) lists what the core refuses. diff --git a/docs/docs/guides/storage.md b/docs/docs/guides/storage.md index 208b294..5a274fa 100644 --- a/docs/docs/guides/storage.md +++ b/docs/docs/guides/storage.md @@ -185,18 +185,18 @@ S: A2 OK Completed INBOX, every namespace and every entry in a `folders` object take these keys. All of them are optional. -| Key | Default | Description | -| --------------------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `uidvalidity` | `1` | UIDVALIDITY, an integer from 1 to 4294967295. A mailbox created later gets a value higher than any in use. | -| `uidnext` | One more than the highest UID | The next UID. A value lower than the highest message UID plus one is raised. | -| `flags` | `[]` | Mailbox attributes, for example `["\\Noselect"]` or `["\\Noinferiors"]`, in any case. `\HasChildren` and `\HasNoChildren` are set by the server. | -| `permanentFlags` | The `systemFlags` option, `\Answered \Flagged \Draft \Deleted \Seen` | The flags clients can store. | -| `allowPermanentFlags` | `true` | When true, clients can create new keywords (PERMANENTFLAGS ends with `\*`). When false, STORE ignores flags that are not in `permanentFlags`. | -| `knownFlags` | `[]` | Keywords that stay in FLAGS and PERMANENTFLAGS even when no message has them. The server adds every flag a message gets here, a snapshot keeps the list. | -| `subscribed` | `true` | Set `false` to leave the mailbox out of the subscription list, see [Subscriptions](#subscriptions). | -| `messages` | `[]` | The messages, see [Messages](#messages). | -| `folders` | none | Child mailboxes by name. | -| `uid` | `1` | Kept from older versions of the format and not used. Snapshots include it. | +| Key | Default | Description | +| --------------------- | -------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `uidvalidity` | `1` | UIDVALIDITY, an integer from 1 to 4294967295. A mailbox created later gets a value higher than any in use. | +| `uidnext` | One more than the highest UID | The next UID. A value lower than the highest message UID plus one is raised. | +| `flags` | `[]` | Mailbox attributes, for example `["\\Noselect"]` or `["\\Noinferiors"]`, in any case. `\HasChildren` and `\HasNoChildren` are set by the server. | +| `permanentFlags` | The `systemFlags` option, `\Answered \Flagged \Draft \Deleted \Seen` | The flags clients can store. | +| `allowPermanentFlags` | `true` | When true, clients can create new keywords (PERMANENTFLAGS ends with `\*`). When false, the permanent flags are `permanentFlags` and every flag a message of the mailbox has or had (`knownFlags`), the list SELECT sends in PERMANENTFLAGS. STORE ignores other flags, APPEND and COPY leave them out, the control API refuses them with `INVALID`. | +| `knownFlags` | `[]` | Keywords that stay in FLAGS and PERMANENTFLAGS even when no message has them. The server adds every flag a message gets here, a snapshot keeps the list. | +| `subscribed` | `true` | Set `false` to leave the mailbox out of the subscription list, see [Subscriptions](#subscriptions). | +| `messages` | `[]` | The messages, see [Messages](#messages). | +| `folders` | none | Child mailboxes by name. | +| `uid` | `1` | Kept from older versions of the format and not used. Snapshots include it. | A `\Noselect` mailbox is only a hierarchy level: it is listed, but SELECT answers `NO [NONEXISTENT]`. A `\Noinferiors` mailbox can not get children, CREATE below it is answered with `NO [CANNOT]`. diff --git a/src/commands/handlers/store.ts b/src/commands/handlers/store.ts index 3ca7b42..bd8c993 100644 --- a/src/commands/handlers/store.ts +++ b/src/commands/handlers/store.ts @@ -16,8 +16,8 @@ function setFlags(connection: IMAPConnection, message: Message, flags: FlagValue flag = normalizeSystemFlag(typeof flag === 'string' ? flag : String(flag.value || '')); checkSystemFlags(connection.server, flag); - // Ignore if it is not in allowed list and only permament flags are allowed to use - if (mailbox.permanentFlags.indexOf(flag) < 0 && !mailbox.allowPermanentFlags) { + // a flag that is not in PERMANENTFLAGS is ignored (RFC 3501 section 7.1, RFC 9051 section 7.1) + if (!connection.server.isPermanentFlag(mailbox, flag)) { return; } @@ -35,8 +35,8 @@ function addFlags(connection: IMAPConnection, message: Message, flags: FlagValue flag = normalizeSystemFlag(typeof flag === 'string' ? flag : String(flag.value || '')); checkSystemFlags(connection.server, flag); - // Ignore if it is not in allowed list and only permament flags are allowed to use - if (mailbox.permanentFlags.indexOf(flag) < 0 && !mailbox.allowPermanentFlags) { + // a flag that is not in PERMANENTFLAGS is ignored (RFC 3501 section 7.1, RFC 9051 section 7.1) + if (!connection.server.isPermanentFlag(mailbox, flag)) { return; } diff --git a/src/server.ts b/src/server.ts index 12f4aa3..3270ddc 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1230,6 +1230,20 @@ class IMAPServer extends Stream { flags.forEach(flag => this.ensureFlag(knownFlags, flag)); } + /** + * Checks if a flag can be stored on messages of a mailbox: any flag when the mailbox allows new keywords + * (PERMANENTFLAGS has `\*`), otherwise only a flag in the PERMANENTFLAGS list that getStatus() builds, the + * `permanentFlags` of the mailbox and every flag its messages had. RFC 3501 section 7.1 (RFC 9051 section 7.1): + * PERMANENTFLAGS "indicates which of the known flags the client can change permanently", STORE ignores the others + * + * @param {Object} mailbox Mailbox object + * @param {String} flag Normalized flag + * @return {Boolean} true if the flag is a permanent flag of the mailbox + */ + isPermanentFlag(mailbox: Mailbox, flag: string): boolean { + return mailbox.allowPermanentFlags || mailbox.permanentFlags.indexOf(flag) >= 0 || (mailbox.knownFlags || []).indexOf(flag) >= 0; + } + /** * The current time for dates the server sets itself (INTERNALDATE of a message without one, SAVEDATE). The `now` * option fixes it for repeatable tests: a Date, a timestamp, or a function that returns one @@ -1321,9 +1335,10 @@ class IMAPServer extends Stream { ): { mailbox: Mailbox; message: Message } { const mailbox = typeof path === 'string' ? (this.getMailbox(path) as Mailbox) : path; - // processMessage() below sets the UID + // processMessage() below sets the UID. Flags the mailbox can not store are left out (APPEND, COPY: the flags + // SHOULD be set, RFC 3501 sections 6.3.11 and 6.4.7), as STORE ignores them (RFC 3501 section 7.1, PERMANENTFLAGS) const message = Object.assign({}, properties, { - flags: flags, + flags: flags.filter(flag => this.isPermanentFlag(mailbox, flag)), internaldate: internaldate, raw: raw, recent: true diff --git a/src/store-operations.ts b/src/store-operations.ts index f39c618..b213101 100644 --- a/src/store-operations.ts +++ b/src/store-operations.ts @@ -186,7 +186,7 @@ function checkFlags(server: IMAPServer, mailbox: Mailbox, flags: unknown, stored } catch (err) { throw storeError((err as Error).message, 'INVALID'); } - if (stored && mailbox.permanentFlags.indexOf(flag) < 0 && !mailbox.allowPermanentFlags) { + if (stored && !server.isPermanentFlag(mailbox, flag)) { throw storeError('Flag ' + flag + ' is not a permanent flag of ' + mailbox.path, 'INVALID'); } if (list.indexOf(flag) < 0) { diff --git a/test/store.test.ts b/test/store.test.ts index 6ff4022..b677688 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -213,3 +213,74 @@ describe('Custom flags not allowed', () => { }); }); }); + +// RFC 3501 section 7.1 (RFC 9051 section 7.1): PERMANENTFLAGS "indicates which of the known flags the client can +// change permanently", a STORE of a flag that is not in the list is ignored. Flags that messages of the mailbox +// already have are in PERMANENTFLAGS, so STORE must accept them, and APPEND and COPY (RFC 3501 sections 6.3.11 and +// 6.4.7, the flags SHOULD be set) keep only flags the target mailbox can store +describe('PERMANENTFLAGS and STORE agree', () => { + const ctx = setupServer(() => ({ + storage: { + INBOX: { + allowPermanentFlags: false, + permanentFlags: ['\\Seen'], + messages: [ + { raw: 'Subject: hello 1\r\n\r\nWorld 1!', flags: ['\\Seen', '\\Flagged', '$Known'] }, + { raw: 'Subject: hello 2\r\n\r\nWorld 2!', flags: [] } + ] + }, + '': { + folders: { + Other: { + messages: [{ raw: 'Subject: other\r\n\r\nOther', flags: ['\\Draft', '$Known', '$Fresh'] }] + } + } + } + } + })); + + it('STORE sets the flags PERMANENTFLAGS lists and ignores the others', (t, done) => { + const cmds = [ + 'A1 LOGIN testuser testpass', + 'A2 SELECT INBOX', + 'A3 STORE 2 +FLAGS (\\Flagged $Known \\Deleted $New)', + 'A4 STORE 2 FLAGS (\\Seen \\Answered $Known $Other)', + 'ZZ LOGOUT' + ]; + + ctx.run(cmds, resp => { + resp = resp.toString(); + assert.match(resp, /^\* OK \[PERMANENTFLAGS \(\\Seen \\Flagged \$Known\)\]/m); + assert.match(resp, /^\* 2 FETCH \(FLAGS \(\\Flagged \$Known\)\)\r$/m); + assert.match(resp, /^\* 2 FETCH \(FLAGS \(\\Seen \$Known\)\)\r$/m); + done(); + }); + }); + + it('APPEND and COPY keep only the permanent flags', (t, done) => { + const cmds = [ + 'A1 LOGIN testuser testpass', + 'A2 APPEND INBOX (\\Flagged \\Deleted $New) {3}\r\nabc', + 'A3 SELECT Other', + 'A4 COPY 1 INBOX', + 'A5 SELECT INBOX', + 'A6 FETCH 3:4 FLAGS', + 'ZZ LOGOUT' + ]; + + ctx.run(cmds, resp => { + resp = resp.toString(); + assert.match(resp, /^\* 3 FETCH \(FLAGS \(\\Flagged \\Recent\)\)\r$/m); + assert.match(resp, /^\* 4 FETCH \(FLAGS \(\$Known \\Recent\)\)\r$/m); + assert.match(resp, /^\* OK \[PERMANENTFLAGS \(\\Seen \\Flagged \$Known\)\]/m); + done(); + }); + }); + + it('the control API uses the same permanent flags', () => { + assert.deepStrictEqual(ctx.server.control.setFlags('INBOX', [2], ['\\Flagged', '$Known'], 'add'), [{ uid: 2, flags: ['\\Flagged', '$Known'] }]); + assert.throws(() => ctx.server.control.setFlags('INBOX', [2], ['\\Deleted'], 'add'), { code: 'INVALID' }); + assert.strictEqual(ctx.server.control.addMessage('INBOX', { raw: 'Subject: x\r\n\r\nx', flags: ['$Known'] }).uid, 3); + assert.throws(() => ctx.server.control.addMessage('INBOX', { raw: 'Subject: x\r\n\r\nx', flags: ['$New'] }), { code: 'INVALID' }); + }); +}); From 371f99aba307e4e042b76b5b9a484fbc4da434b8 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:29:20 +0300 Subject: [PATCH 02/17] fix: LIST patterns match full names with a prefixed personal namespace An empty LIST reference means the mailbox name is interpreted as by SELECT (RFC 3501 section 6.3.8, RFC 9051 section 6.3.9), so LIST "" "INBOX.%" lists the children of INBOX in a Cyrus style "INBOX." namespace and LIST "" "%" lists INBOX. Namespaces other than the personal one are still hidden from wildcards unless the pattern names their prefix. LIST with an empty mailbox name returns the delimiter and root of the reference namespace. Co-Authored-By: Claude Opus 5.5 --- docs/docs/guides/storage.md | 18 ++-- docs/docs/reference/known-issues.md | 15 ---- src/commands/list.ts | 22 ++++- src/server.ts | 77 +++++++++-------- test/list.test.ts | 128 ++++++++++++++++++++++++++++ 5 files changed, 200 insertions(+), 60 deletions(-) diff --git a/docs/docs/guides/storage.md b/docs/docs/guides/storage.md index 208b294..5749ccb 100644 --- a/docs/docs/guides/storage.md +++ b/docs/docs/guides/storage.md @@ -105,14 +105,10 @@ A namespace object takes these keys, plus the [mailbox keys](#mailboxes): | `type` | `"personal"` | `"personal"`, `"user"` (other users' mailboxes) or `"shared"`. The NAMESPACE plugin lists the namespaces in these three groups ([RFC 2342](https://www.rfc-editor.org/rfc/rfc2342)). | | `folders` | `{}` | The mailboxes of the namespace, by name. | -The first personal namespace is where a LIST with an empty reference looks. If the storage has no personal namespace, ImapKit adds `""` as one. INBOX takes the separator of that namespace unless it sets its own `separator`. +LIST patterns match full mailbox names: with an empty reference the name is interpreted as SELECT would interpret it ([RFC 9051 section 6.3.9](https://www.rfc-editor.org/rfc/rfc9051#section-6.3.9)), so with a prefixed personal namespace like `"INBOX."` the pattern includes the prefix (`LIST "" "INBOX.%"`). The wildcards match the mailboxes of the first personal namespace, INBOX and namespaces without a prefix. The mailboxes of other namespaces are only matched when the pattern names the namespace prefix before any wildcard (`LIST "" "user.%"`, `LIST "#shared/" "*"`), which RFC 9051 allows ("Server implementations are permitted to "hide" otherwise accessible mailboxes from the wildcard characters"). If the storage has no personal namespace, ImapKit adds `""` as one. INBOX takes the separator of the first personal namespace unless it sets its own `separator`. New mailboxes can only be created in personal namespaces. CREATE of a name in a `user` or `shared` namespace is answered with `NO [NOPERM]`. -:::note -With a prefixed personal namespace like `"INBOX."`, LIST with an empty reference currently matches the pattern relative to the prefix, see [Known issues](../reference/known-issues.md#list-with-a-prefixed-personal-namespace). -::: - ### Cyrus A Cyrus style layout keeps personal mailboxes under `INBOX.`, other users under `user.` and shared mailboxes without a prefix: @@ -138,6 +134,18 @@ S: * NAMESPACE (("INBOX." ".")) (("user." ".")) (("" "/")) S: A2 OK Completed ``` +LIST patterns include the `INBOX.` prefix. With the mailboxes `INBOX.Drafts` and `INBOX.Sent`, `%` lists INBOX itself, and `INBOX.%` the mailboxes below it: + +```text +C: A3 LIST "" "%" +S: * LIST (\HasChildren) "." "INBOX" +S: A3 OK Completed +C: A4 LIST "" "INBOX.%" +S: * LIST (\HasNoChildren) "." "INBOX.Drafts" +S: * LIST (\HasNoChildren) "." "INBOX.Sent" +S: A4 OK Completed +``` + ### Gmail Gmail keeps its system mailboxes under a `\Noselect` `[Gmail]` level and marks them with special-use attributes: diff --git a/docs/docs/reference/known-issues.md b/docs/docs/reference/known-issues.md index 3c51e91..4585275 100644 --- a/docs/docs/reference/known-issues.md +++ b/docs/docs/reference/known-issues.md @@ -19,21 +19,6 @@ ImapKit implements IMAP4rev1, IMAP4rev2 and more than 50 extensions, but not eve | Case folding | SEARCH matches strings case-insensitively in the ASCII range only. SORT advertises no `I18NLEVEL`. | | Single user | All users share the same mailbox tree. The ACL plugin limits what users other than the owner can do, but there are no per-user mailboxes. | -### LIST with a prefixed personal namespace - -With a personal namespace that has a prefix, such as `"INBOX."` in the [Cyrus layout](../guides/storage.md#cyrus), LIST with an empty reference matches the pattern relative to the prefix instead of interpreting the name as SELECT does ([RFC 9051 section 6.3.9](https://www.rfc-editor.org/rfc/rfc9051#section-6.3.9)). With mailboxes `INBOX.Drafts` and `INBOX.Sent`: - -```text -C: A6 LIST "" "INBOX.%" -S: A6 OK Completed -C: B1 LIST "" "%" -S: * LIST (\HasNoChildren) "." "INBOX.Drafts" -S: * LIST (\HasNoChildren) "." "INBOX.Sent" -S: B1 OK Completed -``` - -`LIST "" "INBOX.%"` should list both mailboxes, and `LIST "" "%"` should list `INBOX` and not the mailboxes below it. `LIST "" "*"` and `LIST "INBOX." "%"` work. The default storage and the Gmail layout, where the personal namespace is `""`, are not affected. - ## Extensions | Extension | Limitation | diff --git a/src/commands/list.ts b/src/commands/list.ts index 8889681..92ced26 100644 --- a/src/commands/list.ts +++ b/src/commands/list.ts @@ -30,8 +30,24 @@ export default function listCommand(connection: IMAPConnection, parsed: ParsedCo } if (!parsed.attributes[1].value) { - // empty reference lists separator only - const namespace = connection.server.storage[parsed.attributes[1].value || connection.server.referenceNamespace]; + // RFC 3501 section 6.3.8: "An empty ("" string) mailbox name argument is a special request to return the + // hierarchy delimiter and the root name of the name given in the reference. The value returned as the root + // MAY be the empty string if the reference is non-rooted or is an empty string." + const server = connection.server; + const reference: string = parsed.attributes[0].value || ''; + // the namespace of the reference, the one with the longest matching prefix + let key = server.referenceNamespace; + let prefix = ''; + for (const name of Object.keys(server.storage)) { + const exported = connection.exportMailboxName(name); + if (name !== 'INBOX' && exported.length > prefix.length && reference.substr(0, exported.length) === exported) { + key = name; + prefix = exported; + } + } + const namespace = key !== false ? server.storage[key] : null; + // the root of a name in another namespace is the prefix of that namespace, like "#news." in the RFC example + const root = key !== server.referenceNamespace ? prefix : ''; if (namespace) { connection.send( { @@ -45,7 +61,7 @@ export default function listCommand(connection: IMAPConnection, parsed: ParsedCo } ], namespace.separator, - '' + root ] }, 'LIST ITEM', diff --git a/src/server.ts b/src/server.ts index 12f4aa3..ed8e839 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1569,58 +1569,61 @@ class IMAPServer extends Stream { exportName?: ((name: string) => string) | null, folders?: Record | null ): ListedMailbox[] { - let includeINBOX = false; - const source = folders || this.folderCache; - const toName = exportName || ((name: string) => name); - let reference = referenceName || ''; - if (reference === '' && this.referenceNamespace !== false) { - reference = toName(this.referenceNamespace); - includeINBOX = true; - } - // the reference does not have to be a namespace, use the namespace it belongs to - let nsKey: string | false = false; - let nsName = ''; - for (const key of Object.keys(this.storage)) { - const name = toName(key); - if (key !== 'INBOX' && reference.substr(0, name.length) === name && (nsKey === false || name.length > nsName.length)) { - nsKey = key; - nsName = name; - } - } + // RFC 3501 section 6.3.8 (RFC 9051 section 6.3.9): "An empty ("" string) reference name argument + // indicates that the mailbox name is interpreted as by SELECT", and without break out characters + // "the canonical form is normally the reference name appended with the mailbox name". The pattern + // matches full mailbox names, also in a personal namespace with a prefix such as "INBOX." + const lookup = (referenceName || '') + match; + + // "%" does not match the hierarchy delimiter, which is the one of the namespace a name belongs to + const queries = new Map(); + const getQuery = (separator: string, flags = '') => { + const key = separator + '/' + flags; + let query = queries.get(key); + if (!query) { + const pattern = lookup + // escape regex symbols + .replace(/([\\^$+?!.():=[\]{}|,-])/g, '\\$1') + .replace(/[*]/g, '.*') + .replace(/[%]/g, '[^' + separator.replace(/([\\^$+*?!.():=[\]{}|,-])/g, '\\$1') + ']*'); + query = new RegExp('^' + pattern + '$', flags); + queries.set(key, query); + } + return query; + }; - if (nsKey === false) { - return []; - } + // RFC 3501 section 6.3.8 allows to "hide" otherwise accessible mailboxes from the wildcards: the + // mailboxes of namespaces other than the personal one are only matched when the pattern names + // the prefix of the namespace before any wildcard (LIST "" "user.%", LIST "#news." "*") + const fixedPrefix = lookup.replace(/[*%].*$/, ''); + const visible = new Map(); + const isVisible = (key: string) => { + if (!visible.has(key)) { + const name = toName(key); + visible.set(key, key === this.referenceNamespace || fixedPrefix.substr(0, name.length) === name); + } + return visible.get(key); + }; - const namespace = this.storage[nsKey]; - const lookup = reference + match; const result: ListedMailbox[] = []; - const pattern = - '^' + - lookup - // escape regex symbols - .replace(/([\\^$+?!.():=[\]{}|,-])/g, '\\$1') - .replace(/[*]/g, '.*') - .replace(/[%]/g, '[^' + namespace.separator.replace(/([\\^$+*?!.():=[\]{}|,-])/g, '\\$1') + ']*') + - '$'; - const query = new RegExp(pattern, ''); - - // INBOX is case-insensitive - if (includeINBOX && source.INBOX && ((reference ? reference + namespace.separator : '') + 'INBOX').match(new RegExp(pattern, 'i'))) { + // "The special name INBOX is included in the output from LIST, if [...] the uppercase string "INBOX" + // matches the interpreted reference and mailbox name arguments", INBOX is case-insensitive + if (source.INBOX && getQuery((this.storage.INBOX && this.storage.INBOX.separator) || '/', 'i').test('INBOX')) { result.push(source.INBOX); } Object.keys(source).forEach(path => { const folder = source[path]; - if (folder.namespace !== nsKey) { + const nsKey = folder.namespace; + if (path === 'INBOX' || nsKey === false || nsKey === 'INBOX' || !this.storage[nsKey] || !isVisible(nsKey)) { return; } const name = toName(path); - if (name.match(query) && (folder.flags.indexOf('\\NonExistent') < 0 || name === match)) { + if (getQuery(this.storage[nsKey].separator).test(name) && (folder.flags.indexOf('\\NonExistent') < 0 || name === lookup)) { result.push(folder); } }); diff --git a/test/list.test.ts b/test/list.test.ts index 10dbb16..78580df 100644 --- a/test/list.test.ts +++ b/test/list.test.ts @@ -86,6 +86,20 @@ describe('ImapKit tests', () => { }); }); + // RFC 3501 section 6.3.8: an empty mailbox name returns "the hierarchy delimiter and the root name of the + // name given in the reference", the example answers LIST #news.comp.mail.misc "" with "." #news. + it('LIST separator of the reference namespace', (t, done) => { + const cmds = ['A1 LOGIN testuser testpass', 'A2 LIST "#news.comp.mail.misc" ""', 'A3 LIST "#juke?" ""', 'A4 LIST "Test" ""', 'ZZ LOGOUT']; + + ctx.run(cmds, resp => { + resp = resp.toString(); + assert.match(resp, /^\* LIST \(\\Noselect\) "\." "?#news\."?\r\nA2 OK/m); + assert.match(resp, /^\* LIST \(\\Noselect\) "\?" "#juke\?"\r\nA3 OK/m); + assert.match(resp, /^\* LIST \(\\Noselect\) "\/" ""\r\nA4 OK/m); + done(); + }); + }); + it('LIST default namespace', (t, done) => { const cmds = ['A1 LOGIN testuser testpass', 'A2 CAPABILITY', 'A3 LIST "" "*"', 'ZZ LOGOUT']; @@ -122,3 +136,117 @@ describe('ImapKit tests', () => { }); }); }); + +// RFC 3501 section 6.3.8 / RFC 9051 section 6.3.9: "An empty ("" string) reference name argument indicates +// that the mailbox name is interpreted as by SELECT", so a pattern matches the full mailbox name, also +// when the personal namespace has a prefix like "INBOX." (Cyrus layout) +describe('LIST with a prefixed personal namespace', () => { + const ctx = setupServer(() => ({ + plugins: ['LIST-EXTENDED'], + storage: { + INBOX: {}, + 'INBOX.': { + folders: { + Drafts: {}, + Sent: { subscribed: false }, + Work: { subscribed: false, folders: { Done: { subscribed: false } } } + } + }, + 'user.': { + type: 'user', + folders: { other: {} } + }, + '': { + type: 'shared', + folders: { Public: {} } + } + } + })); + + // the mailbox names of the LIST (or LSUB) responses in a transcript + const listed = (resp: string, command = 'LIST') => + (resp.match(new RegExp('^\\* ' + command + ' .*$', 'gm')) || []).map(line => + line + .replace(/^\* \w+ \([^)]*\) "[^"]*" /, '') + .replace(/\r$/, '') + .replace(/^"(.*)"$/, '$1') + ); + + it('matches full names with an empty reference', (t, done) => { + const cmds = ['A1 LOGIN testuser testpass', 'A2 LIST "" "INBOX.%"', 'ZZ LOGOUT']; + ctx.run(cmds, resp => { + resp = resp.toString(); + assert.deepStrictEqual(listed(resp), ['INBOX.Drafts', 'INBOX.Sent', 'INBOX.Work']); + assert.match(resp, /^\* LIST \(\\HasChildren\) "\." "?INBOX\.Work"?\r$/m); + assert.match(resp, /^A2 OK/m); + done(); + }); + }); + + it('"*" after the prefix matches every level', (t, done) => { + const cmds = ['A1 LOGIN testuser testpass', 'A2 LIST "" "INBOX.*"', 'ZZ LOGOUT']; + ctx.run(cmds, resp => { + resp = resp.toString(); + assert.deepStrictEqual(listed(resp), ['INBOX.Drafts', 'INBOX.Sent', 'INBOX.Work', 'INBOX.Work.Done']); + done(); + }); + }); + + it('"%" lists the top level, INBOX but not its children', (t, done) => { + const cmds = ['A1 LOGIN testuser testpass', 'A2 LIST "" "%"', 'ZZ LOGOUT']; + ctx.run(cmds, resp => { + resp = resp.toString(); + // the shared namespace "" has no prefix, its mailboxes are top level names too + assert.deepStrictEqual(listed(resp).sort(), ['INBOX', 'Public']); + assert.match(resp, /^\* LIST \(\\HasChildren\) "\." "?INBOX"?\r$/m); + done(); + }); + }); + + it('INBOX matches case-insensitively, its children do not', (t, done) => { + const cmds = ['A1 LOGIN testuser testpass', 'A2 LIST "" "inbox"', 'A3 LIST "" "inbox.%"', 'ZZ LOGOUT']; + ctx.run(cmds, resp => { + resp = resp.toString(); + assert.deepStrictEqual(listed(resp), ['INBOX']); + assert.match(resp, /^A3 OK/m); + done(); + }); + }); + + it('the reference is a level of hierarchy', (t, done) => { + const cmds = ['A1 LOGIN testuser testpass', 'A2 LIST "INBOX." "%"', 'A3 LIST "INBOX.Work." "*"', 'ZZ LOGOUT']; + ctx.run(cmds, resp => { + resp = resp.toString(); + assert.deepStrictEqual(listed(resp), ['INBOX.Drafts', 'INBOX.Sent', 'INBOX.Work', 'INBOX.Work.Done']); + done(); + }); + }); + + it('other users are listed when the pattern names their namespace', (t, done) => { + const cmds = ['A1 LOGIN testuser testpass', 'A2 LIST "" "*"', 'A3 LIST "" "user.%"', 'A4 LIST "user." "*"', 'ZZ LOGOUT']; + ctx.run(cmds, resp => { + resp = resp.toString(); + const a2 = resp.slice(0, resp.indexOf('A2 OK')); + assert.deepStrictEqual(listed(a2).sort(), ['INBOX', 'INBOX.Drafts', 'INBOX.Sent', 'INBOX.Work', 'INBOX.Work.Done', 'Public']); + assert.deepStrictEqual(listed(resp.slice(resp.indexOf('A2 OK'))), ['user.other', 'user.other']); + done(); + }); + }); + + it('LSUB and extended LIST match full names', (t, done) => { + const cmds = [ + 'A1 LOGIN testuser testpass', + 'A2 LSUB "" "INBOX.%"', + 'A3 LIST (SUBSCRIBED) "" "INBOX.*"', + 'A4 LIST "" ("INBOX" "INBOX.W%")', + 'ZZ LOGOUT' + ]; + ctx.run(cmds, resp => { + resp = resp.toString(); + assert.deepStrictEqual(listed(resp, 'LSUB'), ['INBOX.Drafts']); + assert.match(resp, /^\* LIST \(\\Subscribed \\HasNoChildren\) "\." "?INBOX\.Drafts"?\r$/m); + assert.deepStrictEqual(listed(resp.slice(resp.indexOf('A3 OK'))), ['INBOX', 'INBOX.Work']); + done(); + }); + }); +}); From bcf100c0ae4a68cf9ad93064e58318d1a9175e7e Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:29:32 +0300 Subject: [PATCH 03/17] docs: BINARY of a part that does not exist is empty, like BODY RFC 3516 and RFC 9051 section 6.4.5 define no error for a part number that does not exist, and RFC 3516 section 4.2 gives BINARY the semantics of BODY, so the empty string ImapKit (and Dovecot) send stays. The code comments no longer claim that RFC 3501 section 6.4.5 requires it, and the BINARY docs describe the behavior. Co-Authored-By: Claude Opus 5.5 --- docs/docs/extensions/messages.md | 14 +++++++++++++- src/commands/handlers/fetch.ts | 3 ++- src/plugins/binary.ts | 3 ++- test/binary.test.ts | 2 ++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/docs/extensions/messages.md b/docs/docs/extensions/messages.md index 2ed7474..9bb640a 100644 --- a/docs/docs/extensions/messages.md +++ b/docs/docs/extensions/messages.md @@ -210,7 +210,19 @@ S: A4 OK FETCH Completed The same message in a normal `{96}` literal is `BAD`, as NUL octets are not allowed there. -Refused with `BAD`: `BINARY[]`, BINARY of multipart or message/rfc822 parts (RFC 9051 section 6.4.5 allows leaf body parts only), `HEADER`, `TEXT` or `MIME` sections, and a partial range on `BINARY.SIZE`. A literal8 is refused without a continuation request anywhere but in an APPEND or REPLACE message (and a SETMETADATA value with METADATA). +Refused with `BAD`: `BINARY[]`, BINARY of multipart or message/rfc822 parts (RFC 9051 section 6.4.5 allows leaf body parts only), `HEADER`, `TEXT` or `MIME` sections, and a partial range on `BINARY.SIZE`. + +A part number that does not exist is not an error: `BINARY` returns an empty string and `BINARY.SIZE` 0, the same as `BODY[]` and the same as Dovecot. RFC 3516 and RFC 9051 section 6.4.5 define no error for it, and RFC 3516 section 4.2 gives BINARY the semantics of BODY. For a single part `text/plain` message: + +``` +C: A2 FETCH 1 (BODY.PEEK[1.1] BINARY.PEEK[1.1] BINARY.SIZE[1.1]) +S: * 1 FETCH (BODY[1.1] {0} +S: BINARY[1.1] {0} +S: BINARY.SIZE[1.1] 0) +S: A2 OK FETCH Completed +``` + +A literal8 is refused without a continuation request anywhere but in an APPEND or REPLACE message (and a SETMETADATA value with METADATA). ## PREVIEW diff --git a/src/commands/handlers/fetch.ts b/src/commands/handlers/fetch.ts index 99c9f8e..7dbd967 100644 --- a/src/commands/handlers/fetch.ts +++ b/src/commands/handlers/fetch.ts @@ -94,7 +94,8 @@ function getSection(data: { raw: string; tree: MimeNode }, query: Attribute, glo throw new Error((key || 'Part number') + ' does not take any arguments'); } - // RFC 3501 6.4.5: a part that does not exist is returned as an empty string + // a part that does not exist is returned as an empty string, like Dovecot does: RFC 3501 and RFC 9051 section + // 6.4.5 define no error for it, only an empty string for a partial range that starts beyond the end of the text const node = path ? resolveNode(data.tree, path, global) : data.tree; if (!node) { return ''; diff --git a/src/plugins/binary.ts b/src/plugins/binary.ts index d540d75..dcc996f 100644 --- a/src/plugins/binary.ts +++ b/src/plugins/binary.ts @@ -116,7 +116,8 @@ function getDecodedSection(connection: IMAPConnection, message: Message, query: const node = resolveNode(getMessageData(message).tree, section[0].value, connection.messageGlobal); if (!node) { - // like BODY[
], a part that does not exist is empty + // like BODY[
], a part that does not exist is empty: RFC 3516 and RFC 9051 section 6.4.5 define no + // error for it, and BINARY follows the semantics of BODY (RFC 3516 section 4.2) return ''; } if (!isLeaf(node, connection.messageGlobal)) { diff --git a/test/binary.test.ts b/test/binary.test.ts index f9342a7..ffee5c2 100644 --- a/test/binary.test.ts +++ b/test/binary.test.ts @@ -228,6 +228,8 @@ describe('BINARY', () => { }); }); + // RFC 3516 and RFC 9051 section 6.4.5 define no error for a part that does not exist, and BINARY follows + // the semantics of BODY (RFC 3516 section 4.2), so it is empty like BODY[1.1] of a single part message it('returns an empty string for a part that does not exist, like BODY[]', (t, done) => { ctx.run([LOGIN, SELECT, 'A1 FETCH 2 (BINARY.PEEK[3] BINARY.SIZE[3] BINARY.PEEK[1.1])', 'ZZ LOGOUT'], resp => { resp = resp.toString('binary'); From 41c68564afe6d3aff0b3eb28ee4f63edadf76a53 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:30:18 +0300 Subject: [PATCH 04/17] fix: report new messages and flag changes before FETCH, STORE and SEARCH responses RFC 3501 section 7.4.1 forbids only EXPUNGE during FETCH, STORE and SEARCH, and section 5.2 requires mailbox size updates during a command. Commands that refer to messages now first send the EXISTS and flag updates of other sessions that were queued before any pending EXPUNGE, so a session never gets a FETCH response for a message it was not told about. What was queued after a pending EXPUNGE still waits for it, as do the sequence number forms of COPY and MOVE. Co-Authored-By: Claude Opus 5.5 --- src/plugins/context-search.ts | 4 +- src/server.ts | 92 ++++++++++++++++++++++++-------- test/multi-access.test.ts | 7 ++- test/sessions.test.ts | 99 +++++++++++++++++++++++++++++++++-- 4 files changed, 173 insertions(+), 29 deletions(-) diff --git a/src/plugins/context-search.ts b/src/plugins/context-search.ts index c976c44..5e5828a 100644 --- a/src/plugins/context-search.ts +++ b/src/plugins/context-search.ts @@ -307,8 +307,8 @@ export default function contextSearchPlugin(server: IMAPServer) { // knows the current message list, so new matches can be reported after their EXISTS and FETCH responses // (RFC 5267 section 4.3.2). A command that closes the mailbox gets no updates, its tagged response ends them const processNotifications = connection.processNotifications; - connection.processNotifications = function (this: IMAPConnection, data?: CommandContext | null) { - processNotifications.call(this, data); + connection.processNotifications = function (this: IMAPConnection, data?: CommandContext | null, beforeCommand?: boolean) { + processNotifications.call(this, data, beforeCommand); const command = ((data && data.command) || '').toUpperCase(); if (this.searchContexts && this.searchContexts.size && !this.notificationQueue.length && !CLOSING_COMMANDS.has(command)) { checkContexts(this); diff --git a/src/server.ts b/src/server.ts index 12f4aa3..314ec34 100644 --- a/src/server.ts +++ b/src/server.ts @@ -142,6 +142,17 @@ function stateError(command: string, state: string): string { return command + ' is not allowed in the ' + state + ' state'; } +/** + * Checks if a queued notification changes message sequence numbers by removing messages: an EXPUNGE response, or the + * EXISTS that follows the EXPUNGE responses of another session (it carries the snapshot of the old message list) + * + * @param {Object} notification Queued notification + * @return {Boolean} true if the notification must wait while EXPUNGE responses are not allowed + */ +function isPendingExpunge(notification: Notification): boolean { + return !!notification.mailboxCopy || (!!notification.attributes && (notification.attributes[1] || {}).value === 'EXPUNGE'); +} + /** * Creates a new IMAP server, call `listen()` on it to start accepting connections * @@ -2627,7 +2638,7 @@ class IMAPConnection { * @return {Boolean} true if an EXPUNGE response is pending */ hasPendingExpunge(): boolean { - return this.notificationQueue.some(notification => notification.attributes && (notification.attributes[1] || {}).value === 'EXPUNGE'); + return this.notificationQueue.some(isPendingExpunge); } /** @@ -2864,24 +2875,56 @@ class IMAPConnection { return queue; } - processNotifications(data?: CommandContext | null): void { - const options = data && this.server.getCommandOptions(data.command); - if (options && (options.noExpunge || (options.searchCriteria !== false && this.usesSequenceNumbers(data)))) { - // EXPUNGE responses are not allowed during FETCH, STORE and SEARCH (RFC 3501 section 7.4.1), during - // the commands that extensions add to this list (see the noExpunge command option), nor during UID - // SEARCH with message numbers in the search criteria (RFC 7162 section 3.2.10.2 for VANISHED, EXPUNGE - // may wait as well, RFC 3501 only allows it during UID commands) - return; - } + /** + * Checks if EXPUNGE responses must wait while a command runs: during FETCH, STORE and SEARCH (RFC 3501 section + * 7.4.1), during the commands that extensions add to this list (see the noExpunge command option), and during UID + * SEARCH with message numbers in the search criteria (RFC 7162 section 3.2.10.2 for VANISHED, EXPUNGE may wait as + * well, RFC 3501 only allows it during UID commands) + * + * @param {Object} data Parsed command + * @return {Boolean} true if the command holds back EXPUNGE responses + */ + holdsExpunge(data: CommandContext): boolean { + const options = this.server.getCommandOptions(data.command); + return options.noExpunge || (options.searchCriteria !== false && this.usesSequenceNumbers(data)); + } + /** + * Sends the queued notifications. During a command that holds back EXPUNGE responses (see holdsExpunge), or with + * `beforeCommand` before a command that refers to messages by sequence number, only the notifications queued before + * the first pending EXPUNGE go out: new messages (EXISTS) and flag changes. RFC 3501 section 5.2: "A server MUST send + * mailbox size updates automatically if a mailbox size change is observed during the processing of a command", + * section 7.4.1 forbids only EXPUNGE during FETCH, STORE and SEARCH. What was queued after the EXPUNGE waits with it, + * an EXISTS sent before it would describe a list that the client can not know yet + * + * @param {Object} [data] Parsed command that runs, or null between commands + * @param {Boolean} [beforeCommand] true when the command has not run yet, then a command with message sequence + * numbers (COPY, MOVE) also waits with the EXPUNGE responses, its numbers refer to the messages before them + */ + processNotifications(data?: CommandContext | null, beforeCommand?: boolean): void { if (!this.notificationQueue.length) { return; } - const queue = this.prepareNotifications(this.notificationQueue); - this.notificationQueue = []; + + let queue = this.notificationQueue; + let held: Notification[] = []; + if (data && (this.holdsExpunge(data) || (beforeCommand && this.usesSequenceNumbers(data)))) { + const first = queue.findIndex(isPendingExpunge); + if (first >= 0) { + held = queue.slice(first); + queue = queue.slice(0, first); + } + if (!queue.length) { + return; + } + } // Flag updates use the sequence numbers this session knows: before the EXPUNGE responses of // the snapshot are sent, the snapshot, afterwards the current message list + const snapshot = queue.concat(held).find(notification => notification.mailboxCopy); + this.notificationQueue = held; + queue = this.prepareNotifications(queue); + const snapshotIndex = queue.findIndex(notification => notification.mailboxCopy); const sequenceMaps = new Map>(); const getSequence = (messages: Message[]) => { @@ -2898,10 +2941,10 @@ class IMAPConnection { queue.forEach((notification, i) => { if (notification.flagUpdate) { - // i < snapshotIndex only when there is a snapshot + // before the snapshot (or with all of it still held back) the session knows the old list this.sendFlagUpdate( notification.flagUpdate, - getSequence(i < snapshotIndex ? (queue[snapshotIndex].mailboxCopy as Message[]) : current), + getSequence(snapshot && (snapshotIndex < 0 || i < snapshotIndex) ? (snapshot.mailboxCopy as Message[]) : current), reported ); } else { @@ -3586,13 +3629,20 @@ class IMAPConnection { } } - if (command.substr(0, 4) === 'UID ' && this.hasPendingExpunge()) { - // EXPUNGE responses may be sent during UID commands (RFC 3501 section 7.4.1). The expunges of other sessions - // are reported first, then the command runs on the current mailbox, where the UIDs of the expunged messages - // do not exist and are ignored (RFC 3501 section 6.4.8), so the ghost handling of STORE, COPY and MOVE (RFC 2180 - // section 4) only applies to their sequence number forms. Not for UID SEARCH with message numbers in the - // criteria, processNotifications knows when EXPUNGE must wait - this.processNotifications(element.parsed); + if ( + this.state === 'Selected' && + (command.startsWith('UID ') || options.noExpunge || options.sequenceSet !== false || options.searchCriteria !== false) + ) { + // A command that refers to messages runs on the message list the client was told about. RFC 3501 section 5.2: + // "A server MUST send mailbox size updates automatically if a mailbox size change is observed during the + // processing of a command", so the new messages and flag changes of other sessions are reported first, + // before the command resolves its sequence set or search criteria. + // EXPUNGE responses may be sent during UID commands (RFC 3501 section 7.4.1), so these report the expunges of + // other sessions first as well, then the command runs on the current mailbox, where the UIDs of the expunged + // messages do not exist and are ignored (RFC 3501 section 6.4.8). The ghost handling of STORE, COPY and MOVE + // (RFC 2180 section 4) only applies to their sequence number forms, these and UID SEARCH with message numbers + // in the criteria keep the EXPUNGE responses for later, see processNotifications + this.processNotifications(element.parsed, true); } // changes made while the handler runs are attributed to this session (the `origin` of notifications) diff --git a/test/multi-access.test.ts b/test/multi-access.test.ts index 6d6737d..06dbcb0 100644 --- a/test/multi-access.test.ts +++ b/test/multi-access.test.ts @@ -91,7 +91,11 @@ describe('RFC 2180 multi-accessed mailbox practice', () => { assert.deepStrictEqual(fetches(output), []); assert.match(tagged(output), /^T\d+ OK \[EXPUNGEISSUED\] /); + // the flag changes of A go out before the responses of the FETCH (RFC 3501 section 5.2) assert.deepStrictEqual(fetches(await b.cmd('FETCH 1:* FLAGS')), [ + '* 1 FETCH (UID 1 FLAGS (\\Seen))', + '* 2 FETCH (UID 2 FLAGS (\\Seen))', + '* 3 FETCH (UID 3 FLAGS (\\Seen))', '* 1 FETCH (FLAGS (\\Seen))', '* 2 FETCH (FLAGS (\\Seen))', '* 3 FETCH (FLAGS (\\Seen))' @@ -270,7 +274,8 @@ describe('RFC 2180 multi-accessed mailbox practice', () => { const output = await a.cmd('STORE 1:7 (UNCHANGEDSINCE ' + highest + ') +FLAGS (\\Seen)'); assert.deepStrictEqual( fetches(output).map(line => line.replace(/MODSEQ \(\d+\)/, 'MODSEQ (n)')), - ['* 1 FETCH (FLAGS (\\Seen) MODSEQ (n) UID 1)', '* 3 FETCH (FLAGS (\\Seen) MODSEQ (n) UID 3)'] + // the flag change of B is reported first (RFC 3501 section 5.2), the EXPUNGE waits (section 7.4.1) + ['* 2 FETCH (UID 2 FLAGS (\\Flagged) MODSEQ (n))', '* 1 FETCH (FLAGS (\\Seen) MODSEQ (n) UID 1)', '* 3 FETCH (FLAGS (\\Seen) MODSEQ (n) UID 3)'] ); assert.match(tagged(output), /^T\d+ NO \[MODIFIED 2\] /); }); diff --git a/test/sessions.test.ts b/test/sessions.test.ts index f2f348f..3ec2d8c 100644 --- a/test/sessions.test.ts +++ b/test/sessions.test.ts @@ -242,6 +242,73 @@ describe('Multiple sessions', () => { assert.ok(/(^|\r\n)\* 5 EXISTS\r\n/.test(output), output); }); + // RFC 3501 5.2: "A server MUST send mailbox size updates automatically if a mailbox size change is observed during + // the processing of a command". Section 7.4.1 forbids only EXPUNGE during FETCH, STORE and SEARCH, so the EXISTS + // goes out before the responses of these commands and the new message has a sequence number the client knows + it('FETCH, STORE and SEARCH send the EXISTS of a new message before their own responses (RFC 3501 5.2, 7.4.1)', async () => { + const a = await open('INBOX'); + const b = await open(); + await b.cmd('APPEND INBOX {' + message(5).length + '}\r\n' + message(5)); + + let output = await a.cmd('FETCH 5 (UID)'); + assert.match(output, /^\* 5 EXISTS\r\n(?:\* \d+ RECENT\r\n)?\* 5 FETCH \(UID 5\)\r\nT\d+ OK /); + + await b.cmd('APPEND INBOX {' + message(6).length + '}\r\n' + message(6)); + output = await a.cmd('STORE 6 +FLAGS (\\Flagged)'); + assert.match(output, /^\* 6 EXISTS\r\n(?:\* \d+ RECENT\r\n)?\* 6 FETCH \(FLAGS \(\\Flagged( \\Recent)?\)\)\r\nT\d+ OK /); + + await b.cmd('APPEND INBOX {' + message(7).length + '}\r\n' + message(7)); + output = await a.cmd('SEARCH ALL'); + assert.match(output, /^\* 7 EXISTS\r\n(?:\* \d+ RECENT\r\n)?\* SEARCH 1 2 3 4 5 6 7\r\nT\d+ OK /); + + await b.cmd('APPEND INBOX {' + message(8).length + '}\r\n' + message(8)); + output = await a.cmd('UID FETCH 8 (UID)'); + assert.match(output, /^\* 8 EXISTS\r\n(?:\* \d+ RECENT\r\n)?\* 8 FETCH \(UID 8\)\r\nT\d+ OK /); + }); + + it('a new message that arrives after a pending expunge waits for it, its number is not valid yet (RFC 3501 9)', async () => { + const a = await open('INBOX'); + const b = await open('INBOX'); + await b.cmd('STORE 1 +FLAGS.SILENT (\\Deleted)'); + await b.cmd('EXPUNGE'); + await b.cmd('APPEND INBOX {' + message(5).length + '}\r\n' + message(5)); + + // the EXISTS of the new message can not be sent before the EXPUNGE, which FETCH can not send (RFC 3501 5.2: + // "it is NOT permitted to send an EXISTS response that would reduce the number of messages") + let output = await a.cmd('FETCH 1:* (UID)'); + assert.deepStrictEqual(countResponses(output), []); + assert.deepStrictEqual( + fetchedUids(output).map(pair => pair[1]), + [1, 2, 3, 4] + ); + assert.match(output, /^T\d+ OK \[EXPUNGEISSUED\] /m); + output = await a.cmd('FETCH 5 (UID)'); + assert.match(output, /^T\d+ BAD /m); + + output = await a.cmd('NOOP'); + assert.deepStrictEqual(applyCountResponses([1, 2, 3, 4], output), [2, 3, 4, '?']); + }); + + it('a new message that arrives before a pending expunge is announced first, the expunge waits (RFC 3501 7.4.1)', async () => { + const a = await open('INBOX'); + const b = await open('INBOX'); + await b.cmd('APPEND INBOX {' + message(5).length + '}\r\n' + message(5)); + await b.cmd('STORE 1 +FLAGS.SILENT (\\Deleted)'); + await b.cmd('EXPUNGE'); + + let output = await a.cmd('FETCH 1:* (UID)'); + assert.deepStrictEqual(countResponses(output), ['5 EXISTS']); + assert.match(output, /^\* 5 EXISTS\r\n/); + assert.deepStrictEqual( + fetchedUids(output).map(pair => pair[1]), + [1, 2, 3, 4, 5] + ); + assert.match(output, /^T\d+ OK \[EXPUNGEISSUED\] /m); + + output = await a.cmd('NOOP'); + assert.deepStrictEqual(applyCountResponses([1, 2, 3, 4, 5], output), [2, 3, 4, 5]); + }); + it('is not sent while no command is in progress (RFC 3501 5.3)', async () => { const a = await open('INBOX'); const b = await open(); @@ -296,18 +363,25 @@ describe('Multiple sessions', () => { assert.ok(flagged >= 0 && expunge > flagged && answered > expunge, output); }); - it('reports each message once per flush, with its current flags, after changes held back during FETCH', async () => { + it('FETCH reports flag changes of other sessions before its own responses (RFC 3501 5.2, 7.4.1)', async () => { + const a = await open('INBOX'); + const b = await open('INBOX'); + + await b.cmd('STORE 2 +FLAGS (\\Flagged)'); + // RFC 3501 7.4.1 holds back only EXPUNGE during FETCH, STORE and SEARCH, flag updates are sent (5.2: SHOULD) + const output = await a.cmd('FETCH 1 (UID)'); + assert.match(output, /^\* 2 FETCH \(UID 2 FLAGS \(\\Flagged\)\)\r\n\* 1 FETCH \(UID 1\)\r\nT\d+ OK /); + }); + + it('reports each message once per flush, with its current flags', async () => { const a = await open('INBOX'); const b = await open('INBOX'); await b.cmd('STORE 2 +FLAGS (\\Flagged)'); - // flag updates wait while A runs FETCH (RFC 3501 7.4.1 holds back EXPUNGE, flag updates go with it) - let output = await a.cmd('FETCH 1 (UID)'); - assert.ok(!/^\* 2 FETCH/m.test(output), output); await b.cmd('STORE 2 +FLAGS (\\Answered)'); await b.cmd('STORE 3 +FLAGS (\\Draft)'); await b.cmd('STORE 2 -FLAGS (\\Flagged)'); - output = await a.cmd('NOOP'); + let output = await a.cmd('NOOP'); const fetched = lines(output).filter(line => /^\* \d+ FETCH /.test(line)); assert.deepStrictEqual(fetched, ['* 2 FETCH (UID 2 FLAGS (\\Answered))', '* 3 FETCH (UID 3 FLAGS (\\Draft))'], output); // nothing is reported again later @@ -315,6 +389,21 @@ describe('Multiple sessions', () => { assert.ok(!/^\* \d+ FETCH /m.test(output), output); }); + it('flag changes queued after a pending expunge wait for it during FETCH (RFC 3501 7.4.1)', async () => { + const a = await open('INBOX'); + const b = await open('INBOX'); + + await b.cmd('STORE 2 +FLAGS (\\Flagged)'); + await b.cmd('STORE 1 +FLAGS.SILENT (\\Deleted)'); + await b.cmd('EXPUNGE'); + await b.cmd('STORE 3 +FLAGS (\\Answered)'); + // the change before the expunge goes out with the sequence number A knows, the one after it waits + let output = await a.cmd('FETCH 4 (UID)'); + assert.match(output, /^\* 2 FETCH \(UID 2 FLAGS \(\\Flagged\)\)\r\n\* 4 FETCH \(UID 4\)\r\nT\d+ OK \[EXPUNGEISSUED\] /); + output = await a.cmd('NOOP'); + assert.match(output, /^\* 1 EXPUNGE\r\n\* 3 EXISTS\r\n(?:\* \d+ RECENT\r\n)?\* 3 FETCH \(UID 4 FLAGS \(\\Answered\)\)\r\nT\d+ OK /); + }); + it('the session that changes flags gets them in the STORE response', async () => { const a = await open('INBOX'); const output = await a.cmd('STORE 2 +FLAGS (\\Flagged)'); From ed82da84e24b93cc151183ae4d948104351e5398 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:30:42 +0300 Subject: [PATCH 05/17] fix: end UID SEARCH with OK [EXPUNGEISSUED] when it holds back an EXPUNGE UID SEARCH with message numbers in the criteria keeps the EXPUNGE responses of other sessions back like SEARCH does, but ended with a plain OK. It now carries EXPUNGEISSUED too (RFC 5530 section 3, RFC 9051 section 7.1), as do UID SORT and UID THREAD with message numbers in the criteria. Co-Authored-By: Claude Opus 5.5 --- src/server.ts | 5 +++-- test/multi-access.test.ts | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/server.ts b/src/server.ts index 314ec34..e37b597 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3001,11 +3001,12 @@ class IMAPConnection { response.tag === parsed.tag && response.command === 'OK' && this.hasPendingExpunge() && - this.server.getCommandOptions(parsed.command).noExpunge && + this.holdsExpunge(parsed) && !(Array.isArray(response.attributes) && response.attributes.some(attr => attr && attr.type === 'SECTION')) ) { // After the output handlers, they might add a response code of their own (MODIFIED of CONDSTORE). - // FETCH, STORE, SEARCH and the like can not report the EXPUNGE of another session (RFC 3501 section 7.4.1), + // FETCH, STORE, SEARCH and the like can not report the EXPUNGE of another session (RFC 3501 section 7.4.1), nor + // can UID SEARCH with message numbers in the criteria (see holdsExpunge), // EXPUNGEISSUED tells the client to issue NOOP soon (RFC 5530 section 3, RFC 9051 section 7.1) response.attributes = [{ type: 'SECTION', section: [{ type: 'ATOM', value: 'EXPUNGEISSUED' }] }].concat(response.attributes || []); } diff --git a/test/multi-access.test.ts b/test/multi-access.test.ts index 06dbcb0..65fd96a 100644 --- a/test/multi-access.test.ts +++ b/test/multi-access.test.ts @@ -7,7 +7,8 @@ // - SEARCH: the session view still holds the expunged messages, the tagged OK carries EXPUNGEISSUED (4.3) // - COPY and MOVE: refused with NO [EXPUNGEISSUED] after the pending EXPUNGE responses (4.4.1) // - UID commands: the pending EXPUNGE responses go first (RFC 3501 section 7.4.1), then the UIDs of the expunged -// messages do not exist and are ignored (RFC 3501 section 6.4.8) +// messages do not exist and are ignored (RFC 3501 section 6.4.8). UID SEARCH with message numbers in the criteria +// keeps them back like SEARCH and ends with OK [EXPUNGEISSUED] // - DELETE: other sessions that have the mailbox selected get an untagged BYE (3.3) // - RENAME: the mailbox keeps its messages under the new name, other sessions keep working (3.4) @@ -234,7 +235,8 @@ describe('RFC 2180 multi-accessed mailbox practice', () => { let output = await a.cmd('UID SEARCH 1:7'); // message numbers in the criteria refer to the messages before any EXPUNGE response of the command assert.deepStrictEqual(lines(output), ['* SEARCH 1 2 3 4 5 6 7', tagged(output)]); - assert.match(tagged(output), /^T\d+ OK /); + // like SEARCH, it could not report the EXPUNGE, EXPUNGEISSUED tells the client (RFC 5530 section 3) + assert.match(tagged(output), /^T\d+ OK \[EXPUNGEISSUED\] /); output = await a.cmd('UID SEARCH ALL'); assert.deepStrictEqual(lines(output).slice(0, 5), ['* 4 EXPUNGE', '* 4 EXPUNGE', '* 4 EXPUNGE', '* 4 EXPUNGE', '* 3 EXISTS']); From 1d7d7e77cb660b204c99eafb4bba60125a1b9421 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:31:32 +0300 Subject: [PATCH 06/17] fix: CREATE-SPECIAL-USE answers BAD for USE entries that are not use-attr CREATE W (USE (NIL)) crashed with NO [SERVERBUG]. RFC 6154 section 6 defines use-attr-ext = "\" atom, so NIL, quoted strings, literals, numbers and atoms without a backslash are BAD, and unsupported attributes keep getting NO [USEATTR] (section 3). Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + docs/docs/extensions/mailboxes.md | 8 +++++--- docs/docs/guides/strict-by-design.md | 25 +++++++++++++------------ src/plugins/create-special-use.ts | 9 ++++++++- test/conformance.test.ts | 27 +++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 4f7b0f0..4e02e32 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,7 @@ ImapKit is meant for developing standards compliant IMAP clients, so it follows - the QRESYNC `SELECT` parameter or the `VANISHED` modifier without `ENABLE QRESYNC`, `VANISHED` with `FETCH` or without `CHANGEDSINCE`, and QRESYNC values that break the RFC 7162 grammar (UIDVALIDITY or mod-sequence `0`, `*` in the UID sets, sequence match sets that are not ascending or not of the same size) - unknown `SEARCH RETURN` options or `RETURN` after `CHARSET` (RFC 4466 section 2.6.1), `$` combined with numbers, and `SEARCH MODSEQ` values or entry names that break the RFC 7162 grammar - extended LIST commands (RFC 5258) with unknown options, `RECURSIVEMATCH` without a base option like `SUBSCRIBED` (also `(SPECIAL-USE RECURSIVEMATCH)`, RFC 6154 section 6), an empty pattern list, options with values they do not take, a repeated `STATUS` return option with different items, and invalid `STATUS` items (RFC 5819) +- `CREATE ... (USE (...))` (CREATE-SPECIAL-USE) with entries that are not an atom starting with a backslash, like `NIL`, quoted strings or `Sent` (RFC 6154 section 6: `use-attr-ext = "\" atom`); an attribute the server does not support gets `NO [USEATTR]` (section 3) - METADATA entry names that break RFC 5464 section 3.2 (`//`, a trailing `/`, `*`, `%`, 8-bit or control characters, a scope other than `/private` or `/shared`), values that are atoms or use bare CR or LF as line ends, empty entry or option lists, and GETMETADATA options after the mailbox name (errata 2785) - unknown or uppercase ACL rights, and empty identifiers or identifiers with control characters or invalid UTF-8 (RFC 4314 section 3) - more than one message in `APPEND` without MULTIAPPEND, and with MULTIAPPEND a zero-length message literal cancels the whole `APPEND` with `NO` (RFC 3502) diff --git a/docs/docs/extensions/mailboxes.md b/docs/docs/extensions/mailboxes.md index 51d7390..93b5aac 100644 --- a/docs/docs/extensions/mailboxes.md +++ b/docs/docs/extensions/mailboxes.md @@ -220,16 +220,18 @@ const server = imapkit({ }); ``` -An attribute that is not allowed gets `NO [USEATTR]`, and an attribute that is not an atom or a string gets `BAD`. Load SPECIAL-USE as well so that LIST shows the attributes. +An attribute that is not allowed gets `NO [USEATTR]` ([RFC 6154 section 3](https://www.rfc-editor.org/rfc/rfc6154#section-3)). An entry that is not an atom starting with a backslash (`use-attr-ext = "\" atom`, [RFC 6154 section 6](https://www.rfc-editor.org/rfc/rfc6154#section-6)), such as `NIL`, a quoted string, a literal or `Sent`, gets `BAD`. Load SPECIAL-USE as well so that LIST shows the attributes. ```text C: A2 CREATE Drafts (USE (\Drafts)) S: A2 OK CREATE completed C: A3 CREATE Stuff (USE (\Important)) S: A3 NO [USEATTR] \Important not supported -C: A4 LIST "" "Drafts" +C: A4 CREATE Stuff (USE ("\\Sent")) +S: A4 BAD Invalid syntax for special use flag #1 +C: A5 LIST "" "Drafts" S: * LIST (\HasNoChildren \Drafts) "/" "Drafts" -S: A4 OK Completed +S: A5 OK Completed ``` ## STATUS=SIZE diff --git a/docs/docs/guides/strict-by-design.md b/docs/docs/guides/strict-by-design.md index 17a7edd..ad7b9eb 100644 --- a/docs/docs/guides/strict-by-design.md +++ b/docs/docs/guides/strict-by-design.md @@ -131,18 +131,19 @@ See [Authentication](./authentication.md) for transcripts. These apply when the plugin is loaded. Without it, the extension's commands and arguments are unknown and get `BAD` anyway. -| Extension | Rule | -| ------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| QRESYNC ([RFC 7162](https://www.rfc-editor.org/rfc/rfc7162)) | The QRESYNC `SELECT` parameter or the `VANISHED` modifier without `ENABLE QRESYNC`, `VANISHED` with `FETCH` or without `CHANGEDSINCE`, and values that break the grammar: UIDVALIDITY or mod-sequence `0`, `*` in the UID sets, sequence match sets that are not ascending or not of the same size | -| LIST-EXTENDED ([RFC 5258](https://www.rfc-editor.org/rfc/rfc5258)) | Unknown options, `RECURSIVEMATCH` without a base option like `SUBSCRIBED` (also `(SPECIAL-USE RECURSIVEMATCH)`, [RFC 6154 section 6](https://www.rfc-editor.org/rfc/rfc6154#section-6)), an empty pattern list, options with values they do not take, a repeated `STATUS` return option with different items, invalid `STATUS` items ([RFC 5819](https://www.rfc-editor.org/rfc/rfc5819)) | -| METADATA ([RFC 5464 section 3.2](https://www.rfc-editor.org/rfc/rfc5464#section-3.2)) | Entry names with `//`, a trailing `/`, `*`, `%`, 8-bit or control characters, or a scope other than `/private` or `/shared`, values that are atoms or use bare CR or LF as line ends, empty entry or option lists, and GETMETADATA options after the mailbox name (errata 2785) | -| ACL ([RFC 4314 section 3](https://www.rfc-editor.org/rfc/rfc4314#section-3)) | Unknown or uppercase rights, empty identifiers, identifiers with control characters or invalid UTF-8 | -| CATENATE ([RFC 4469](https://www.rfc-editor.org/rfc/rfc4469)) | URLs that are not absolute-path references (`/INBOX/;UID=1`), including relative-path references like `;UID=1` that [RFC 5092 section 7.2](https://www.rfc-editor.org/rfc/rfc5092#section-7.2) forbids, and URLs of message parts that do not exist (`NO [BADURL ...]`) | -| UIDONLY ([RFC 9586](https://www.rfc-editor.org/rfc/rfc9586)) | Every command that takes message sequence numbers, and sequence sets in search criteria, get `BAD [UIDREQUIRED]` | -| UTF8=ACCEPT ([RFC 9755](https://www.rfc-editor.org/rfc/rfc9755)) | Invalid UTF-8 in quoted strings, `SEARCH CHARSET` after `ENABLE UTF8=ACCEPT`, mailbox names with control characters (in UTF-8, or encoded in modified UTF-7 like `&AA0-`), U+2028, U+2029, a leading BOM, unassigned code points or a name that is not in Unicode Normalization Form C. `NO` for `APPEND` of a message with an 8-bit header before `ENABLE UTF8=ACCEPT` (section 4) | -| BINARY ([RFC 3516](https://www.rfc-editor.org/rfc/rfc3516)) | `BINARY[]`, `BINARY` of multipart or message/rfc822 parts ([RFC 9051 section 6.4.5](https://www.rfc-editor.org/rfc/rfc9051#section-6.4.5) allows leaf body parts only), `HEADER`, `TEXT` or `MIME` sections, and a partial range on `BINARY.SIZE` | -| NOTIFY ([RFC 5465](https://www.rfc-editor.org/rfc/rfc5465)) | MessageNew without MessageExpunge or the other way round, FlagChange without both (section 5), mailbox events or two selected filters with `selected`/`selected-delayed` (section 6.1), fetch attributes outside the selected filters, empty event or mailbox lists, `NOTIFY SET` without event groups. Unknown events get `NO [BADEVENT (...)]` listing the supported ones (section 3.1) | -| IMAP4rev2 ([RFC 9051](https://www.rfc-editor.org/rfc/rfc9051)) | After `ENABLE IMAP4rev2`: `CHECK`, `LSUB`, the `RFC822`, `RFC822.HEADER` and `RFC822.TEXT` FETCH items, the `NEW`, `OLD` and `RECENT` SEARCH keys and the `RECENT` STATUS item (none are in the RFC 9051 grammar, Appendix E), numbers above 63 bits in LARGER and SMALLER, invalid UTF-8, and mailbox names that are not Net-Unicode (section 5.1). Before it: 8-bit quoted strings (Appendix A), and partial ranges, LARGER and SMALLER values above 32 bits | +| Extension | Rule | +| ------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| QRESYNC ([RFC 7162](https://www.rfc-editor.org/rfc/rfc7162)) | The QRESYNC `SELECT` parameter or the `VANISHED` modifier without `ENABLE QRESYNC`, `VANISHED` with `FETCH` or without `CHANGEDSINCE`, and values that break the grammar: UIDVALIDITY or mod-sequence `0`, `*` in the UID sets, sequence match sets that are not ascending or not of the same size | +| LIST-EXTENDED ([RFC 5258](https://www.rfc-editor.org/rfc/rfc5258)) | Unknown options, `RECURSIVEMATCH` without a base option like `SUBSCRIBED` (also `(SPECIAL-USE RECURSIVEMATCH)`, [RFC 6154 section 6](https://www.rfc-editor.org/rfc/rfc6154#section-6)), an empty pattern list, options with values they do not take, a repeated `STATUS` return option with different items, invalid `STATUS` items ([RFC 5819](https://www.rfc-editor.org/rfc/rfc5819)) | +| CREATE-SPECIAL-USE ([RFC 6154 section 6](https://www.rfc-editor.org/rfc/rfc6154#section-6)) | `USE` entries that are not an atom starting with a backslash (`use-attr-ext = "\" atom`), like `NIL`, quoted strings, literals or `Sent`. An attribute the server does not support gets `NO [USEATTR]` (section 3) | +| METADATA ([RFC 5464 section 3.2](https://www.rfc-editor.org/rfc/rfc5464#section-3.2)) | Entry names with `//`, a trailing `/`, `*`, `%`, 8-bit or control characters, or a scope other than `/private` or `/shared`, values that are atoms or use bare CR or LF as line ends, empty entry or option lists, and GETMETADATA options after the mailbox name (errata 2785) | +| ACL ([RFC 4314 section 3](https://www.rfc-editor.org/rfc/rfc4314#section-3)) | Unknown or uppercase rights, empty identifiers, identifiers with control characters or invalid UTF-8 | +| CATENATE ([RFC 4469](https://www.rfc-editor.org/rfc/rfc4469)) | URLs that are not absolute-path references (`/INBOX/;UID=1`), including relative-path references like `;UID=1` that [RFC 5092 section 7.2](https://www.rfc-editor.org/rfc/rfc5092#section-7.2) forbids, and URLs of message parts that do not exist (`NO [BADURL ...]`) | +| UIDONLY ([RFC 9586](https://www.rfc-editor.org/rfc/rfc9586)) | Every command that takes message sequence numbers, and sequence sets in search criteria, get `BAD [UIDREQUIRED]` | +| UTF8=ACCEPT ([RFC 9755](https://www.rfc-editor.org/rfc/rfc9755)) | Invalid UTF-8 in quoted strings, `SEARCH CHARSET` after `ENABLE UTF8=ACCEPT`, mailbox names with control characters (in UTF-8, or encoded in modified UTF-7 like `&AA0-`), U+2028, U+2029, a leading BOM, unassigned code points or a name that is not in Unicode Normalization Form C. `NO` for `APPEND` of a message with an 8-bit header before `ENABLE UTF8=ACCEPT` (section 4) | +| BINARY ([RFC 3516](https://www.rfc-editor.org/rfc/rfc3516)) | `BINARY[]`, `BINARY` of multipart or message/rfc822 parts ([RFC 9051 section 6.4.5](https://www.rfc-editor.org/rfc/rfc9051#section-6.4.5) allows leaf body parts only), `HEADER`, `TEXT` or `MIME` sections, and a partial range on `BINARY.SIZE` | +| NOTIFY ([RFC 5465](https://www.rfc-editor.org/rfc/rfc5465)) | MessageNew without MessageExpunge or the other way round, FlagChange without both (section 5), mailbox events or two selected filters with `selected`/`selected-delayed` (section 6.1), fetch attributes outside the selected filters, empty event or mailbox lists, `NOTIFY SET` without event groups. Unknown events get `NO [BADEVENT (...)]` listing the supported ones (section 3.1) | +| IMAP4rev2 ([RFC 9051](https://www.rfc-editor.org/rfc/rfc9051)) | After `ENABLE IMAP4rev2`: `CHECK`, `LSUB`, the `RFC822`, `RFC822.HEADER` and `RFC822.TEXT` FETCH items, the `NEW`, `OLD` and `RECENT` SEARCH keys and the `RECENT` STATUS item (none are in the RFC 9051 grammar, Appendix E), numbers above 63 bits in LARGER and SMALLER, invalid UTF-8, and mailbox names that are not Net-Unicode (section 5.1). Before it: 8-bit quoted strings (Appendix A), and partial ranges, LARGER and SMALLER values above 32 bits | ## Response codes diff --git a/src/plugins/create-special-use.ts b/src/plugins/create-special-use.ts index 109fb58..d8101be 100644 --- a/src/plugins/create-special-use.ts +++ b/src/plugins/create-special-use.ts @@ -6,6 +6,10 @@ import type { Attribute, Callback, CommandHandler, IMAPConnection, IMAPResponse, * @help option "special-use" */ +// use-attr-ext = "\" atom (RFC 6154 section 6), ATOM-CHAR is any CHAR except atom-specials (RFC 3501 section 9) +// eslint-disable-next-line no-control-regex +const USE_ATTR = /^\\[^\x00-\x20\x7f-\xff(){%*"\\\]]+$/; + export default function createSpecialUsePlugin(server: IMAPServer) { // Register capability server.registerCapability('CREATE-SPECIAL-USE'); @@ -40,7 +44,10 @@ export default function createSpecialUsePlugin(server: IMAPServer) { if (specialUseList) { for (i = 0, len = specialUseList.length; i < len; i++) { - if (['ATOM', 'STRING', 'LITERAL'].indexOf(specialUseList[i].type) < 0) { + // RFC 6154 section 6: use-attr-ext = "\" atom, so every entry is an atom that starts with a backslash, + // not NIL, a string, a literal, a number or a list + const entry = specialUseList[i]; + if (!entry || Array.isArray(entry) || entry.type !== 'ATOM' || typeof entry.value !== 'string' || !USE_ATTR.test(entry.value)) { connection.send( { tag: parsed.tag, diff --git a/test/conformance.test.ts b/test/conformance.test.ts index 17a09f4..412486e 100644 --- a/test/conformance.test.ts +++ b/test/conformance.test.ts @@ -464,6 +464,33 @@ describe('Strict extended LIST', () => { defineCases(ctx, LIST_EXTENDED_CASES); }); +// CREATE-SPECIAL-USE, RFC 6154 section 6: create-param =/ "USE" SP "(" [use-attr *(SP use-attr)] ")", +// use-attr-ext = "\" atom; the create-params of RFC 4466 section 2.2 +const CREATE_SPECIAL_USE_CASES: Case[] = [ + ['CREATE USE (NIL)', 'auth', ['A1 CREATE W (USE (NIL))'], { A1: 'BAD' }], + ['CREATE USE with a quoted attribute', 'auth', ['A1 CREATE W (USE ("\\\\Sent"))'], { A1: 'BAD' }], + ['CREATE USE with a literal attribute', 'auth', ['A1 CREATE W (USE ({5}\r\n\\Sent))'], { A1: 'BAD' }], + ['CREATE USE with an attribute without a backslash', 'auth', ['A1 CREATE W (USE (Sent))'], { A1: 'BAD' }], + ['CREATE USE with a lone backslash', 'auth', ['A1 CREATE W (USE (\\))'], { A1: 'BAD' }], + ['CREATE USE with a nested list', 'auth', ['A1 CREATE W (USE ((\\Sent)))'], { A1: 'BAD' }], + ['CREATE USE with a number', 'auth', ['A1 CREATE W (USE (1))'], { A1: 'BAD' }], + ['CREATE USE without a list', 'auth', ['A1 CREATE W (USE \\Sent)'], { A1: 'BAD' }], + ['CREATE USE without a value', 'auth', ['A1 CREATE W (USE)'], { A1: 'BAD' }], + ['CREATE USE with an unknown parameter', 'auth', ['A1 CREATE W (USE (\\Sent) FOO)'], { A1: 'BAD' }], + // RFC 6154 section 3: an attribute the server does not support is NO, with the USEATTR response code + ['CREATE USE with an unsupported attribute', 'auth', ['A1 CREATE W (USE (\\Important))'], { A1: 'NO' }], + ['CREATE USE with an empty list', 'auth', ['A1 CREATE W (USE ())'], { A1: 'OK' }], + ['CREATE USE', 'auth', ['A1 CREATE W (USE (\\Sent \\Drafts))'], { A1: 'OK' }] +]; + +describe('Strict CREATE-SPECIAL-USE', () => { + const ctx = setupServer(() => ({ + plugins: ['SPECIAL-USE', 'CREATE-SPECIAL-USE'] + })); + + defineCases(ctx, CREATE_SPECIAL_USE_CASES); +}); + describe('Strict METADATA handling', () => { const ctx = setupServer(() => ({ plugins: ['METADATA'], From 636354387f58b94c82655cec567cbaf5562b648f Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:31:43 +0300 Subject: [PATCH 07/17] fix: ENVELOPE sends an obsolete source route as addr-adl For <@route.example:a@b.c> the route leaked into the mailbox name of the ENVELOPE address. RFC 9051 section 7.5.2 makes the second field the at-domain-list (the obs-route of RFC 5322 section 4.4) and the third the local-part, so the route now goes to addr-adl as "@route.example", the way Dovecot sends it (golden form in test/fixtures/mime/source-route.eml). SORT FROM/TO/CC use the local part too. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/docs/extensions/overview.md | 1 + docs/docs/reference/known-issues.md | 1 - src/envelope.ts | 31 +++++++++++++++++++++++++++-- test/fixtures/mime/source-route.eml | 9 +++++++++ test/mime-fidelity.test.ts | 6 ++++++ test/mime.test.ts | 22 ++++++++++++++++++++ 7 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/mime/source-route.eml diff --git a/README.md b/README.md index 603c1f1..85f9e25 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,7 @@ All RFC 3501 commands are supported. Some choices that the RFCs leave to the ser - CREATE `a/b` also creates `a` as a normal mailbox if it does not exist (RFC 3501 section 6.3.3, Dovecot creates a `\Noselect` level instead). An existing `\Noselect` level stays `\Noselect` - DELETE of a mailbox with children leaves a `\Noselect` level that keeps nothing but the children, CREATE of that name makes a new mailbox with a new UIDVALIDITY - A keyword stays in the FLAGS and PERMANENTFLAGS of a mailbox once a message in it had the keyword, also after that message is expunged (RFC 3501 section 7.2.6, like Dovecot). In a mailbox with `"allowPermanentFlags": false` STORE accepts exactly the flags PERMANENTFLAGS lists (`permanentFlags` and the flags its messages have or had) and ignores the others, APPEND and COPY leave them out (RFC 3501 section 7.1) +- An obsolete source route in an address (`<@route.example:a@b.c>`, RFC 5322 section 4.4) goes to the at-domain-list field of the ENVELOPE address (`(NIL "@route.example" "a" "b.c")`), the mailbox name is the local part only (RFC 9051 section 7.5.2), like Dovecot sends it ### Supported Plugins @@ -255,7 +256,6 @@ The XTOYBIRD commands map to the [control API](#control-api): # Known issues -- **addr-adl** (at-domain-list) values are not supported, NIL is always used - **anonymous namespaces** are not supported - **LIST** does not insert a hierarchy delimiter between a reference without one and the mailbox name (RFC 2683 section 3.4.9 recommends it), the two are concatenated as RFC 9051 section 6.3.9 describes, like Dovecot does - **CHARSET** values other than US-ASCII and UTF-8 are not supported diff --git a/docs/docs/extensions/overview.md b/docs/docs/extensions/overview.md index 458beea..27bcf52 100644 --- a/docs/docs/extensions/overview.md +++ b/docs/docs/extensions/overview.md @@ -165,6 +165,7 @@ Some choices that the RFCs leave to the server: - CREATE `a/b` also creates `a` as a normal mailbox if it does not exist (RFC 3501 section 6.3.3). An existing `\Noselect` level stays `\Noselect`. - DELETE of a mailbox with children leaves a `\Noselect` level that keeps nothing but the children. CREATE of that name makes a new mailbox with a new UIDVALIDITY. - A keyword stays in the FLAGS and PERMANENTFLAGS of a mailbox once a message in it had the keyword, also after that message is expunged (RFC 3501 section 7.2.6). In a mailbox with `"allowPermanentFlags": false` STORE accepts exactly the flags PERMANENTFLAGS lists (`permanentFlags` and the flags its messages have or had) and ignores the others, APPEND and COPY leave them out (RFC 3501 section 7.1). +- An obsolete source route in an address (`<@route.example:a@b.c>`, RFC 5322 section 4.4) goes to the at-domain-list field of the ENVELOPE address (`(NIL "@route.example" "a" "b.c")`), the mailbox name is the local part only (RFC 9051 section 7.5.2), like Dovecot sends it. - SEARCH, SORT and THREAD support the `US-ASCII` and `UTF-8` charsets. Any other charset gets `NO [BADCHARSET (US-ASCII UTF-8)]`. The [Mailboxes](./mailboxes.md#core-list-lsub-and-subscriptions) page shows these in transcripts, and [Strict by design](../guides/strict-by-design.md) lists what the core refuses. diff --git a/docs/docs/reference/known-issues.md b/docs/docs/reference/known-issues.md index 3c51e91..d6d01f1 100644 --- a/docs/docs/reference/known-issues.md +++ b/docs/docs/reference/known-issues.md @@ -12,7 +12,6 @@ ImapKit implements IMAP4rev1, IMAP4rev2 and more than 50 extensions, but not eve | Area | Issue | | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| addr-adl | Address lists in ENVELOPE always have NIL in the at-domain-list (source route) field, values from the message are not used. | | Anonymous namespaces | Not supported. Namespaces are personal, other users (`user`) or shared, see [Storage](../guides/storage.md#namespaces). | | LIST reference | A reference without a trailing hierarchy delimiter is concatenated with the mailbox name as it is: `LIST "Work" "Sub"` looks for `WorkSub`, not `Work/Sub`. [RFC 2683 section 3.4.9](https://www.rfc-editor.org/rfc/rfc2683#section-3.4.9) recommends inserting the delimiter, ImapKit concatenates the two as [RFC 9051 section 6.3.9](https://www.rfc-editor.org/rfc/rfc9051#section-6.3.9) describes, like Dovecot does. | | Search charsets | Only `US-ASCII` and `UTF-8` are supported. Other charsets in SEARCH, SORT and THREAD get `NO [BADCHARSET (US-ASCII UTF-8)]`. | diff --git a/src/envelope.ts b/src/envelope.ts index 78e7fb4..e433293 100644 --- a/src/envelope.ts +++ b/src/envelope.ts @@ -4,7 +4,32 @@ import type { ParsedAddress } from './addressparser.js'; import type { ParsedHeader } from './mimeparser.js'; /** An address of the ENVELOPE (RFC 3501 section 9): name, source route, mailbox and host, all NIL for a group end */ -export type EnvelopeAddress = [name: string | null, adl: null, mailbox: string | null, host: string | null]; +export type EnvelopeAddress = [name: string | null, adl: string | null, mailbox: string | null, host: string | null]; + +// RFC 5322 section 4.4: obs-angle-addr = [CFWS] "<" obs-route addr-spec ">" [CFWS], obs-route = obs-domain-list ":", +// obs-domain-list = *(CFWS / ",") "@" domain *("," [CFWS] ["@" domain]). A domain-literal can hold a colon +const OBS_ROUTE = /^[\s,]*(@(?:[^:[]|\[[^\]]*\])*):/; + +/** + * Splits an obsolete source route from an address: the route goes to addr-adl, the rest is the addr-spec + * (RFC 9051 section 7.5.2: the at-domain-list is the "source route and obs-route ABNF production from [RFC5322]", + * the mailbox name the "local-part ABNF production"). The route is sent like Dovecot does, "@a,@b" + * + * @param {String} address Address from the angle brackets + * @return {Array} [route or null, addr-spec] + */ +function splitRoute(address: string): [string | null, string] { + const match = address.match(OBS_ROUTE); + if (!match) { + return [null, address]; + } + const route = match[1] + .split(',') + .map(domain => domain.trim()) + .filter(domain => domain) + .join(','); + return [route, address.substr(match[0].length).trim()]; +} /** The fields of the ENVELOPE (RFC 3501 section 7.4.2), in order */ export type Envelope = [ @@ -83,13 +108,15 @@ function processAddress(arr: ParsedAddress | ParsedAddress[] | undefined, def?: return; } + const [route, addrSpec] = splitRoute(address); + address = addrSpec; const at = address.lastIndexOf('@'); const user = at >= 0 ? address.substr(0, at) : address; // RFC 3501 7.4.2 reserves a NIL host for group markers, so an address without a domain gets // the placeholder host that Dovecot uses for the same input const domain = (at >= 0 ? address.substr(at + 1) : '') || 'MISSING_DOMAIN'; - result.push([name, null, user || null, domain]); + result.push([name, route, user || null, domain]); }); // env-from = "(" 1*address ")", there is no SP between the addresses (RFC 3501 section 9) diff --git a/test/fixtures/mime/source-route.eml b/test/fixtures/mime/source-route.eml new file mode 100644 index 0000000..0144994 --- /dev/null +++ b/test/fixtures/mime/source-route.eml @@ -0,0 +1,9 @@ +Date: Thu, 08 Oct 2026 12:00:00 +0000 +From: <@route.example:a@b.c> +Sender: Sender <@one.example,@two.example:sender@example.com> +To: Bob <@r1.example, @r2.example:bob@d.e>, plain@x.y +Cc: "Group" <@[192.0.2.1]:carol@f.g> +Subject: Source routes +Message-ID: + +Obsolete source routes in angle addresses (RFC 5322 section 4.4). diff --git a/test/mime-fidelity.test.ts b/test/mime-fidelity.test.ts index 9bfd917..e137be4 100644 --- a/test/mime-fidelity.test.ts +++ b/test/mime-fidelity.test.ts @@ -112,6 +112,12 @@ const GOLDEN: Record> = { BODYSTRUCTURE: '(("TEXT" "PLAIN" NIL NIL NIL "7BIT" 10 0 NIL NIL NIL)("TEXT" "PLAIN" NIL NIL NIL "7BIT" 27 1 NIL NIL NIL) "MIXED" ("BOUNDARY" "nb") NIL NIL)' }, + // RFC 9051 section 7.5.2: the obs-route of RFC 5322 section 4.4 is the at-domain-list, the mailbox name is the local-part + 'test/fixtures/mime/source-route.eml': { + ENVELOPE: + '("Thu, 08 Oct 2026 12:00:00 +0000" "Source routes" ((NIL "@route.example" "a" "b.c")) (("Sender" "@one.example,@two.example" "sender" "example.com")) ((NIL "@route.example" "a" "b.c")) ' + + '(("Bob" "@r1.example,@r2.example" "bob" "d.e")(NIL NIL "plain" "x.y")) (("Group" "@[192.0.2.1]" "carol" "f.g")) NIL NIL "")' + }, 'test/fixtures/mime/eightbit-headers.eml': { BODYSTRUCTURE: '("TEXT" "PLAIN" ("CHARSET" "utf-8") NIL NIL "8BIT" 41 2 NIL NIL NIL)', ENVELOPE: diff --git a/test/mime.test.ts b/test/mime.test.ts index 22fba2f..e18d2be 100644 --- a/test/mime.test.ts +++ b/test/mime.test.ts @@ -179,6 +179,28 @@ describe('MIME parser', () => { ]); }); + it('puts an obsolete source route in addr-adl, not in the mailbox name', () => { + // RFC 9051 section 7.5.2: the second field is the "[SMTP] at-domain-list (source route and obs-route ABNF + // production from [RFC5322])", the third the "mailbox name (local-part ABNF production from [RFC5322])". + // RFC 5322 section 4.4: obs-domain-list = *(CFWS / ",") "@" domain *("," [CFWS] ["@" domain]) + const env = envelope( + mimeParser( + 'From: <@route.example:a@b.c>\r\n' + + 'To: Bob <@r1.example, @r2.example:bob@d.e>, Leading <,@one.example,,@two.example:lead@example.com>\r\n' + + 'Cc: <@[192.0.2.1]:carol@f.g>, "Not a route" \r\n\r\nbody' + ).parsedHeader + ); + assert.deepStrictEqual(env[2], [[null, '@route.example', 'a', 'b.c']]); + assert.deepStrictEqual(env[5], [ + ['Bob', '@r1.example,@r2.example', 'bob', 'd.e'], + ['Leading', '@one.example,@two.example', 'lead', 'example.com'] + ]); + assert.deepStrictEqual(env[6], [ + [null, '@[192.0.2.1]', 'carol', 'f.g'], + ['Not a route', null, 'x', 'y.z'] + ]); + }); + it('parses deeply nested groups in linear time', () => { const start = Date.now(); const result = addressparser('g:'.repeat(50000) + 'a@b;'); From 953e9652874fbd8bc9d983579fe1a7004b5c1c28 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:32:57 +0300 Subject: [PATCH 08/17] fix: deprecate the ignored xoauth2.sessionTimeout user option sessionTimeout came from hoodiecrow and was stored and defaulted to an hour but never used, access tokens do not expire. Enforcing it would make the default token of a long running server stop working after an hour, so it stays accepted and is marked deprecated in the types and docs, which now show how to test an expired token with control.updateUser(). Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/docs/control-api/users-and-sessions.md | 12 ++++----- docs/docs/guides/authentication.md | 2 +- docs/docs/reference/known-issues.md | 14 +++++----- docs/docs/reference/server-options.md | 2 +- src/control.ts | 11 +++++++- src/store-operations.ts | 5 +++- src/types.ts | 11 +++++++- test/xoauth2.test.ts | 29 +++++++++++++++++++++ 9 files changed, 69 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 85f9e25..b290714 100644 --- a/README.md +++ b/README.md @@ -405,7 +405,7 @@ Mailboxes are addressed by their storage name (modified UTF-7, the name a client | `createMailbox(path, { subscribed })`, `deleteMailbox(path)`, `renameMailbox(path, newPath)` | Like CREATE, DELETE and RENAME | | `resetUidValidity(path, { uidvalidity, uids, offset, seed })` | Gives the mailbox a new, greater UIDVALIDITY. `uids` is `keep` (default), `renumber` (1 to n), `shuffle` (1 to n in a random order, repeatable with `seed`, so an old UID points to another message) or `offset` (every UID moves above the old UIDNEXT, plus `offset`, so old UIDs find nothing). A UID must not change during a session (RFC 9051 section 2.3.1.1), so sessions that have the mailbox selected get `BYE`. Returns `{ uidvalidity, uidnext, uids: [{ uid, newUid }] }` | | `subscribe(path)`, `unsubscribe(path)` | Return true if the subscription changed | -| `listUsers()`, `addUser(name, { password, xoauth2 })`, `updateUser(name, {...})`, `deleteUser(name)` | `xoauth2` is `{ accessToken, sessionTimeout }`. Deleting a user disconnects its sessions, `{ disconnect: false }` keeps them. No credentials in `listUsers()` | +| `listUsers()`, `addUser(name, { password, xoauth2 })`, `updateUser(name, {...})`, `deleteUser(name)` | `xoauth2` is `{ accessToken, sessionTimeout }`, `sessionTimeout` is deprecated and ignored (access tokens do not expire). Deleting a user disconnects its sessions, `{ disconnect: false }` keeps them. No credentials in `listUsers()` | | `disconnect(session or { user }, { text, reset })` | Disconnects sessions with an untagged `BYE`, or resets the TCP connection | | `inject(session, data)` | Writes bytes to a session as they are, e.g. `* OK [ALERT] ...` between commands | | `reset()` | Restores the mailboxes and users of the server options and disconnects every session with `BYE`, so a long running server can be reused between tests. Script rules stay, with their `hits` counts | diff --git a/docs/docs/control-api/users-and-sessions.md b/docs/docs/control-api/users-and-sessions.md index 402c7e9..968e672 100644 --- a/docs/docs/control-api/users-and-sessions.md +++ b/docs/docs/control-api/users-and-sessions.md @@ -29,12 +29,12 @@ server.control.listUsers(); addUser(name: string, options: { password?: string; xoauth2?: { accessToken: string; sessionTimeout?: number } }): void ``` -| Parameter | Description | -| -------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| `name` | user name, a non-empty string | -| `options.password` | password for the LOGIN command and AUTHENTICATE PLAIN | -| `options.xoauth2.accessToken` | access token for XOAUTH2 and OAUTHBEARER | -| `options.xoauth2.sessionTimeout` | stored with the token in milliseconds, default 3600000. ImapKit does not expire tokens, so it has no effect on logins | +| Parameter | Description | +| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | user name, a non-empty string | +| `options.password` | password for the LOGIN command and AUTHENTICATE PLAIN | +| `options.xoauth2.accessToken` | access token for XOAUTH2 and OAUTHBEARER | +| `options.xoauth2.sessionTimeout` | deprecated and ignored, stored with the token in milliseconds, default 3600000. ImapKit does not expire tokens, replace the token with `updateUser` to test an expired one | **Errors:** `INVALID` for an empty name, a password that is not a string or an `xoauth2` without an `accessToken` string, `ALREADYEXISTS` for an existing user. diff --git a/docs/docs/guides/authentication.md b/docs/docs/guides/authentication.md index 8c23b94..9a0fc05 100644 --- a/docs/docs/guides/authentication.md +++ b/docs/docs/guides/authentication.md @@ -32,7 +32,7 @@ const server = imapkit({ `testuser` is not added when `users` is set, so here only `alice` and `bob` can log in, and only `alice` has an access token. User names are Unicode strings, in `users`, in SASL exchanges and in ACL identifiers. -The `xoauth2` object also takes a `sessionTimeout` (milliseconds, default one hour). It is kept with the user but has no effect on logins. +The `xoauth2` object also takes a `sessionTimeout` (milliseconds, default one hour). It is deprecated: ImapKit keeps it from hoodiecrow, stores and lists it, but never uses it, access tokens do not expire. To test how a client handles an expired token, replace the token with [`control.updateUser()`](#users-at-runtime): a login with the old token then fails like an expired one. ## LOGIN diff --git a/docs/docs/reference/known-issues.md b/docs/docs/reference/known-issues.md index d6d01f1..f60d398 100644 --- a/docs/docs/reference/known-issues.md +++ b/docs/docs/reference/known-issues.md @@ -51,12 +51,12 @@ S: B1 OK Completed ## Server -| Area | Limitation | -| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Plugins | Plugins are chosen when the server is built. They can not be loaded or unloaded while it runs. | -| Timeouts | There is no inactivity timeout, sessions stay open until the client or the test closes them. Use a [script rule](../faults/scripted-faults.md) to simulate an autologout. | -| `sessionTimeout` | The `xoauth2.sessionTimeout` value of a user is kept, but has no effect: access tokens never expire. | -| Command lines | Up to 1 MiB, a longer line is answered with BAD. | -| Literals | Up to 64 MiB after login (the `maxLiteralSize` option changes it) and 64 KiB before login. | +| Area | Limitation | +| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Plugins | Plugins are chosen when the server is built. They can not be loaded or unloaded while it runs. | +| Timeouts | There is no inactivity timeout, sessions stay open until the client or the test closes them. Use a [script rule](../faults/scripted-faults.md) to simulate an autologout. | +| `sessionTimeout` | The `xoauth2.sessionTimeout` value of a user is deprecated, it is kept but has no effect: access tokens never expire. Replace the token with `control.updateUser()` to test an expired one. | +| Command lines | Up to 1 MiB, a longer line is answered with BAD. | +| Literals | Up to 64 MiB after login (the `maxLiteralSize` option changes it) and 64 KiB before login. | For differences between ImapKit and Dovecot that are not ImapKit bugs, see [Comparing with Dovecot](../contributing/comparing-with-dovecot.md). diff --git a/docs/docs/reference/server-options.md b/docs/docs/reference/server-options.md index d827c56..ddb24d5 100644 --- a/docs/docs/reference/server-options.md +++ b/docs/docs/reference/server-options.md @@ -29,7 +29,7 @@ The `imapkit` command passes its `--config` JSON file to the same factory, so ev | ------------------ | -------------------------------------------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `storage` | `Record` | `{ INBOX: {}, '': {} }` | The mailbox tree, keyed by namespace prefix. Checked with `validateStorage()` when the server is built, a typo or wrong type throws with the path of the problem. See [Storage](../guides/storage.md). | | `plugins` | `(string \| Plugin)[]`, or a single `string` or `Plugin` | none | Plugins to load: built-in names (case insensitive, capability spellings like `LITERAL+` work too) or plugin functions. An unknown name throws. A single string is one name, not a comma separated list. See [Extensions](../extensions/overview.md) and [Custom Plugins](./custom-plugins.md). | -| `users` | `Record` | `testuser` with password `testpass` and XOAUTH2 token `testtoken` | User accounts. Each user is `{ password, xoauth2: { accessToken, sessionTimeout } }`. Setting this option replaces the default user. See [Authentication](../guides/authentication.md). | +| `users` | `Record` | `testuser` with password `testpass` and XOAUTH2 token `testtoken` | User accounts. Each user is `{ password, xoauth2: { accessToken, sessionTimeout } }`, `sessionTimeout` is deprecated and ignored (access tokens do not expire). Setting this option replaces the default user. See [Authentication](../guides/authentication.md). | | `secureConnection` | `boolean` | `false` | Implicit TLS: the server accepts only TLS connections (port 993 style). | | `credentials` | `{ key: string \| Buffer, cert: string \| Buffer }` | the bundled self-signed certificate for `localhost` | TLS key and certificate for `secureConnection`, the STARTTLS plugin and the SMTP listener. | | `systemFlags` | `string[]` | `['\\Answered', '\\Flagged', '\\Draft', '\\Deleted', '\\Seen']` | The system flags that can be stored. A STORE or APPEND with another `\`-flag is refused. Also the default `permanentFlags` of mailboxes that do not set their own. | diff --git a/src/control.ts b/src/control.ts index 44641d4..a9bc243 100644 --- a/src/control.ts +++ b/src/control.ts @@ -88,7 +88,16 @@ interface NewMessage { interface UserOptions { password?: string | undefined; - xoauth2?: { accessToken?: string | undefined; sessionTimeout?: number | undefined } | undefined; + xoauth2?: + | { + accessToken?: string | undefined; + /** + * @deprecated kept from hoodiecrow and ignored: access tokens never expire. Change the token with + * `control.updateUser()` to test a client against an expired one + */ + sessionTimeout?: number | undefined; + } + | undefined; } /** What a REST route handler gets, see src/rest.ts */ diff --git a/src/store-operations.ts b/src/store-operations.ts index b213101..0980bf6 100644 --- a/src/store-operations.ts +++ b/src/store-operations.ts @@ -11,7 +11,10 @@ import { seededRandom } from './random.js'; import { MAX_NUMBER } from './numbers.js'; import type { IMAPConnection, IMAPServer, Mailbox, Message } from './types.js'; -/** the XOAUTH2 session timeout of the default user, and of a user the control API adds without one */ +/** + * the deprecated XOAUTH2 session timeout of the default user, and of a user the control API adds without one. It is + * stored and listed but never used, access tokens do not expire + */ const DEFAULT_SESSION_TIMEOUT = 3600 * 1000; /** An error of a failed store operation or control API call, `code` is a RFC 5530 response code or INVALID */ diff --git a/src/types.ts b/src/types.ts index 4bfd74f..2d7e7db 100644 --- a/src/types.ts +++ b/src/types.ts @@ -232,7 +232,16 @@ export interface MailboxStatus { export interface UserData { password?: string | undefined; - xoauth2?: { accessToken?: string | undefined; sessionTimeout?: number | undefined } | undefined; + xoauth2?: + | { + accessToken?: string | undefined; + /** + * @deprecated kept from hoodiecrow and ignored: access tokens never expire. Change the token with + * `control.updateUser()` to test a client against an expired one + */ + sessionTimeout?: number | undefined; + } + | undefined; [key: string]: any; } diff --git a/test/xoauth2.test.ts b/test/xoauth2.test.ts index 742384a..cc5f92d 100644 --- a/test/xoauth2.test.ts +++ b/test/xoauth2.test.ts @@ -94,3 +94,32 @@ describe('XOAUTH2 edge cases', () => { }); }); }); + +// xoauth2.sessionTimeout is deprecated and ignored: access tokens do not expire, a test changes the token with +// control.updateUser() to see how a client handles an expired one +describe('XOAUTH2 sessionTimeout', () => { + const ctx = setupServer(() => ({ + plugins: ['SASL-IR', 'XOAUTH2'], + users: { alice: { xoauth2: { accessToken: 'alice-token', sessionTimeout: 1 } } } + })); + const auth = (token: string) => 'AUTHENTICATE XOAUTH2 ' + Buffer.from(['user=alice', 'auth=Bearer ' + token, '', ''].join('\x01')).toString('base64'); + + it('does not expire the access token', (t, done) => { + setTimeout(() => { + ctx.run(['A1 ' + auth('alice-token'), 'ZZ LOGOUT'], resp => { + assert.match(resp.toString(), /^A1 OK/m); + done(); + }); + }, 20); + }); + + it('a replaced token fails like an expired one', (t, done) => { + ctx.server.control.updateUser('alice', { xoauth2: { accessToken: 'fresh-token' } }); + ctx.run(['A1 ' + auth('alice-token'), '', 'A2 ' + auth('fresh-token'), 'ZZ LOGOUT'], resp => { + resp = resp.toString(); + assert.match(resp, /^A1 NO \[AUTHENTICATIONFAILED\]/m); + assert.match(resp, /^A2 OK/m); + done(); + }); + }); +}); From 0747678d007dbee1a0d247041d24d149dc8a72b1 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:33:05 +0300 Subject: [PATCH 09/17] fix: AUTHENTICATE checks the connection state before the mechanism After login, AUTHENTICATE with an unknown mechanism got NO while a known one got BAD. AUTHENTICATE is only valid in the not authenticated state (RFC 3501 section 6.2), so both are now BAD with the same text. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/docs/extensions/overview.md | 2 +- docs/docs/guides/authentication.md | 15 ++++++++++++++- docs/docs/guides/strict-by-design.md | 2 +- src/server.ts | 7 +++++++ test/conformance.test.ts | 11 +++++++++++ 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4e02e32..f4705c6 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ ImapKit is extendable: any command can be overridden and plugins can be added (s ImapKit is meant for developing standards compliant IMAP clients, so it follows the RFCs strictly instead of accepting whatever clients send. Most production servers are lenient, which hides client bugs until the client meets a stricter server. ImapKit answers these with `BAD` (or `NO` where the RFC requires it): -- commands sent in the wrong state (RFC 3501 section 3), for example `FETCH` before `SELECT` or `LOGIN` after login +- commands sent in the wrong state (RFC 3501 section 3), for example `FETCH` before `SELECT` or `LOGIN` and `AUTHENTICATE` (with any mechanism, known or not) after login - arguments to commands that take none (`NOOP x`, `CLOSE x`), missing or extra arguments, and values that break the RFC 3501 grammar - command lines that end with a bare LF instead of CRLF - literal data sent before the server's `+` continuation request (RFC 3501 section 4.3); `{n+}` is only accepted when LITERAL+ or LITERAL- is enabled, and with LITERAL- only up to 4096 octets, a larger one is answered with `BAD [TOOBIG]` (RFC 7888 section 5) diff --git a/docs/docs/extensions/overview.md b/docs/docs/extensions/overview.md index e04f375..4d30640 100644 --- a/docs/docs/extensions/overview.md +++ b/docs/docs/extensions/overview.md @@ -157,7 +157,7 @@ The Plugin column shows the file name spelling. The capability spellings listed ## What core IMAP4rev1 supports -Without any plugin, ImapKit supports every RFC 3501 command: `CAPABILITY`, `NOOP`, `LOGOUT`, `LOGIN`, `AUTHENTICATE` (no mechanism is built in, so it answers `NO Unsupported authentication mechanism` until an AUTH plugin is loaded), `SELECT`, `EXAMINE`, `CREATE`, `DELETE`, `RENAME`, `SUBSCRIBE`, `UNSUBSCRIBE`, `LIST`, `LSUB`, `STATUS`, `APPEND`, `CHECK`, `CLOSE`, `EXPUNGE`, `SEARCH`, `FETCH`, `STORE`, `COPY` and the `UID` variants of `COPY`, `FETCH`, `STORE` and `SEARCH`. `STARTTLS` is a plugin. +Without any plugin, ImapKit supports every RFC 3501 command: `CAPABILITY`, `NOOP`, `LOGOUT`, `LOGIN`, `AUTHENTICATE` (no mechanism is built in, so it answers `NO Unsupported authentication mechanism` until an AUTH plugin is loaded, and `BAD` after login), `SELECT`, `EXAMINE`, `CREATE`, `DELETE`, `RENAME`, `SUBSCRIBE`, `UNSUBSCRIBE`, `LIST`, `LSUB`, `STATUS`, `APPEND`, `CHECK`, `CLOSE`, `EXPUNGE`, `SEARCH`, `FETCH`, `STORE`, `COPY` and the `UID` variants of `COPY`, `FETCH`, `STORE` and `SEARCH`. `STARTTLS` is a plugin. Some choices that the RFCs leave to the server: diff --git a/docs/docs/guides/authentication.md b/docs/docs/guides/authentication.md index 8c23b94..f8fd802 100644 --- a/docs/docs/guides/authentication.md +++ b/docs/docs/guides/authentication.md @@ -49,7 +49,20 @@ LOGIN takes exactly two strings (atoms, quoted strings or literals). 8-bit user ## Mechanisms -AUTHENTICATE mechanisms come from plugins. AUTHENTICATE with a mechanism that no loaded plugin provides is answered with NO. +AUTHENTICATE mechanisms come from plugins. AUTHENTICATE with a mechanism that no loaded plugin provides is answered with NO ([RFC 3501 section 6.2.2](https://www.rfc-editor.org/rfc/rfc3501#section-6.2.2)). After login every AUTHENTICATE is BAD, whatever the mechanism, as the command is only valid in the Not Authenticated state ([RFC 3501 section 6.2](https://www.rfc-editor.org/rfc/rfc3501#section-6.2)). With AUTH-PLAIN loaded: + +```text +C: A1 AUTHENTICATE FOO +S: A1 NO Unsupported authentication mechanism +C: A2 LOGIN testuser testpass +S: A2 OK User logged in +C: A3 AUTHENTICATE FOO +S: A3 BAD AUTHENTICATE FOO is not allowed in the Authenticated state +C: A4 AUTHENTICATE PLAIN +S: A4 BAD AUTHENTICATE PLAIN is not allowed in the Authenticated state +``` + +The mechanism plugins: | Plugin | Capability | Notes | | ----------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/docs/docs/guides/strict-by-design.md b/docs/docs/guides/strict-by-design.md index ad7b9eb..56fb83e 100644 --- a/docs/docs/guides/strict-by-design.md +++ b/docs/docs/guides/strict-by-design.md @@ -65,7 +65,7 @@ ImapKit answers these with `BAD`, or `NO` where noted. The list matches the READ | Rule | Reference | | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -| Commands sent in the wrong state, for example `FETCH` before `SELECT` or `LOGIN` after login | [RFC 3501 section 3](https://www.rfc-editor.org/rfc/rfc3501#section-3) | +| Commands sent in the wrong state, for example `FETCH` before `SELECT` or `LOGIN` and `AUTHENTICATE` (with any mechanism) after login | [RFC 3501 section 3](https://www.rfc-editor.org/rfc/rfc3501#section-3) | | Arguments to commands that take none (`NOOP x`, `CLOSE x`), missing or extra arguments, and values that break the grammar | [RFC 3501 section 9](https://www.rfc-editor.org/rfc/rfc3501#section-9) | | Invalid sequence sets (`0`, `abc`, `5:`, `1:2:3`) | RFC 3501 section 9 | | Message sequence numbers greater than the number of messages in FETCH, STORE, COPY and MOVE, also `*` in an empty mailbox. UID sets and SEARCH keys can point past the end | RFC 3501 section 9, seq-number | diff --git a/src/server.ts b/src/server.ts index ed8e839..bf4b613 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3380,6 +3380,13 @@ class IMAPConnection { } this.processQueue(); } else if (/^AUTHENTICATE /i.test(parsed.command)) { + // AUTHENTICATE is only valid in the not authenticated state (RFC 3501 section 6.2), which is checked + // first, as for a supported mechanism (processQueue), so the answer does not depend on the mechanism + const states = this.server.getCommandOptions(parsed.command).states; + if (states && states.indexOf(this.state) < 0) { + this.sendStatus(parsed, data, 'BAD', stateError(parsed.command.toUpperCase(), this.state)); + return; + } // an unsupported mechanism is a NO, not a syntax error (RFC 3501 section 6.2.2) this.send( { diff --git a/test/conformance.test.ts b/test/conformance.test.ts index 412486e..315a58b 100644 --- a/test/conformance.test.ts +++ b/test/conformance.test.ts @@ -826,6 +826,17 @@ describe('Strict SASL handling', () => { ); // RFC 3501 section 6.2.2: an unsupported mechanism is NO it('AUTHENTICATE with an unknown mechanism', run(['A1 AUTHENTICATE FOO'], { A1: 'NO' })); + // RFC 3501 section 6.2: AUTHENTICATE is only valid in the not authenticated state, "Once authenticated + // (including as anonymous), it is not possible to re-enter not authenticated state". The state is checked + // before the mechanism, so a known and an unknown mechanism get the same BAD + it( + 'AUTHENTICATE with an unknown mechanism after login', + run(['A1 LOGIN testuser testpass', 'A2 AUTHENTICATE FOO', 'A3 AUTHENTICATE PLAIN', 'A4 SELECT INBOX', 'A5 AUTHENTICATE FOO'], { + A2: 'BAD', + A3: 'BAD', + A5: 'BAD' + }) + ); // RFC 2177 section 3: IDLE is ended by "DONE" only it('IDLE ended by something else than DONE', run(['A1 LOGIN testuser testpass', 'A2 IDLE', 'NOOP'], { A2: 'BAD' })); }); From e25c9a8971b4566ede897dfef67a8199bb08ed5a Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:34:52 +0300 Subject: [PATCH 10/17] fix: METADATA and METADATA-SERVER load ENABLE RFC 5464 section 4.1: a server that sends unsolicited METADATA responses MUST support the ENABLE command, and sends them only after ENABLE METADATA (or METADATA-SERVER). Without the ENABLE plugin a client could not turn them on, so both plugins now list ENABLE in plugin.requires, like UTF8=ACCEPT and UIDONLY. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++-- docs/docs/extensions/metadata-and-quota.md | 6 +++--- docs/docs/extensions/overview.md | 2 +- src/plugins/metadata-server.ts | 5 ++++- src/plugins/metadata.ts | 4 ++++ test/metadata.test.ts | 4 ++-- test/plugins.test.ts | 10 ++++++++++ 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index b290714..deacfc2 100644 --- a/README.md +++ b/README.md @@ -163,9 +163,9 @@ An unknown plugin name throws an error, and a plugin listed more than once is lo - **LITERALPLUS** Enables LITERAL+ [RFC7888] capability. Can not be loaded together with LITERALMINUS, but replaces the LITERAL- that IMAP4rev2 loads - **LOGINDISABLED** Disables LOGIN support for unencrypted connections - **MESSAGELIMIT** Adds MESSAGELIMIT [RFC9738] capability, advertised as `MESSAGELIMIT=`, where the server option `messageLimit` sets n (default 1000, any positive number is accepted so that small test mailboxes can hit it). FETCH, STORE, SEARCH, MOVE, UID EXPUNGE and their UID variants only work on the n messages with the highest UIDs (UID EXPUNGE counts the `\Deleted` ones) and add `[MESSAGELIMIT n uid]` with the lowest processed UID to the tagged OK, or send it in an untagged `NO` when the tagged OK already has a response code (like `HIGHESTMODSEQ` or `MODIFIED`). SEARCH counts the searched messages, which its top level sequence set, `UID`, `UIDAFTER` and `UIDBEFORE` keys narrow down. COPY, APPEND (MULTIAPPEND), SORT and THREAD of more messages, and a FETCH `PARTIAL` range (PARTIAL plugin) of more messages, fail with `NO [MESSAGELIMIT ...]`. EXPUNGE, CLOSE and STATUS are not limited. Adds the `UIDAFTER` and `UIDBEFORE` search keys. Can not be loaded together with SAVELIMIT -- **METADATA** Adds METADATA [RFC5464] capability (GETMETADATA and SETMETADATA) for server and mailbox annotations. Values can be binary: SETMETADATA takes a literal8 (`~{n}`), and values with NUL are sent back as a literal8. Initial mailbox entries come from a `metadata` object on the mailbox in storage (`"INBOX": { "metadata": { "/private/comment": "My comment" } }`), server entries from the `metadata` option. Server options `metadataMaxSize` (largest value in octets, default 65536), `metadataMaxEntries` (entries per mailbox and for the server, default 100) and `metadataPrivate: false` (refuse `/private` entries with `[METADATA NOPRIVATE]`) let you test the client's error handling. `/shared/admin` on the server is read-only. Annotations move with RENAME (renaming INBOX copies them), DELETE removes them. After `ENABLE METADATA` (needs the ENABLE plugin), changes made by other sessions are announced with unsolicited `METADATA` responses. With SPECIAL-USE loaded, the read-only `/private/specialuse` entry shows the special-use attributes of a mailbox (RFC 6154 section 4) +- **METADATA** Adds METADATA [RFC5464] capability (GETMETADATA and SETMETADATA) for server and mailbox annotations, and loads ENABLE, which RFC 5464 section 4.1 requires for unsolicited METADATA responses. Values can be binary: SETMETADATA takes a literal8 (`~{n}`), and values with NUL are sent back as a literal8. Initial mailbox entries come from a `metadata` object on the mailbox in storage (`"INBOX": { "metadata": { "/private/comment": "My comment" } }`), server entries from the `metadata` option. Server options `metadataMaxSize` (largest value in octets, default 65536), `metadataMaxEntries` (entries per mailbox and for the server, default 100) and `metadataPrivate: false` (refuse `/private` entries with `[METADATA NOPRIVATE]`) let you test the client's error handling. `/shared/admin` on the server is read-only. Annotations move with RENAME (renaming INBOX copies them), DELETE removes them. After `ENABLE METADATA` (needs the ENABLE plugin), changes made by other sessions are announced with unsolicited `METADATA` responses. With SPECIAL-USE loaded, the read-only `/private/specialuse` entry shows the special-use attributes of a mailbox (RFC 6154 section 4) - **MULTISEARCH** Adds MULTISEARCH [RFC7377] capability, also loads ESEARCH: the ESEARCH command, also in the authenticated state. `ESEARCH IN (mailboxes "a" subtree "b" subtree-one "c" personal subscribed inboxes selected) RETURN (...) criteria` sends one ESEARCH response with UIDs and the `TAG`, `MAILBOX` and `UIDVALIDITY` correlators for every mailbox with matches. Mailboxes that do not exist or are `\Noselect` are skipped (with ACL also those without the `r` right, and without `l` unless named under `mailboxes` or as a subtree root), a mailbox named twice is searched once, and `inboxes` is INBOX. `SAVE` is only allowed when the selected mailbox is the only one searched, `UPDATE` (with CONTEXT=SEARCH) only applies to the selected mailbox -- **METADATA-SERVER** Same as METADATA, but only for server annotations (mailbox name `""`) +- **METADATA-SERVER** Same as METADATA (also loads ENABLE), but only for server annotations (mailbox name `""`) - **MOVE** Adds MOVE [RFC6851] capability (MOVE and UID MOVE commands) - **MULTIAPPEND** Adds MULTIAPPEND [RFC3502] capability. APPEND takes several messages and appends all or none of them. With UIDPLUS, APPENDUID lists the UIDs as a UID set - **NAMESPACE** Adds NAMESPACE [RFC2342] capability diff --git a/docs/docs/extensions/metadata-and-quota.md b/docs/docs/extensions/metadata-and-quota.md index 1442f52..201e0b5 100644 --- a/docs/docs/extensions/metadata-and-quota.md +++ b/docs/docs/extensions/metadata-and-quota.md @@ -16,7 +16,7 @@ APPENDLIMIT, the other size limit a server can announce, is on the [Messages](./ ## METADATA -Adds `GETMETADATA` and `SETMETADATA` for server annotations (mailbox name `""`) and mailbox annotations. +Adds `GETMETADATA` and `SETMETADATA` for server annotations (mailbox name `""`) and mailbox annotations. Loads ENABLE, which [RFC 5464 section 4.1](https://www.rfc-editor.org/rfc/rfc5464#section-4.1) requires for the unsolicited METADATA responses. | Option | Default | Effect | | -------------------- | ------- | -------------------------------------------------------------------------- | @@ -71,7 +71,7 @@ What is implemented: - Values can be binary: SETMETADATA takes a literal8 (`~{n}`), and values with NUL octets are sent back as a literal8. - `/shared/admin` on the server is read-only (`NO [CANNOT]`). - RENAME moves the annotations of a mailbox, renaming INBOX copies them (RFC 5464 section 4.1). DELETE removes them. -- After `ENABLE METADATA` (needs the ENABLE plugin), changes made by other sessions are announced with unsolicited `METADATA` responses. +- After `ENABLE METADATA`, changes made by other sessions are announced with unsolicited `METADATA` responses. - With SPECIAL-USE loaded, the read-only `/private/specialuse` entry shows the special-use attributes of a mailbox (RFC 6154 section 4). - With NOTIFY loaded, the `MailboxMetadataChange` and `ServerMetadataChange` events are available. - With ACL loaded, mailbox annotations need the rights listed on [Access control](./access-control.md#with-other-plugins). @@ -86,7 +86,7 @@ S: A4 BAD GETMETADATA expects options, a mailbox name and entries ## METADATA-SERVER -The same as METADATA, but only for server annotations (mailbox name `""`). A mailbox annotation command gets `NO`. The same options apply, and `ENABLE METADATA-SERVER` turns on unsolicited responses. When METADATA is loaded too, only `METADATA` is advertised. +The same as METADATA (it loads ENABLE too), but only for server annotations (mailbox name `""`). A mailbox annotation command gets `NO`. The same options apply, and `ENABLE METADATA-SERVER` turns on unsolicited responses. When METADATA is loaded too, only `METADATA` is advertised. ```javascript const server = imapkit({ plugins: ['METADATA-SERVER'], metadataPrivate: false }); diff --git a/docs/docs/extensions/overview.md b/docs/docs/extensions/overview.md index 27bcf52..2e038d7 100644 --- a/docs/docs/extensions/overview.md +++ b/docs/docs/extensions/overview.md @@ -51,7 +51,7 @@ Some extensions are defined on top of others, so their plugins load what they ne | ------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `IMAP4rev2` | ENABLE, NAMESPACE, UNSELECT, UIDPLUS, ESEARCH, SEARCHRES, IDLE, SASL-IR, LIST-EXTENDED, LIST-STATUS, MOVE, BINARY, SPECIAL-USE, STATUS=SIZE, AUTH=PLAIN, and LITERAL- unless LITERAL+ is loaded | | `QRESYNC` | ENABLE, CONDSTORE | -| `UIDONLY`, `UTF8=ACCEPT` | ENABLE | +| `UIDONLY`, `UTF8=ACCEPT`, `METADATA`, `METADATA-SERVER` | ENABLE | | `SEARCHRES`, `PARTIAL`, `MULTISEARCH`, `CONTEXT=SEARCH` | ESEARCH | | `ESORT` | SORT, ESEARCH | | `CONTEXT=SORT` | ESORT, SORT, ESEARCH, CONTEXT=SEARCH | diff --git a/src/plugins/metadata-server.ts b/src/plugins/metadata-server.ts index fc491ba..274ddbf 100644 --- a/src/plugins/metadata-server.ts +++ b/src/plugins/metadata-server.ts @@ -3,10 +3,13 @@ import type { IMAPServer } from '../types.js'; /** * @help Adds METADATA-SERVER [RFC5464] capability, like METADATA but - * @help only for server annotations (mailbox name ""). With METADATA + * @help only for server annotations (mailbox name ""), loads ENABLE. With METADATA * @help also loaded, only METADATA is advertised */ export default function metadataServerPlugin(server: IMAPServer) { setup(server, false); } + +// RFC 5464 section 4.1: a server that sends unsolicited METADATA responses "MUST support the ENABLE command" +metadataServerPlugin.requires = ['ENABLE']; diff --git a/src/plugins/metadata.ts b/src/plugins/metadata.ts index be65c9f..04a08ec 100644 --- a/src/plugins/metadata.ts +++ b/src/plugins/metadata.ts @@ -20,6 +20,7 @@ interface MetadataHolder { * @help default 65536), "metadataMaxEntries" (per mailbox and for the * @help server, default 100). "metadataPrivate": false turns /private * @help entries off. ENABLE METADATA turns on unsolicited METADATA responses + * @help (loads the ENABLE plugin, RFC 5464 section 4.1) */ // Default limits, RFC 5464 section 4.1 requires at least 1024 octets and 10 entries @@ -573,4 +574,7 @@ export default function metadataPlugin(server: IMAPServer) { setup(server, true); } +// RFC 5464 section 4.1: a server that sends unsolicited METADATA responses "MUST support the ENABLE command" +metadataPlugin.requires = ['ENABLE']; + export { setup }; diff --git a/test/metadata.test.ts b/test/metadata.test.ts index 41641c2..647585e 100644 --- a/test/metadata.test.ts +++ b/test/metadata.test.ts @@ -477,8 +477,8 @@ describe('METADATA-SERVER', () => { ], resp => { const capability = resp.match(/^\* CAPABILITY .*$/m)![0]; - assert.match(capability, / METADATA-SERVER(?: |\r)/); - assert.doesNotMatch(capability, / METADATA(?: |\r)/); + assert.match(capability, / METADATA-SERVER(?: |$)/); + assert.doesNotMatch(capability, / METADATA(?: |$)/); assert.match(resp, /^\* METADATA "" \(\/shared\/comment "x"\)\r$/m); assert.match(resp, /^A4 NO /m); assert.match(resp, /^A5 NO /m); diff --git a/test/plugins.test.ts b/test/plugins.test.ts index afa98b7..f490447 100644 --- a/test/plugins.test.ts +++ b/test/plugins.test.ts @@ -29,6 +29,16 @@ describe('Plugin loading', () => { assert.strictEqual(server.literalPlus, true); }); + it('loads ENABLE with METADATA and METADATA-SERVER', () => { + // RFC 5464 section 4.1: a server that sends unsolicited METADATA responses "MUST support the ENABLE + // command", they go only to sessions that used ENABLE METADATA (or METADATA-SERVER) + for (const plugin of ['METADATA', 'METADATA-SERVER']) { + const server = imapkit({ plugins: [plugin] }); + assert.ok(server.capabilities.ENABLE, plugin); + assert.ok(server.capabilities[plugin], plugin); + } + }); + it('loads repeated plugins only once', () => { let calls = 0; const custom = () => { From 04babcde6a073ed7f703c16767f1f0efb332db3c Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:35:43 +0300 Subject: [PATCH 11/17] fix: refuse storage internal dates that are not RFC 3501 date-time values A storage message with an internaldate like "Thu, 1 Jan 2026 10:00:00 +0000" was accepted, FETCH then sent an invalid INTERNALDATE and SORT ARRIVAL fell back. validateStorage now requires a date-time string of a real date and time (RFC 3501 section 9) or a valid Date for internaldate and SAVEDATE (RFC 8514 section 4.2), so the server fails when it is built. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/docs/faults/repeatable-tests.md | 5 +++ docs/docs/guides/storage.md | 16 ++++----- src/dates.ts | 25 +++++++++++++- src/server.ts | 21 ++---------- src/storage-schema.ts | 22 +++++++++---- test/savedate.test.ts | 5 ++- test/search.test.ts | 5 +-- test/storage-schema.test.ts | 49 ++++++++++++++++++++++++++++ 9 files changed, 112 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index f4705c6..ec9e2f7 100644 --- a/README.md +++ b/README.md @@ -594,7 +594,7 @@ const server = imapkit({ plugins: ['IDLE', 'MOVE'], quirks: ['james-fetchgroup', - `now`: the time the server uses for the dates it sets itself, the INTERNALDATE of a message without one and SAVEDATE: a Date, a timestamp, or a function that returns one. The dates are formatted in the time zone of the process. - `resetUidValidity(path, { uids: 'shuffle', seed })` of the control API. -The `storage` option is checked when the server is built: a key that looks like a typo of a known one (`message` for `messages`, `uidValidity`) or a wrong type fails with the path of the problem, e.g. `Invalid storage at "INBOX".messages[2]: unknown key "flag", did you mean "flags"?`. Plugins keep their own data on mailboxes and messages, so other keys are allowed. The package exports the check as `validateStorage(storage)` and the shape as a JSON Schema, `storageSchema`, for editors and fixture tooling. `server.control.snapshot()` returns the same shape. +The `storage` option is checked when the server is built: a key that looks like a typo of a known one (`message` for `messages`, `uidValidity`) or a wrong type fails with the path of the problem, e.g. `Invalid storage at "INBOX".messages[2]: unknown key "flag", did you mean "flags"?`. A message `internaldate` (or `SAVEDATE`) must be an RFC 3501 date-time string like `"14-Sep-2013 21:22:28 -0300"` or a valid Date, a Date header value like `"Thu, 1 Jan 2026 10:00:00 +0000"` is refused. Plugins keep their own data on mailboxes and messages, so other keys are allowed. The package exports the check as `validateStorage(storage)` and the shape as a JSON Schema, `storageSchema`, for editors and fixture tooling. `server.control.snapshot()` returns the same shape. ## Creating custom plugins diff --git a/docs/docs/faults/repeatable-tests.md b/docs/docs/faults/repeatable-tests.md index 8b0d025..02f5320 100644 --- a/docs/docs/faults/repeatable-tests.md +++ b/docs/docs/faults/repeatable-tests.md @@ -185,8 +185,13 @@ imapkit({ storage: { INBOX: { uidValidity: 5 } } }); imapkit({ storage: { INBOX: { uidnext: 0 } } }); // Error: Invalid storage at "INBOX".uidnext: must be an integer from 1 to 4294967295 + +imapkit({ storage: { INBOX: { messages: [{ raw: 'Subject: hi\r\n\r\nHello\r\n', internaldate: 'Thu, 1 Jan 2026 10:00:00 +0000' }] } } }); +// Error: Invalid storage at "INBOX".messages[0].internaldate: must be a date-time string like "14-Sep-2013 21:22:28 -0300" or a Date, not "Thu, 1 Jan 2026 10:00:00 +0000" ``` +`internaldate` (and `SAVEDATE`) must be an RFC 3501 date-time string of a real date and time, or a valid `Date`. A Date header style value like the one above is refused, as FETCH would send it as an invalid INTERNALDATE and SORT ARRIVAL could not read it. + A key counts as a typo when it is a known key in another case, or one edit away from a known key (for keys longer than 3 characters). Plugins keep their own data on mailboxes and messages (`acl`, `metadata`, `MODSEQ` ...), so other keys are allowed. The package exports the check and the shape, so fixtures can be checked in a unit test of their own, or in an editor: diff --git a/docs/docs/guides/storage.md b/docs/docs/guides/storage.md index 5749ccb..c4eb0e1 100644 --- a/docs/docs/guides/storage.md +++ b/docs/docs/guides/storage.md @@ -214,13 +214,13 @@ Flags of messages in the storage become flags of the mailbox: they are added to A message is either a string with the full message source, or an object: -| Key | Default | Description | -| -------------- | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| `raw` | `""` | The message source. A string, or a `Buffer` / `Uint8Array` in JavaScript. | -| `uid` | Assigned | The UID, an integer from 1 to 4294967295. Two messages with the same UID in one mailbox throw `Duplicate UID in mailbox `. | -| `flags` | `[]` | A flag or a list of flags. `\Recent` here is turned into `recent: true`. | -| `internaldate` | The current time | An RFC 3501 date-time string such as `"14-Sep-2013 21:22:28 -0300"`, or a `Date`. The month name can be in any case, it is sent as `Sep`. | -| `recent` | `false` | `true` makes the message `\Recent` for the first session that selects the mailbox, see [`\Recent`](#recent). | +| Key | Default | Description | +| -------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `raw` | `""` | The message source. A string, or a `Buffer` / `Uint8Array` in JavaScript. | +| `uid` | Assigned | The UID, an integer from 1 to 4294967295. Two messages with the same UID in one mailbox throw `Duplicate UID in mailbox `. | +| `flags` | `[]` | A flag or a list of flags. `\Recent` here is turned into `recent: true`. | +| `internaldate` | The current time | An RFC 3501 date-time string such as `"14-Sep-2013 21:22:28 -0300"`, or a `Date`. The month name can be in any case, it is sent as `Sep`. Other forms, such as a Date header value (`"Thu, 1 Jan 2026 10:00:00 +0000"`), impossible dates and invalid `Date` objects, are refused when the server is built, see [Validation](#validation). | +| `recent` | `false` | `true` makes the message `\Recent` for the first session that selects the mailbox, see [`\Recent`](#recent). | Messages are sorted by UID when the server loads them. Messages without a `uid` get UIDs after the highest UID of the mailbox, in the order they are listed. That is why the plain string in the first example got UID 46 and not 1. @@ -318,7 +318,7 @@ try { } ``` -The check covers the types of the known keys, `type` and `separator` of namespaces, and keys that look like a typo of a known key (the same key in another case, or one edit away, such as `uidValidity` or `flagz`). Other unknown keys are allowed, since plugins keep their own data on mailboxes and messages. +The check covers the types of the known keys, `type` and `separator` of namespaces, `internaldate` and `SAVEDATE` values that are not an RFC 3501 date-time string or a valid `Date`, and keys that look like a typo of a known key (the same key in another case, or one edit away, such as `uidValidity` or `flagz`). Other unknown keys are allowed, since plugins keep their own data on mailboxes and messages. The package exports the same check and a JSON Schema (draft 2020-12) of the format: diff --git a/src/dates.ts b/src/dates.ts index b76e189..b08886e 100644 --- a/src/dates.ts +++ b/src/dates.ts @@ -57,6 +57,29 @@ function dateKey(day: number | string, month: number, year: number | string): st return String(year).padStart(4, '0') + '-' + String(month + 1).padStart(2, '0') + '-' + String(day).padStart(2, '0'); } +/** + * Checks a date-time string of RFC 3501 section 9, like an INTERNALDATE ("14-Sep-2013 21:22:28 -0300"): + * date-time = DQUOTE date-day-fixed "-" date-month "-" date-year SP time SP zone DQUOTE + * + * @param {String} value Value to check + * @return {Boolean} true if the value is a date-time string of a real date and time + */ +function isDateTime(value: unknown): boolean { + if (!value || typeof value !== 'string') { + return false; + } + // month names are case-insensitive like all ABNF strings + const match = value.match(/^( \d|\d\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4}) (\d{2}):(\d{2}):(\d{2}) [-+](\d{2})(\d{2})$/i); + if (!match) { + return false; + } + + // the values must also make a real date and time + return ( + isRealDate(match[1], monthIndex(match[2]), match[3]) && Number(match[4]) < 24 && Number(match[5]) < 60 && Number(match[6]) < 61 && Number(match[8]) < 60 + ); +} + /** * Parses a date-time value of the RFC 3501 section 9 form, like an INTERNALDATE ("14-Sep-2013 21:22:28 -0300"). * A value with only the date part is accepted as well, its time fields are left undefined @@ -131,4 +154,4 @@ function toTimestamp(date: DateParts): number { return Date.UTC(date.year, date.month, date.day, date.hours || 0, date.minutes || 0, date.seconds || 0) - offset * 60 * 1000; } -export { MONTHS, monthIndex, isRealDate, dateKey, parseDateTime, parseHeaderDate, toTimestamp }; +export { MONTHS, monthIndex, isRealDate, isDateTime, dateKey, parseDateTime, parseHeaderDate, toTimestamp }; diff --git a/src/server.ts b/src/server.ts index bf4b613..a45ba01 100644 --- a/src/server.ts +++ b/src/server.ts @@ -9,7 +9,7 @@ import { commands as builtinCommands } from './commands/index.js'; import { getCommandOptions, commandOptions } from './command-states.js'; import type { ResolvedCommandOptions } from './command-states.js'; import validateMailboxName from './mailbox-name.js'; -import { MONTHS, monthIndex, isRealDate } from './dates.js'; +import { MONTHS, monthIndex, isDateTime } from './dates.js'; import fetchHandlers from './commands/handlers/fetch.js'; import { hasSequenceSetKey } from './commands/handlers/search.js'; import { isSequenceSet } from './numbers.js'; @@ -722,23 +722,8 @@ class IMAPServer extends Stream { * @return {Boolean} Returns true if the date string is in IMAP date-time format */ validateInternalDate(date: unknown): boolean { - if (!date || typeof date !== 'string') { - return false; - } - // date-time from RFC 3501 section 9, month names are case-insensitive like all ABNF strings - const match = date.match(/^( \d|\d\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4}) (\d{2}):(\d{2}):(\d{2}) [-+](\d{2})(\d{2})$/i); - if (!match) { - return false; - } - - // the values must also make a real date and time - return ( - isRealDate(match[1], monthIndex(match[2]), match[3]) && - Number(match[4]) < 24 && - Number(match[5]) < 60 && - Number(match[6]) < 61 && - Number(match[8]) < 60 - ); + // date-time from RFC 3501 section 9 + return isDateTime(date); } /** diff --git a/src/storage-schema.ts b/src/storage-schema.ts index e1d0e46..f12e4f2 100644 --- a/src/storage-schema.ts +++ b/src/storage-schema.ts @@ -6,6 +6,7 @@ */ import { MAX_NUMBER } from './numbers.js'; +import { isDateTime } from './dates.js'; // keys of plugins, an unknown key that looks like a typo of a known key is refused unless a plugin uses it const PLUGIN_KEYS = [ @@ -156,13 +157,20 @@ export function validateStorage(storage: unknown): void { if (message.flags !== undefined && typeof message.flags !== 'string' && !isFlagList(message.flags)) { fail(path + '.flags', 'must be a flag or a list of flags'); } - if ( - message.internaldate !== undefined && - message.internaldate !== false && - typeof message.internaldate !== 'string' && - !(message.internaldate instanceof Date) - ) { - fail(path + '.internaldate', 'must be a date-time string or a Date'); + // FETCH sends the internal date as it is and SORT ARRIVAL reads it, so it must be a date-time of RFC 3501 + // section 9 (month names in any case, they are sent as "Jan", "Feb", ...) or a valid Date. The save date + // of the SAVEDATE plugin is a date-time too (RFC 8514 section 4.2) + for (const key of ['internaldate', 'SAVEDATE'] as const) { + const value = message[key]; + if (value === undefined || value === false || (key === 'SAVEDATE' && value === null)) { + continue; + } + if (typeof value !== 'string' && !(value instanceof Date)) { + fail(path + '.' + key, 'must be a date-time string or a Date'); + } + if (value instanceof Date ? isNaN(value.getTime()) : !isDateTime(value)) { + fail(path + '.' + key, 'must be a date-time string like "14-Sep-2013 21:22:28 -0300" or a Date, not ' + JSON.stringify(value)); + } } if (message.recent !== undefined && typeof message.recent !== 'boolean') { fail(path + '.recent', 'must be true or false'); diff --git a/test/savedate.test.ts b/test/savedate.test.ts index 3f0cb96..a601f18 100644 --- a/test/savedate.test.ts +++ b/test/savedate.test.ts @@ -163,7 +163,10 @@ describe('SAVEDATE', () => { }); it('refuses invalid save dates in storage', () => { - assert.throws(() => imapkit({ plugins: ['SAVEDATE'], storage: { INBOX: { messages: [{ raw: 'x', SAVEDATE: 'yesterday' }] } } }), /Invalid SAVEDATE/); + assert.throws( + () => imapkit({ plugins: ['SAVEDATE'], storage: { INBOX: { messages: [{ raw: 'x', SAVEDATE: 'yesterday' }] } } }), + /Invalid storage at "INBOX"\.messages\[0\]\.SAVEDATE: must be a date-time/ + ); }); }); diff --git a/test/search.test.ts b/test/search.test.ts index 8526835..fcbdb79 100644 --- a/test/search.test.ts +++ b/test/search.test.ts @@ -569,8 +569,9 @@ describe('Search with unusual data', () => { INBOX: { messages: [ { + // no Date header; an internal date that is not a date-time is refused when the server is built raw: 'Subject: folded\r\n subject line\r\nX-Foo: bar\r\n\r\nbody', - internaldate: 'not a date' + internaldate: '30-Sep-2026 23:59:59 +0000' }, { raw: 'Subject: second\r\nDate: Mon, 5 Oct 26 10:00:00 +0300\r\n\r\nbody', @@ -585,7 +586,7 @@ describe('Search with unusual data', () => { } })); - it('a bad internal date does not break date searches', (t, done) => { + it('a message without a Date header does not break date searches', (t, done) => { const cmds = ['A1 LOGIN testuser testpass', 'A2 SELECT INBOX', 'A3 SEARCH SINCE 1-Oct-2026', 'A4 SEARCH SENTON 5-Oct-2026', 'ZZ LOGOUT']; ctx.run(cmds, resp => { diff --git a/test/storage-schema.test.ts b/test/storage-schema.test.ts index 00db83b..ab515c8 100644 --- a/test/storage-schema.test.ts +++ b/test/storage-schema.test.ts @@ -25,6 +25,36 @@ describe('storage validation', () => { ['a UID of 0', { INBOX: { messages: [{ raw: 'x', uid: 0 }] } }, /messages\[0\]\.uid: must be an integer/], ['flags that are not strings', { INBOX: { messages: [{ raw: 'x', flags: [1] }] } }, /\.flags: must be a flag or a list of flags/], ['a numeric internal date', { INBOX: { messages: [{ raw: 'x', internaldate: 5 }] } }, /\.internaldate: must be a date-time string or a Date/], + // RFC 3501 section 9: date-time = DQUOTE date-day-fixed "-" date-month "-" date-year SP time SP zone DQUOTE + [ + 'an RFC 5322 date as the internal date', + { INBOX: { messages: [{ raw: 'x', internaldate: 'Thu, 1 Jan 2026 10:00:00 +0000' }] } }, + /"INBOX"\.messages\[0\]\.internaldate: must be a date-time string like "14-Sep-2013 21:22:28 -0300" or a Date/ + ], + [ + 'an internal date without a zone', + { INBOX: { messages: [{ raw: 'x', internaldate: '01-Jan-2026 10:00:00' }] } }, + /\.internaldate: must be a date-time/ + ], + [ + 'an internal date that does not exist', + { INBOX: { messages: [{ raw: 'x', internaldate: '31-Feb-2026 10:00:00 +0000' }] } }, + /\.internaldate: must be a date-time/ + ], + [ + 'an internal date with a time out of range', + { INBOX: { messages: [{ raw: 'x', internaldate: '01-Jan-2026 24:00:00 +0000' }] } }, + /\.internaldate: must be a date-time/ + ], + ['an empty internal date', { INBOX: { messages: [{ raw: 'x', internaldate: '' }] } }, /\.internaldate: must be a date-time/], + ['an invalid Date as the internal date', { INBOX: { messages: [{ raw: 'x', internaldate: new Date('x') }] } }, /\.internaldate: must be a date-time/], + // RFC 8514 section 4.2: the save date is a date-time too + [ + 'an RFC 5322 date as the save date', + { '': { folders: { A: { messages: [{ raw: 'x', SAVEDATE: 'Thu, 1 Jan 2026 10:00:00 +0000' }] } } } }, + /""\.folders\["A"\]\.messages\[0\]\.SAVEDATE: must be a date-time string like "14-Sep-2013 21:22:28 -0300" or a Date/ + ], + ['a numeric save date', { INBOX: { messages: [{ raw: 'x', SAVEDATE: 5 }] } }, /\.SAVEDATE: must be a date-time/], ['a raw source that is a number', { INBOX: { messages: [{ raw: 5 }] } }, /\.raw: must be a string/], ['recent that is not a boolean', { INBOX: { messages: [{ raw: 'x', recent: 'yes' }] } }, /\.recent: must be true or false/], ['a UIDVALIDITY above 32 bits', { INBOX: { uidvalidity: 2 ** 32 } }, /"INBOX"\.uidvalidity: must be an integer/], @@ -44,6 +74,25 @@ describe('storage validation', () => { it('fails when the server is built', () => { assert.throws(() => imapkit({ storage: { INBOX: { message: [] } } as never }), /Invalid storage at "INBOX"/); + assert.throws( + () => imapkit({ storage: { INBOX: { messages: [{ raw: 'x', internaldate: 'Thu, 1 Jan 2026 10:00:00 +0000' }] } } }), + /Invalid storage at "INBOX"\.messages\[0\]\.internaldate/ + ); + }); + + it('accepts date-time strings in any month case, Dates and no internal date', () => { + validateStorage({ + INBOX: { + messages: [ + { raw: 'x', internaldate: '14-Sep-2013 21:22:28 -0300' }, + { raw: 'x', internaldate: ' 1-jan-2026 00:00:60 +0000' }, + { raw: 'x', internaldate: new Date(0) as never }, + { raw: 'x', internaldate: false }, + { raw: 'x', SAVEDATE: '29-Feb-2024 23:59:59 +1400' } as never, + { raw: 'x' } + ] + } + }); }); it('allows the data of plugins and raw sources as strings or Buffers', () => { From 172d6becf6e7f9e32975ed075b1febf6d92687fc Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:36:03 +0300 Subject: [PATCH 12/17] fix: send RECENT after the EXISTS of new messages to IMAP4rev1 sessions RFC 3501 section 7.3.2: the RECENT response occurs "if the size of the mailbox changes (e.g., new messages)" and the client MUST record it. A run of EXISTS responses that announces new messages (from another session, the session's own APPEND, COPY or MOVE, SMTP or the control API) is now followed by one RECENT response with the number of \Recent messages the session knows. With NOTIFY it follows the FETCH of the new message (RFC 5465 section 5.2). Sessions that enabled IMAP4rev2 do not get it (RFC 9051 Appendix E). Co-Authored-By: Claude Opus 5.5 --- src/plugins/imap4rev2.ts | 5 ++++ src/server.ts | 55 +++++++++++++++++++++++++++++++++---- test/context-search.test.ts | 4 +-- test/esort.test.ts | 5 +++- test/idle.test.ts | 2 +- test/imap4rev2.test.ts | 38 ++++++++++++++++++++----- test/mailbox.test.ts | 3 +- test/move.test.ts | 2 +- test/multiappend.test.ts | 2 +- test/notify.test.ts | 14 ++++++---- test/replace.test.ts | 8 +++--- test/script.test.ts | 2 +- test/sessions.test.ts | 51 ++++++++++++++++++++++++++++++++++ test/uidonly.test.ts | 2 +- 14 files changed, 161 insertions(+), 32 deletions(-) diff --git a/src/plugins/imap4rev2.ts b/src/plugins/imap4rev2.ts index 3ab9a3c..df3599d 100644 --- a/src/plugins/imap4rev2.ts +++ b/src/plugins/imap4rev2.ts @@ -155,6 +155,11 @@ export default function imap4rev2Plugin(server: IMAPServer) { }; server.outputHandlers.push((connection: IMAPConnection, response: IMAPResponse, description: string, parsed: ParsedCommand, data: string) => { + if (description === 'RECENT NOTIFICATION' && isRev2(connection)) { + // Appendix E item 12: no RECENT response after the EXISTS of new messages either + response.skipResponse = true; + return; + } if (!parsed || !response || !isRev2(connection)) { return; } diff --git a/src/server.ts b/src/server.ts index e37b597..6322075 100644 --- a/src/server.ts +++ b/src/server.ts @@ -153,6 +153,17 @@ function isPendingExpunge(notification: Notification): boolean { return !!notification.mailboxCopy || (!!notification.attributes && (notification.attributes[1] || {}).value === 'EXPUNGE'); } +/** + * Checks if a notification is an untagged EXISTS response + * + * @param {Object} [notification] Notification to send + * @return {Boolean} true for an EXISTS response + */ +function isExists(notification?: Notification): boolean { + const name = notification && !notification.command && notification.attributes && notification.attributes[1]; + return !!name && name.type === 'ATOM' && name.value === 'EXISTS'; +} + /** * Creates a new IMAP server, call `listen()` on it to start accepting connections * @@ -2939,17 +2950,49 @@ class IMAPConnection { // a message changed several times is reported once, its FETCH response carries the current flags const reported = new Set(); + // before the snapshot (or with all of it still held back) the session knows the old list + const sessionList = (i: number) => (snapshot && (snapshotIndex < 0 || i < snapshotIndex) ? (snapshot.mailboxCopy as Message[]) : current); + + // the last EXISTS response of a run of EXISTS responses, and if the run announces new messages + let lastExists = -1; + let newMessages = false; queue.forEach((notification, i) => { if (notification.flagUpdate) { - // before the snapshot (or with all of it still held back) the session knows the old list - this.sendFlagUpdate( - notification.flagUpdate, - getSequence(snapshot && (snapshotIndex < 0 || i < snapshotIndex) ? (snapshot.mailboxCopy as Message[]) : current), - reported - ); + this.sendFlagUpdate(notification.flagUpdate, getSequence(sessionList(i)), reported); } else { this.send(notification); } + if (isExists(notification)) { + lastExists = i; + // the EXISTS after the EXPUNGE responses of another session carries the snapshot instead of a message + newMessages = newMessages || !!notification.message; + } + const next = queue[i + 1]; + if (lastExists >= 0 && !isExists(next) && !(next && next.fetchedMessage)) { + const announced = newMessages; + const existsIndex = lastExists; + lastExists = -1; + newMessages = false; + if (!announced) { + // after expunges, the EXPUNGE responses report the change (RFC 3501 section 7.4.1) + return; + } + // RFC 3501 section 7.3.2: the RECENT response "occurs as a result of a SELECT or EXAMINE command, and if + // the size of the mailbox changes (e.g., new messages)". One for consecutive EXISTS responses that announce + // new messages, with the number of \Recent messages among those the client was told about. It goes after + // the FETCH responses that NOTIFY sends for new messages, RFC 5465 section 5.2: "an unsolicited EXISTS + // response, followed by an unsolicited FETCH response [...] The server MAY also send a RECENT response" + const count = Number(queue[existsIndex].attributes[0]); + const known = sessionList(existsIndex).slice(0, count); + this.send( + { + tag: '*', + notification: true, + attributes: [known.filter(message => this.isRecent(message)).length, { type: 'ATOM', value: 'RECENT' }] + }, + 'RECENT NOTIFICATION' + ); + } }); } diff --git a/test/context-search.test.ts b/test/context-search.test.ts index 682c6d2..46dfdce 100644 --- a/test/context-search.test.ts +++ b/test/context-search.test.ts @@ -129,7 +129,7 @@ describe('CONTEXT=SEARCH', () => { it('sends ADDTO after EXISTS for appended messages', (t, done) => { ctx.run([...LOGIN, 'A1 SEARCH RETURN (UPDATE) UNSEEN', 'A2 APPEND INBOX {12}\r\nSubject: x\r\n', 'ZZ LOGOUT'], resp => { resp = resp.toString(); - assert.match(resp, /^\* 5 EXISTS\r\n\* ESEARCH \(TAG "A1"\) ADDTO \(0 5\)\r\nA2 OK /m); + assert.match(resp, /^\* 5 EXISTS\r\n\* 1 RECENT\r\n\* ESEARCH \(TAG "A1"\) ADDTO \(0 5\)\r\nA2 OK /m); done(); }); }); @@ -147,7 +147,7 @@ describe('CONTEXT=SEARCH', () => { let output = await first.cmd('A3 NOOP'); assert.match( output, - /^\* 1 FETCH \(UID 10 FLAGS \(\)\)\r\n\* 2 FETCH \(UID 20 FLAGS \(\\Seen \\Deleted\)\)\r\n\* 5 EXISTS\r\n\* ESEARCH \(TAG "A1"\) UID REMOVEFROM \(0 20\) ADDTO \(0 10,41\)\r\n\* ESEARCH \(TAG "A2"\) REMOVEFROM \(0 2\) ADDTO \(0 1,5\)\r\nA3 OK /m + /^\* 1 FETCH \(UID 10 FLAGS \(\)\)\r\n\* 2 FETCH \(UID 20 FLAGS \(\\Seen \\Deleted\)\)\r\n\* 5 EXISTS\r\n\* 1 RECENT\r\n\* ESEARCH \(TAG "A1"\) UID REMOVEFROM \(0 20\) ADDTO \(0 10,41\)\r\n\* ESEARCH \(TAG "A2"\) REMOVEFROM \(0 2\) ADDTO \(0 1,5\)\r\nA3 OK /m ); await second.cmd('B4 STORE 1 +FLAGS (\\Deleted)'); diff --git a/test/esort.test.ts b/test/esort.test.ts index 5fcded1..005c46e 100644 --- a/test/esort.test.ts +++ b/test/esort.test.ts @@ -220,7 +220,10 @@ describe('CONTEXT=SORT', () => { // bravo (3) is the second result and the third one reversed assert.match(resp, /^\* ESEARCH \(TAG "A1"\) REMOVEFROM \(2 3\)\r\n\* ESEARCH \(TAG "A2"\) UID REMOVEFROM \(3 3\)\r\n\* 3 EXPUNGE\r\nA7 OK /m); // bingo is second, after alpha - assert.match(resp, /^\* 4 EXISTS\r\n\* ESEARCH \(TAG "A1"\) ADDTO \(2 4\)\r\n\* ESEARCH \(TAG "A2"\) UID ADDTO \(3 5\)\r\nA8 OK /m); + assert.match( + resp, + /^\* 4 EXISTS\r\n\* 1 RECENT\r\n\* ESEARCH \(TAG "A1"\) ADDTO \(2 4\)\r\n\* ESEARCH \(TAG "A2"\) UID ADDTO \(3 5\)\r\nA8 OK /m + ); done(); } ); diff --git a/test/idle.test.ts b/test/idle.test.ts index 9f10413..a60364b 100644 --- a/test/idle.test.ts +++ b/test/idle.test.ts @@ -82,7 +82,7 @@ describe('IDLE', () => { } client.send('A4 NOOP'); client.waitFor('A4 OK', () => { - assert.ok(client.output.indexOf('* 3 EXISTS\r\nA4 OK') >= 0, client.output); + assert.ok(client.output.indexOf('* 3 EXISTS\r\n* 2 RECENT\r\nA4 OK') >= 0, client.output); client.close(); done(); }); diff --git a/test/imap4rev2.test.ts b/test/imap4rev2.test.ts index e55a501..08d37f9 100644 --- a/test/imap4rev2.test.ts +++ b/test/imap4rev2.test.ts @@ -437,14 +437,15 @@ describe('IMAP4rev2', () => { assert.match(resp, /^\* 1 RECENT\r$/m); }); - it('unsolicited FETCH responses include the UID and no \\Recent (section 7.5.2)', async () => { - const open = () => - new Promise<{ cmd: (line: string) => Promise; session: Session }>(resolve => { - openSession(ctx.port, session => { - const cmd = (line: string) => new Promise(done => session.run(line, done)); - resolve({ cmd, session }); - }); + const open = () => + new Promise<{ cmd: (line: string) => Promise; session: Session }>(resolve => { + openSession(ctx.port, session => { + const cmd = (line: string) => new Promise(done => session.run(line, done)); + resolve({ cmd, session }); }); + }); + + it('unsolicited FETCH responses include the UID and no \\Recent (section 7.5.2)', async () => { const a = await open(); const b = await open(); try { @@ -461,4 +462,27 @@ describe('IMAP4rev2', () => { b.session.close(); } }); + + // Appendix E item 12 removes the RECENT response, an IMAP4rev1 session gets it after the EXISTS (RFC 3501 section 7.3.2) + it('new messages are reported with EXISTS and without RECENT (Appendix E item 12)', async () => { + const a = await open(); + const b = await open(); + const c = await open(); + try { + await a.cmd(LOGIN); + await a.cmd(ENABLE); + await a.cmd('A1 SELECT INBOX'); + await b.cmd(LOGIN); + await b.cmd('B1 EXAMINE INBOX'); + await c.cmd(LOGIN); + await c.cmd('C1 APPEND INBOX {12}\r\nSubject: x\r\n'); + assert.match(await a.cmd('A2 NOOP'), /^\* 4 EXISTS\r\nA2 OK /); + // A selected INBOX first and took \\Recent of the messages, the EXAMINE session B has none + assert.match(await b.cmd('B2 NOOP'), /^\* 4 EXISTS\r\n\* 0 RECENT\r\nB2 OK /); + } finally { + a.session.close(); + b.session.close(); + c.session.close(); + } + }); }); diff --git a/test/mailbox.test.ts b/test/mailbox.test.ts index 730b157..ed90d3b 100644 --- a/test/mailbox.test.ts +++ b/test/mailbox.test.ts @@ -143,7 +143,8 @@ describe('SELECT and EXAMINE', () => { ctx.run(cmds, resp => { resp = resp.toString(); - assert.ok(resp.indexOf('\r\n* 4 EXISTS\r\nA3 OK') >= 0); + // message 2 and the new one are \Recent in this session (RFC 3501 section 7.3.2) + assert.match(resp, /^\* 4 EXISTS\r\n\* 2 RECENT\r\nA3 OK /m); done(); }); }); diff --git a/test/move.test.ts b/test/move.test.ts index 8987f61..f93ddac 100644 --- a/test/move.test.ts +++ b/test/move.test.ts @@ -77,7 +77,7 @@ describe('ImapKit tests', () => { ctx.run(cmds, resp => { resp = resp.toString(); - assert.ok(resp.indexOf('\r\n* OK [COPYUID 1 1 4] Copied\r\n* 4 EXISTS\r\n* 1 EXPUNGE\r\nA3 OK') >= 0, resp); + assert.ok(resp.indexOf('\r\n* OK [COPYUID 1 1 4] Copied\r\n* 4 EXISTS\r\n* 1 RECENT\r\n* 1 EXPUNGE\r\nA3 OK') >= 0, resp); assert.equal(ctx.server.getMailbox('INBOX')!.messages.length, 3); done(); }); diff --git a/test/multiappend.test.ts b/test/multiappend.test.ts index 22d2023..3f9ab7f 100644 --- a/test/multiappend.test.ts +++ b/test/multiappend.test.ts @@ -65,7 +65,7 @@ describe('MULTIAPPEND', () => { it('sends EXISTS to the session that has the mailbox selected', (t, done) => { ctx.run(['A1 LOGIN testuser testpass', 'A2 SELECT INBOX', 'A3 APPEND INBOX ' + literal(msg(2)) + ' ' + literal(msg(3)), 'ZZ LOGOUT'], resp => { resp = resp.toString(); - assert.match(resp, /^\* 3 EXISTS\r\nA3 OK \[APPENDUID 1 2:3\] /m); + assert.match(resp, /^\* 2 EXISTS\r\n\* 3 EXISTS\r\n\* 2 RECENT\r\nA3 OK \[APPENDUID 1 2:3\] /m); done(); }); }); diff --git a/test/notify.test.ts b/test/notify.test.ts index e0eb504..a0bf92e 100644 --- a/test/notify.test.ts +++ b/test/notify.test.ts @@ -206,7 +206,7 @@ describe('NOTIFY', () => { await run(a, 'S1 SELECT INBOX'); await run(b, append('B1', 'INBOX')); const resp = await run(a, 'A1 NOTIFY SET (selected (MessageNew MessageExpunge))'); - assert.match(resp, /^\* 4 EXISTS\r\nA1 OK/m); + assert.match(resp, /^\* 4 EXISTS\r\n\* 1 RECENT\r\nA1 OK/m); }); }); @@ -219,10 +219,11 @@ describe('NOTIFY', () => { await run(a, 'S1 SELECT INBOX'); await run(a, 'A1 NOTIFY SET (selected (MessageNew (UID FLAGS BODY.PEEK[HEADER.FIELDS (SUBJECT)]) MessageExpunge))'); await run(b, append('B1', 'INBOX', '\\Flagged')); - const resp = await expect(a, /^\* 4 FETCH/); + // the RECENT response of an IMAP4rev1 session follows the FETCH ("MAY also send a RECENT response") + const resp = await expect(a, /^\* 1 RECENT/); assert.match( resp, - /^\* 4 EXISTS\r\n\* 4 FETCH \(UID 4 FLAGS \(\\Flagged \\Recent\) BODY\[HEADER\.FIELDS \(SUBJECT\)\] \{20\}\r\nSubject: new one\r\n\r\n\)$/m + /^\* 4 EXISTS\r\n\* 4 FETCH \(UID 4 FLAGS \(\\Flagged \\Recent\) BODY\[HEADER\.FIELDS \(SUBJECT\)\] \{20\}\r\nSubject: new one\r\n\r\n\)\r\n\* 1 RECENT$/m ); // the fetch attributes never set \Seen const flags = await run(a, 'A2 UID FETCH 4 FLAGS'); @@ -323,7 +324,7 @@ describe('NOTIFY', () => { assert.match(resp, /^A2 OK \[EXPUNGEISSUED\] /m); await assertQuiet(a); resp = await run(a, 'A3 NOOP'); - assert.match(resp, /^\* 3 EXPUNGE\r\n\* 2 EXISTS\r\n\* 3 EXISTS\r\n\* 3 FETCH \(UID 4\)\r\nA3 OK/m); + assert.match(resp, /^\* 3 EXPUNGE\r\n\* 2 EXISTS\r\n\* 3 EXISTS\r\n\* 3 FETCH \(UID 4\)\r\n\* 1 RECENT\r\nA3 OK/m); }); it('holds notifications during FETCH and sends them after it with SELECTED (RFC 3501 section 7.4.1)', async () => { @@ -437,7 +438,8 @@ describe('NOTIFY', () => { await run(a, 'S1 SELECT INBOX'); await run(a, 'A1 NOTIFY SET (selected (MessageNew MessageExpunge)) (personal (MessageNew MessageExpunge FlagChange))'); await run(b, append('B1', 'INBOX')); - const resp = await expect(a, /^\* 4 EXISTS/); + const resp = await expect(a, /^\* 1 RECENT/); + assert.match(resp, /^\* 4 EXISTS\r\n\* 1 RECENT$/m); assert.doesNotMatch(resp, /STATUS/); await assertQuiet(a); }); @@ -591,7 +593,7 @@ describe('NOTIFY with other extensions', () => { await run(a, 'A2 SEARCH RETURN (UPDATE) FROM new'); await run(b, append('B1', 'INBOX')); const resp = await expect(a, /^\* ESEARCH/); - assert.match(resp, /^\* 4 EXISTS\r\n\* 4 FETCH \(UID 4\)\r\n\* ESEARCH \(TAG "A2"\) ADDTO \(0 4\)$/m); + assert.match(resp, /^\* 4 EXISTS\r\n\* 4 FETCH \(UID 4\)\r\n\* 1 RECENT\r\n\* ESEARCH \(TAG "A2"\) ADDTO \(0 4\)$/m); }); }); diff --git a/test/replace.test.ts b/test/replace.test.ts index 59f8efe..8c596a3 100644 --- a/test/replace.test.ts +++ b/test/replace.test.ts @@ -39,7 +39,7 @@ describe('REPLACE', () => { ctx.run(cmds, resp => { resp = resp.toString(); // APPENDUID in an untagged OK before the EXISTS and EXPUNGE responses, like the example in section 3.2 - assert.match(resp, /^\+ Go ahead\r\n\* OK \[APPENDUID 1 4\] [^\r\n]+\r\n\* 4 EXISTS\r\n\* 2 EXPUNGE\r\nA3 OK [^\r\n]+\r\n/m); + assert.match(resp, /^\+ Go ahead\r\n\* OK \[APPENDUID 1 4\] [^\r\n]+\r\n\* 4 EXISTS\r\n\* 1 RECENT\r\n\* 2 EXPUNGE\r\nA3 OK [^\r\n]+\r\n/m); // no flags are inherited from the replaced message (RFC 8508 section 1) assert.match( resp, @@ -53,7 +53,7 @@ describe('REPLACE', () => { const cmds = [LOGIN, SELECT, 'A3 UID REPLACE 3 INBOX ' + literal(msg(4)), 'A4 UID FETCH 1:* UID', 'ZZ LOGOUT']; ctx.run(cmds, resp => { resp = resp.toString(); - assert.match(resp, /^\* OK \[APPENDUID 1 4\] [^\r\n]+\r\n\* 4 EXISTS\r\n\* 3 EXPUNGE\r\nA3 OK /m); + assert.match(resp, /^\* OK \[APPENDUID 1 4\] [^\r\n]+\r\n\* 4 EXISTS\r\n\* 1 RECENT\r\n\* 3 EXPUNGE\r\nA3 OK /m); assert.match(resp, /^\* 1 FETCH \(UID 1\)\r\n\* 2 FETCH \(UID 2\)\r\n\* 3 FETCH \(UID 4\)\r\n/m); done(); }); @@ -211,7 +211,7 @@ describe('REPLACE', () => { ctx.run([LOGIN, SELECT, 'A3 REPLACE 1 INBOX ' + literal(msg(4)), 'ZZ LOGOUT'], () => { a.run('S3 NOOP', output => { a.close(); - assert.match(output, /^\* 4 EXISTS\r\n\* 1 EXPUNGE\r\n/m); + assert.match(output, /^\* 4 EXISTS\r\n\* 1 RECENT\r\n\* 1 EXPUNGE\r\n/m); done(); }); }); @@ -246,7 +246,7 @@ describe('REPLACE without UIDPLUS', () => { ctx.run([LOGIN, SELECT, 'A3 REPLACE 1 INBOX ' + literal(msg(4)), 'ZZ LOGOUT'], resp => { resp = resp.toString(); assert.doesNotMatch(resp, /APPENDUID|^\* OK Replacement/m); - assert.match(resp, /^\* 4 EXISTS\r\n\* 1 EXPUNGE\r\nA3 OK /m); + assert.match(resp, /^\* 4 EXISTS\r\n\* 1 RECENT\r\n\* 1 EXPUNGE\r\nA3 OK /m); done(); }); }); diff --git a/test/script.test.ts b/test/script.test.ts index 6426ac4..4f3a954 100644 --- a/test/script.test.ts +++ b/test/script.test.ts @@ -262,7 +262,7 @@ describe('Script rules', () => { await first.waitFor(/^A1 OK/m); const secondNoop = await command(second, 'N1 NOOP'); // the first session sees every EXISTS changed, the second one the real count - assert.match(first.output(), /^\* 99 EXISTS\r\n[\s\S]*^\* 99 EXISTS\r\nA1 OK/m); + assert.match(first.output(), /^\* 99 EXISTS\r\n[\s\S]*^\* 99 EXISTS\r\n\* 1 RECENT\r\nA1 OK/m); assert.doesNotMatch(first.output(), /^\* \d EXISTS/m); assert.match(secondNoop, /^\* 3 EXISTS\r\n/m); first.close(); diff --git a/test/sessions.test.ts b/test/sessions.test.ts index 3ec2d8c..4932255 100644 --- a/test/sessions.test.ts +++ b/test/sessions.test.ts @@ -493,6 +493,14 @@ describe('Multiple sessions', () => { ]); }); + it('delivers RECENT after the EXISTS of a new message while idling (RFC 3501 7.3.2)', async () => { + const watcher = await startIdle('INBOX'); + const c = await open(); + await c.cmd('APPEND INBOX {' + message(5).length + '}\r\n' + message(5)); + const output = await watcher.waitFor('RECENT\r\n'); + assert.match(output, /^\* 5 EXISTS\r\n\* 1 RECENT\r\n$/m); + }); + it('delivers flag changes while idling', async () => { const watcher = await startIdle('INBOX'); const b = await open('INBOX'); @@ -569,5 +577,48 @@ describe('Multiple sessions', () => { const seen = [await recentIn(a), await recentIn(b)]; assert.strictEqual(seen.filter(Boolean).length, 1, JSON.stringify(seen)); }); + + // RFC 3501 7.3.2: the RECENT response "occurs as a result of a SELECT or EXAMINE command, and if the size of the + // mailbox changes (e.g., new messages)", and "The update from the RECENT response MUST be recorded by the client" + it('a RECENT response with the new count follows the EXISTS of new messages (RFC 3501 7.3.2)', async () => { + const a = await open('INBOX'); + const b = await open('Fresh', true); + const c = await open(); + + await c.cmd('APPEND INBOX {' + message(5).length + '}\r\n' + message(5)); + await c.cmd('APPEND INBOX {' + message(6).length + '}\r\n' + message(6)); + // A is the only session with INBOX selected, both new messages are recent in A + let output = await a.cmd('NOOP'); + assert.match(output, /^\* 5 EXISTS\r\n\* 6 EXISTS\r\n\* 2 RECENT\r\nT\d+ OK /); + output = await a.cmd('SEARCH RECENT'); + assert.match(output, /^\* SEARCH 5 6\r\n/); + + // the session that appends to its selected mailbox gets them as well (RFC 3501 6.3.11) + output = await a.cmd('APPEND INBOX {' + message(7).length + '}\r\n' + message(7)); + assert.match(output, /^\* 7 EXISTS\r\n\* 3 RECENT\r\nT\d+ OK \[APPENDUID /); + + // B examined Fresh and sees its 2 recent messages, a new message there stays recent for the next session + // that selects the mailbox read-write (RFC 3501 6.3.2), so the count of B stays the same + await c.cmd('APPEND Fresh {' + message(8).length + '}\r\n' + message(8)); + output = await b.cmd('FETCH 4 (UID)'); + assert.match(output, /^\* 4 EXISTS\r\n\* 2 RECENT\r\n\* 4 FETCH \(UID 4\)\r\n/); + }); + + it('expunges by another session are reported without RECENT, new messages after them with it (RFC 3501 7.3.2)', async () => { + const a = await open('Fresh'); + const b = await open('Fresh'); + await b.cmd('STORE 1 +FLAGS.SILENT (\\Deleted)'); + await b.cmd('EXPUNGE'); + // the EXPUNGE response reports the new size (RFC 3501 7.4.1), the EXISTS after it is not needed + let output = await a.cmd('NOOP'); + assert.match(output, /^\* 1 EXPUNGE\r\n\* 2 EXISTS\r\nT\d+ OK /); + + await b.cmd('STORE 1 +FLAGS.SILENT (\\Deleted)'); + await b.cmd('EXPUNGE'); + await b.cmd('APPEND Fresh {' + message(5).length + '}\r\n' + message(5)); + // both messages that were recent in A are gone, the new message is recent in A + output = await a.cmd('NOOP'); + assert.match(output, /^\* 1 EXPUNGE\r\n\* 1 EXISTS\r\n\* 2 EXISTS\r\n\* 1 RECENT\r\nT\d+ OK /); + }); }); }); diff --git a/test/uidonly.test.ts b/test/uidonly.test.ts index e8a39aa..6db0c74 100644 --- a/test/uidonly.test.ts +++ b/test/uidonly.test.ts @@ -246,7 +246,7 @@ describe('UIDONLY', () => { ], resp => { assert.match(resp, /^\* 20 UIDFETCH \(FLAGS \(\\Seen\)\)\r\n\* 30 UIDFETCH \(FLAGS \(\)\)\r\nA5 OK/m); - assert.match(resp, /^\* OK \[APPENDUID 42 41\] .*\r\n\* 5 EXISTS\r\n\* VANISHED 30\r\nA6 OK/m); + assert.match(resp, /^\* OK \[APPENDUID 42 41\] .*\r\n\* 5 EXISTS\r\n\* 1 RECENT\r\n\* VANISHED 30\r\nA6 OK/m); assert.match(resp, /^\* 20 UIDFETCH \(FLAGS \(\\Seen\)\)\r\nA7 OK/m); done(); } From a8b13c1444a54f3fe0afc25e32e51f17503a39ce Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:36:47 +0300 Subject: [PATCH 13/17] fix: refuse quirk presets that remove a plugin IMAP4rev2 requires no-move and no-uidplus removed MOVE and UIDPLUS even when IMAP4rev2 was loaded, which advertised IMAP4rev2 without the MOVE command, UID EXPUNGE, APPENDUID and COPYUID that RFC 9051 folds into the base protocol (Appendix E). The server constructor now throws "IMAP4rev2 requires MOVE, which the "no-move" quirk removes" when a loaded plugin requires a plugin a quirk removes. Co-Authored-By: Claude Opus 5.5 --- README.md | 14 +++++++------- docs/docs/extensions/overview.md | 11 ++++++----- docs/docs/faults/quirk-presets.md | 2 +- src/load-plugins.ts | 26 ++++++++++++++++++++++---- src/quirks.ts | 8 ++++---- src/server.ts | 2 +- test/quirks.test.ts | 12 ++++++++++++ 7 files changed, 53 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index deacfc2..964b996 100644 --- a/README.md +++ b/README.md @@ -575,13 +575,13 @@ Faults change only the output and the handling of the lines a rule matches, the The `quirks` option (`--quirk` for the `imapkit` command) turns on named presets that make the server behave like a known real server, so a client test reproduces that server's bug in every run, without the server itself. A preset is a set of script rules, after the rules of the `script` option, and plugins it leaves out. The presets are exported as data (`import { quirks } from 'imapkit'`), copy one into your own script rules to adjust it. -| Quirk | Behavior | -| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `james-fetchgroup` | Apache James FetchGroup: only the first section asked for a part in one FETCH is answered, later ones for the same part are empty (`BODY[2.MIME] BODY[2]` gives a zero-length body) | -| `james-late-fetch` | Apache James: 1 in 4 FETCH responses come after the tagged OK of their command | -| `yahoo-quoted-sections` | Yahoo: short body sections (up to 100 octets without line breaks) are quoted strings instead of literals | -| `m365-throttle` | Microsoft 365: 1 in 10 commands (not LOGOUT) is refused with `BAD Request is throttled. Suggested Backoff Time: 1000 milliseconds` | -| `no-uidplus`, `no-move` | servers without UIDPLUS or MOVE, the plugins are not loaded even when `plugins` lists them | +| Quirk | Behavior | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `james-fetchgroup` | Apache James FetchGroup: only the first section asked for a part in one FETCH is answered, later ones for the same part are empty (`BODY[2.MIME] BODY[2]` gives a zero-length body) | +| `james-late-fetch` | Apache James: 1 in 4 FETCH responses come after the tagged OK of their command | +| `yahoo-quoted-sections` | Yahoo: short body sections (up to 100 octets without line breaks) are quoted strings instead of literals | +| `m365-throttle` | Microsoft 365: 1 in 10 commands (not LOGOUT) is refused with `BAD Request is throttled. Suggested Backoff Time: 1000 milliseconds` | +| `no-uidplus`, `no-move` | servers without UIDPLUS or MOVE, the plugins are not loaded even when `plugins` lists them, and with IMAP4rev2 (which requires both, RFC 9051 Appendix E) the server constructor throws | ```javascript const server = imapkit({ plugins: ['IDLE', 'MOVE'], quirks: ['james-fetchgroup', 'm365-throttle'], scriptSeed: 42 }); diff --git a/docs/docs/extensions/overview.md b/docs/docs/extensions/overview.md index 2e038d7..b882915 100644 --- a/docs/docs/extensions/overview.md +++ b/docs/docs/extensions/overview.md @@ -66,10 +66,11 @@ Some plugins only add to others when both are loaded: LIST-MYRIGHTS comes with A A few extensions exclude each other, and loading both throws an error when the server is created: -| Combination | Error | Reason | -| ------------------------------ | ----------------------------------------------------------------------------- | ------------------------------------------------------ | -| `LITERAL+` and `LITERAL-` | `LITERAL- can not be enabled together with LITERAL+` (or the other way round) | RFC 7888 section 5: a server must not advertise both | -| `MESSAGELIMIT` and `SAVELIMIT` | `SAVELIMIT can not be enabled together with MESSAGELIMIT` | RFC 9738 section 3: a server advertises one of the two | +| Combination | Error | Reason | +| --------------------------------------------------- | ----------------------------------------------------------------------------- | -------------------------------------------------------- | +| `LITERAL+` and `LITERAL-` | `LITERAL- can not be enabled together with LITERAL+` (or the other way round) | RFC 7888 section 5: a server must not advertise both | +| `MESSAGELIMIT` and `SAVELIMIT` | `SAVELIMIT can not be enabled together with MESSAGELIMIT` | RFC 9738 section 3: a server advertises one of the two | +| `IMAP4rev2` and the `no-move` or `no-uidplus` quirk | `IMAP4rev2 requires MOVE, which the "no-move" quirk removes` (or UIDPLUS) | RFC 9051 Appendix E: IMAP4rev2 folds in MOVE and UIDPLUS | IMAP4rev2 loads LITERAL- only when LITERAL+ is not loaded, and a LITERAL+ loaded after it replaces that implied LITERAL-, so `['IMAP4rev2', 'LITERAL+']` works. @@ -77,7 +78,7 @@ IMAP4rev2 loads LITERAL- only when LITERAL+ is not loaded, and a LITERAL+ loaded A plugin that is not loaded leaves no trace. Without CONDSTORE, messages have no MODSEQ value and `SELECT INBOX (CONDSTORE)` is answered with `BAD`. Without MOVE, `MOVE` is an unknown command. This lets you check that your client only uses what the server advertises. -The `no-uidplus` and `no-move` [quirk presets](../faults/quirk-presets.md) remove UIDPLUS and MOVE even when the plugin list names them. +The `no-uidplus` and `no-move` [quirk presets](../faults/quirk-presets.md) remove UIDPLUS and MOVE even when the plugin list names them. With IMAP4rev2, which requires both (RFC 9051 Appendix E), they throw `IMAP4rev2 requires MOVE, which the "no-move" quirk removes` (or the same for UIDPLUS) when the server is created. ### Capabilities diff --git a/docs/docs/faults/quirk-presets.md b/docs/docs/faults/quirk-presets.md index 36fbb63..b50b50d 100644 --- a/docs/docs/faults/quirk-presets.md +++ b/docs/docs/faults/quirk-presets.md @@ -30,7 +30,7 @@ imapkit -p 1143 --plugin=IDLE,MOVE --quirk=no-move --quirk=m365-throttle --scrip - An unknown name fails the server constructor with the list of known ones: `Unknown quirk "james". Available quirks: james-fetchgroup, james-late-fetch, yahoo-quoted-sections, m365-throttle, no-uidplus, no-move`. - The rules of the presets are added after the rules of the `script` option, in the order the presets are listed. Rules you add later with `server.script.add()` come after them. As the first matching rule handles an event, a rule of your own in `script` can take an event before a preset sees it. - The rules of a preset are ordinary script rules: they show up in `server.script.rules`, count `hits`, emit the `script` event, and `server.script.clear()` removes them too. -- `removePlugins` of a preset keeps the plugins out even when the `plugins` option lists them, and also when another plugin requires them. With `IMAP4rev2`, `no-move` and `no-uidplus` still remove MOVE and UIDPLUS, which gives a server that [RFC 9051](https://www.rfc-editor.org/rfc/rfc9051) does not allow. Use them with IMAP4rev1 servers. +- `removePlugins` of a preset keeps the plugins out even when the `plugins` option lists them. When another loaded plugin requires one of them, the server constructor throws instead. IMAP4rev2 folds in MOVE and UIDPLUS ([RFC 9051 Appendix E](https://www.rfc-editor.org/rfc/rfc9051#appendix-E), sections 6.3.12, 6.4.7, 6.4.8 and 6.4.9), so a server can not advertise IMAP4rev2 without them: `plugins: ['IMAP4rev2'], quirks: ['no-move']` throws `IMAP4rev2 requires MOVE, which the "no-move" quirk removes`, and `no-uidplus` throws the same for UIDPLUS. Use these presets with IMAP4rev1 servers. ## The presets diff --git a/src/load-plugins.ts b/src/load-plugins.ts index 5e3e790..b3ae721 100644 --- a/src/load-plugins.ts +++ b/src/load-plugins.ts @@ -51,11 +51,19 @@ function resolvePlugin(name: string): string | false { * * @param {Object} server IMAPServer instance * @param {Array|String|Function} plugins List of plugins to load + * @param {Map} [exclude] Plugins a quirk preset leaves out, plugin name to quirk name + * @throws {Error} when a loaded plugin requires an excluded one */ -function loadPlugins(server: IMAPServer, plugins: (string | Plugin)[] | string | Plugin | null | undefined, exclude?: string[]): void { +function loadPlugins(server: IMAPServer, plugins: (string | Plugin)[] | string | Plugin | null | undefined, exclude?: Map): void { const loaded = new Set(); - // plugins a quirk preset leaves out (no-move), they are not loaded even when listed - const excluded = new Set((exclude || []).map(name => resolvePlugin(name)).flatMap(name => (name ? [builtinPlugins[name]] : []))); + // plugins a quirk preset leaves out (no-move), they are not loaded even when listed, and the quirk that removes them + const excluded = new Map(); + (exclude || new Map()).forEach((quirk, name) => { + const resolved = resolvePlugin(name); + if (resolved) { + excluded.set(builtinPlugins[resolved], quirk); + } + }); const load = (entry: string | Plugin) => { let plugin = entry; @@ -81,7 +89,17 @@ function loadPlugins(server: IMAPServer, plugins: (string | Plugin)[] | string | } loaded.add(plugin); - ([] as string[]).concat(plugin.requires || []).forEach(load); + const requires = ([] as string[]).concat(plugin.requires || []); + requires.forEach(name => { + const required = resolvePlugin(name); + const quirk = required && excluded.get(builtinPlugins[required]); + if (quirk) { + // the plugin can not work without it (IMAP4rev2 folds in MOVE and UIDPLUS, RFC 9051 Appendix E), so + // advertising it without the required plugin would break its RFC + throw new Error((typeof entry === 'string' ? entry.trim() : plugin.name) + ' requires ' + name + ', which the "' + quirk + '" quirk removes'); + } + }); + requires.forEach(load); plugin(server); }; diff --git a/src/quirks.ts b/src/quirks.ts index b0626c7..05704bd 100644 --- a/src/quirks.ts +++ b/src/quirks.ts @@ -123,19 +123,19 @@ export const quirks: Record = { * Resolves the `quirks` option * * @param {Array|String} names Quirk names - * @return {Object} `{ rules, removePlugins }` of all of them + * @return {Object} `{ rules, removePlugins }` of all of them, removePlugins maps a plugin name to the quirk that removes it * @throws {Error} for an unknown name */ -export function resolveQuirks(names: string[] | string | null | undefined): { rules: ScriptRule[]; removePlugins: Set } { +export function resolveQuirks(names: string[] | string | null | undefined): { rules: ScriptRule[]; removePlugins: Map } { const rules: ScriptRule[] = []; - const removePlugins = new Set(); + const removePlugins = new Map(); ([] as string[]).concat(names || []).forEach(name => { const quirk = Object.hasOwn(quirks, String(name).toLowerCase()) ? quirks[String(name).toLowerCase()] : null; if (!quirk) { throw new Error('Unknown quirk "' + name + '". Available quirks: ' + Object.keys(quirks).join(', ')); } rules.push(...(quirk.rules || [])); - (quirk.removePlugins || []).forEach(plugin => removePlugins.add(plugin)); + (quirk.removePlugins || []).forEach(plugin => removePlugins.set(plugin, String(name).toLowerCase())); }); return { rules, removePlugins }; } diff --git a/src/server.ts b/src/server.ts index 3270ddc..a2acd37 100644 --- a/src/server.ts +++ b/src/server.ts @@ -307,7 +307,7 @@ class IMAPServer extends Stream { this.control = new Control(this); // a quirk preset can leave plugins out (no-move, no-uidplus) - loadPlugins(this, this.options.plugins, [...quirks.removePlugins]); + loadPlugins(this, this.options.plugins, quirks.removePlugins); if (this.options.storage) { // a typo in a fixture fails here with the path of the problem diff --git a/test/quirks.test.ts b/test/quirks.test.ts index ad9af31..ac274d9 100644 --- a/test/quirks.test.ts +++ b/test/quirks.test.ts @@ -22,6 +22,18 @@ describe('quirks', () => { assert.throws(() => imapkit({ quirks: ['nope'] }), /Unknown quirk "nope". Available quirks: james-fetchgroup/); }); + it('refuse to remove a plugin that another listed plugin requires', () => { + // RFC 9051 Appendix E: IMAP4rev2 folds in UIDPLUS and MOVE (sections 6.3.12, 6.4.7, 6.4.8 and 6.4.9), so a + // server that advertises IMAP4rev2 can not leave them out + assert.throws(() => imapkit({ plugins: ['IMAP4rev2'], quirks: ['no-move'] }), /^Error: IMAP4rev2 requires MOVE, which the "no-move" quirk removes$/); + assert.throws( + () => imapkit({ plugins: ['IDLE', 'imap4rev2'], quirks: ['no-uidplus'] }), + /^Error: imap4rev2 requires UIDPLUS, which the "no-uidplus" quirk removes$/ + ); + // a plugin the quirk removes only because it is listed is still left out + assert.strictEqual(imapkit({ plugins: ['MOVE', 'QRESYNC'], quirks: ['no-move'] }).capabilities.MOVE, undefined); + }); + it('are exported as data', () => { assert.deepStrictEqual(Object.keys(quirks), [ 'james-fetchgroup', From 49d07ad76b2b64cfdd0b92dbf54c5fc75479d976 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:43:08 +0300 Subject: [PATCH 14/17] docs: describe EXISTS, RECENT and EXPUNGEISSUED timing for multiple sessions Real transcripts for new messages reported before FETCH responses, a new message that waits for a pending EXPUNGE, UID SEARCH with OK [EXPUNGEISSUED] and the RECENT response after EXISTS. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 +-- .../control-api/mailboxes-and-messages.md | 11 +-- docs/docs/control-api/overview.md | 20 ++--- docs/docs/extensions/imap4rev2.md | 20 ++--- docs/docs/extensions/messages.md | 3 +- docs/docs/extensions/synchronization.md | 3 +- docs/docs/guides/multiple-sessions.md | 73 ++++++++++++++++--- docs/docs/reference/custom-plugins.md | 2 +- 8 files changed, 102 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 4f7b0f0..aa468f8 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ ImapKit is meant for developing standards compliant IMAP clients, so it follows - with NOTIFY (RFC 5465): MessageNew without MessageExpunge or the other way round, FlagChange without both (section 5), mailbox events or two selected filters with `selected`/`selected-delayed` (section 6.1), fetch attributes outside the selected filters, empty event or mailbox lists, `NOTIFY SET` without event groups; unknown events get `NO [BADEVENT (...)]` listing the supported ones (section 3.1) - after `ENABLE IMAP4rev2` (RFC 9051): `CHECK`, `LSUB`, the `RFC822`, `RFC822.HEADER` and `RFC822.TEXT` FETCH items, the `NEW`, `OLD` and `RECENT` SEARCH keys and the `RECENT` STATUS item, none of which are in the RFC 9051 grammar (Appendix E), numbers above 63 bits in LARGER and SMALLER, and invalid UTF-8 or mailbox names that are not Net-Unicode (section 5.1); before it, 8-bit quoted strings (Appendix A) and partial ranges, LARGER and SMALLER values above 32 bits (RFC 3501 section 9 number) -Responses follow the grammar strictly too: strings that can not be quoted are sent as literals. Failures carry the RFC 5530 response codes that RFC 9051 section 7.1 lists, in both protocol revisions: `AUTHENTICATIONFAILED` and `AUTHORIZATIONFAILED` for logins, `ALREADYEXISTS`, `NONEXISTENT`, `CANNOT`, `HASCHILDREN` and `NOPERM` for mailbox operations, `TRYCREATE` when the target of APPEND, COPY or MOVE does not exist or is a `\Noselect` name, `CLIENTBUG` for `STATUS` on the selected mailbox and for `STORE`, `EXPUNGE`, `UID EXPUNGE`, `MOVE` and `REPLACE` in a mailbox selected read-only, and `EXPUNGEISSUED` when FETCH, STORE, SEARCH, SORT or THREAD completes while the EXPUNGE of another session can not be reported yet. +Responses follow the grammar strictly too: strings that can not be quoted are sent as literals. Failures carry the RFC 5530 response codes that RFC 9051 section 7.1 lists, in both protocol revisions: `AUTHENTICATIONFAILED` and `AUTHORIZATIONFAILED` for logins, `ALREADYEXISTS`, `NONEXISTENT`, `CANNOT`, `HASCHILDREN` and `NOPERM` for mailbox operations, `TRYCREATE` when the target of APPEND, COPY or MOVE does not exist or is a `\Noselect` name, `CLIENTBUG` for `STATUS` on the selected mailbox and for `STORE`, `EXPUNGE`, `UID EXPUNGE`, `MOVE` and `REPLACE` in a mailbox selected read-only, and `EXPUNGEISSUED` when FETCH, STORE, SEARCH, SORT or THREAD (or UID SEARCH, UID SORT or UID THREAD with message numbers in the criteria) completes while the EXPUNGE of another session can not be reported yet. Some client side recommendations of RFC 2683 are checked too: `STATUS` on the selected mailbox gets `CLIENTBUG` (section 3.1.1), mailbox names must be valid modified UTF-7 (section 3.4.2), and EXPUNGE or STORE after EXAMINE, which answered `[READ-ONLY]`, get `NO` (section 3.3.2). Command lines are not limited to the 1000 octets that section 3.2.1.5 suggests for clients, the server accepts up to 1 MiB (the section asks servers for at least 8000 octets) and answers a longer line with `BAD`. @@ -112,7 +112,9 @@ ImapKit follows these of the strategies that RFC 2180 (IMAP4 Multi-Accessed Mail - a session that has not been told about the EXPUNGE of another session yet keeps its message numbers. FETCH still returns the expunged messages (section 4.1.1) and SEARCH still finds them (section 4.3), both end with `OK [EXPUNGEISSUED]` - STORE does not change expunged messages: with `.SILENT` it ends with `OK` (section 4.2.1), otherwise the other messages are stored and get their FETCH responses, and the tagged response is `NO [EXPUNGEISSUED]` (sections 4.2.2 and 4.2.3; with CONDSTORE `NO [MODIFIED ...]` when that applies, RFC 7162 section 3.1.3) - COPY and MOVE of a set that includes an expunged message copy nothing and return the pending EXPUNGE responses with `NO [EXPUNGEISSUED]` (section 4.4.1) -- UID commands report the pending EXPUNGE responses before they run (RFC 3501 section 7.4.1), the UIDs of the expunged messages then no longer exist and are ignored (RFC 3501 section 6.4.8). UID SEARCH with message numbers in its criteria still uses the old numbers +- UID commands report the pending EXPUNGE responses before they run (RFC 3501 section 7.4.1), the UIDs of the expunged messages then no longer exist and are ignored (RFC 3501 section 6.4.8). UID SEARCH (UID SORT, UID THREAD) with message numbers in its criteria still uses the old numbers and ends with `OK [EXPUNGEISSUED]` +- a command that refers to messages (FETCH, STORE, SEARCH, COPY, MOVE, SORT, THREAD and the UID commands) first reports the new messages (EXISTS) and flag changes of other sessions, so every message number in its responses is one the client was told about (RFC 3501 section 5.2: "A server MUST send mailbox size updates automatically if a mailbox size change is observed during the processing of a command"). While an EXPUNGE is pending, only what was queued before it goes out, a message that arrived after it is announced after the EXPUNGE, and until then its number is answered with BAD (RFC 3501 section 9, seq-number) +- the EXISTS responses of new messages are followed by `* n RECENT` with the number of `\Recent` messages of the session (RFC 3501 section 7.3.2), also for its own APPEND, COPY and MOVE into the selected mailbox, but not after `ENABLE IMAP4rev2` - DELETE of a mailbox that other sessions have selected disconnects them with `* BYE` (section 3.3) - RENAME keeps the messages of the mailbox under the new name, sessions that have it selected keep working, the old name no longer exists (section 3.4) - a session that ends without LOGOUT or CLOSE does not expunge anything (RFC 2683 section 3.1.2), and there is no inactivity timeout @@ -168,7 +170,7 @@ An unknown plugin name throws an error, and a plugin listed more than once is lo - **MOVE** Adds MOVE [RFC6851] capability (MOVE and UID MOVE commands) - **MULTIAPPEND** Adds MULTIAPPEND [RFC3502] capability. APPEND takes several messages and appends all or none of them. With UIDPLUS, APPENDUID lists the UIDs as a UID set - **NAMESPACE** Adds NAMESPACE [RFC2342] capability -- **NOTIFY** Adds NOTIFY [RFC5465] capability: `NOTIFY SET [STATUS] (filter events) ...` and `NOTIFY NONE` with the `selected`, `selected-delayed`, `inboxes` (same as `personal`), `personal`, `subscribed`, `subtree` and `mailboxes` filters and the MessageNew (with fetch attributes for the selected mailbox), MessageExpunge, FlagChange, MailboxName (LIST with `OLDNAME` for RENAME) and SubscriptionChange events, plus MailboxMetadataChange and ServerMetadataChange with METADATA. Events are sent as soon as they happen, also between commands, except EXPUNGE (or VANISHED) with `selected-delayed` and during FETCH, STORE and SEARCH. After the first NOTIFY a session only hears about the events it asked for, also for the selected mailbox; changes made by the session itself are not reported. Other mailboxes are reported with STATUS (UNSEEN when the `\Seen` count changed, HIGHESTMODSEQ when CONDSTORE is enabled), with ACL only mailboxes with the `l` and `r` rights, and granting or revoking `l` counts as MailboxName. Fetch attributes never set `\Seen`. `server.notifyOverflow([connection])` sends `* OK [NOTIFICATIONOVERFLOW]` and turns NOTIFY off. AnnotationChange (no ANNOTATE support) is refused with `NO [BADEVENT]`, the fetch attributes of the CONTEXT=SEARCH `UPDATE` option (RFC 5465 section 7) are not supported +- **NOTIFY** Adds NOTIFY [RFC5465] capability: `NOTIFY SET [STATUS] (filter events) ...` and `NOTIFY NONE` with the `selected`, `selected-delayed`, `inboxes` (same as `personal`), `personal`, `subscribed`, `subtree` and `mailboxes` filters and the MessageNew (with fetch attributes for the selected mailbox), MessageExpunge, FlagChange, MailboxName (LIST with `OLDNAME` for RENAME) and SubscriptionChange events, plus MailboxMetadataChange and ServerMetadataChange with METADATA. Events are sent as soon as they happen, also between commands, while a command runs they wait for its tagged response, and EXPUNGE (or VANISHED) waits longer with `selected-delayed` and during FETCH, STORE and SEARCH. For a new message in the selected mailbox an IMAP4rev1 session gets EXISTS, the requested FETCH and then RECENT. After the first NOTIFY a session only hears about the events it asked for, also for the selected mailbox; changes made by the session itself are not reported. Other mailboxes are reported with STATUS (UNSEEN when the `\Seen` count changed, HIGHESTMODSEQ when CONDSTORE is enabled), with ACL only mailboxes with the `l` and `r` rights, and granting or revoking `l` counts as MailboxName. Fetch attributes never set `\Seen`. `server.notifyOverflow([connection])` sends `* OK [NOTIFICATIONOVERFLOW]` and turns NOTIFY off. AnnotationChange (no ANNOTATE support) is refused with `NO [BADEVENT]`, the fetch attributes of the CONTEXT=SEARCH `UPDATE` option (RFC 5465 section 7) are not supported - **OAUTHBEARER** Adds AUTH=OAUTHBEARER [RFC7628] capability, with or without SASL-IR. Uses the same credentials as XOAUTH2: access token `"testtoken"`, the authzid in the GS2 header (`n,a=testuser,`) is optional. A failed login gets the JSON error result as a continuation request (`invalid_token` or `invalid_request`), the client must answer it with `AQ==` (a single `%x01`) - **OBJECTID** Adds OBJECTID [RFC8474] capability: `MAILBOXID` for CREATE, SELECT, EXAMINE and STATUS, `EMAILID` and `THREADID` for FETCH and SEARCH. Ids are generated (`F1`, `M1`, `T1`, ...) unless the storage sets a `MAILBOXID` for a mailbox or an `EMAILID` / `THREADID` for a message. COPY, MOVE and RENAME INBOX keep the EMAILID and THREADID of a message. Messages are threaded by their `Message-ID`, `In-Reply-To` and `References` headers across all mailboxes, a message joins the thread of the nearest known parent when it is added - **PARTIAL** Adds PARTIAL [RFC9394] capability, also loads ESEARCH: the `PARTIAL` result option of SEARCH (`RETURN (PARTIAL 1:100)`, `RETURN (PARTIAL -1:-100)` counts from the last result) and of SORT with ESORT, and the `PARTIAL` modifier of FETCH and UID FETCH (`UID FETCH 1:* (FLAGS) (PARTIAL -1:-50)`), which combines with CHANGEDSINCE. With PARTIAL loaded a command takes only one PARTIAL or ALL result option @@ -376,7 +378,7 @@ describe('IMAP tests', () => { ## Control API -`server.control` changes and inspects the server from your test, without an IMAP session. Every change reaches the connected sessions the way a change by another session would: a selected session gets `EXISTS` for a new message, `EXPUNGE` (or `VANISHED` after `ENABLE QRESYNC`) for a removed one, an unsolicited `FETCH` with the UID and the new flags (with `MODSEQ` after `ENABLE CONDSTORE`), and `BYE` when its mailbox is deleted. NOTIFY and CONTEXT=SEARCH sessions get their updates too. ACL does not apply to the control API, but every argument is checked. +`server.control` changes and inspects the server from your test, without an IMAP session. Every change reaches the connected sessions the way a change by another session would: a selected session gets `EXISTS` (and `RECENT` in IMAP4rev1) for a new message, `EXPUNGE` (or `VANISHED` after `ENABLE QRESYNC`) for a removed one, an unsolicited `FETCH` with the UID and the new flags (with `MODSEQ` after `ENABLE CONDSTORE`), and `BYE` when its mailbox is deleted. NOTIFY and CONTEXT=SEARCH sessions get their updates too. ACL does not apply to the control API, but every argument is checked. ```javascript const server = imapkit({ plugins: ['IDLE', 'CONDSTORE'] }); @@ -694,7 +696,7 @@ Where - **mailboxArguments** lists the positions of arguments that are mailbox names, these must be valid modified UTF-7 (RFC 3501 section 5.1.3) - **astringArguments** lists the positions of other astring arguments (user names, identifiers). In these, in mailbox names and in search criteria an atom `NIL` reaches the handler as an atom, not as `null` - **searchCriteria** is the position where SEARCH style criteria start, **sequenceSet** the position of an argument with message sequence numbers. Both are used for the RFC 3501 section 5.5 pipelining check - - **noExpunge** if true, EXPUNGE responses are held back while the command runs (like FETCH, STORE and SEARCH, RFC 3501 section 7.4.1) + - **noExpunge** if true, EXPUNGE responses are held back while the command runs (like FETCH, STORE and SEARCH, RFC 3501 section 7.4.1), the notifications queued before them are still sent - **literal8** if true (or the name of the capability that allows it), the command accepts `~{n}` literals (RFC 3516) - **noPipelining** if true, the command is refused with BAD when the client sent more input after it (STARTTLS, COMPRESS), and so are the commands sent with it - **appendMessage** if true, the command takes a message after its mailbox argument like APPEND (REPLACE), so a message literal to a missing mailbox is refused with `NO [TRYCREATE]` before it is sent diff --git a/docs/docs/control-api/mailboxes-and-messages.md b/docs/docs/control-api/mailboxes-and-messages.md index 6db415e..d2d2ae5 100644 --- a/docs/docs/control-api/mailboxes-and-messages.md +++ b/docs/docs/control-api/mailboxes-and-messages.md @@ -154,13 +154,13 @@ Adds a message like a delivery from outside. Returns the new UID and the UIDVALIDITY of the mailbox. A `Date` is stored in the local time zone of the process, a string is kept as it is. -**What sessions see:** sessions that have the mailbox selected get `* n EXISTS`. The first read-write session that has it selected sees the message as `\Recent`, like a delivery. +**What sessions see:** sessions that have the mailbox selected get `* n EXISTS`, and IMAP4rev1 sessions `* n RECENT` with their new count of `\Recent` messages. The first read-write session that has it selected sees the message as `\Recent`, like a delivery. ```javascript const server = imapkit({ plugins: ['IDLE'] }); // ... the client logs in, selects INBOX and starts IDLE server.control.addMessage('INBOX', { raw: 'Subject: three\r\n\r\nz\r\n' }); -// the idling client receives: * 2 EXISTS +// the idling client receives: * 2 EXISTS and * 1 RECENT ``` **The `checks` option.** Without it, the message is added whatever the limits are, which is how you fill a mailbox above its quota. With `checks: true` the checks of the loaded plugins run first: QUOTA refuses a message over a hard limit with `OVERQUOTA`, APPENDLIMIT one over the limit with `TOOBIG`. A soft quota does not refuse anything. @@ -231,7 +231,7 @@ copyMessages(path: string, uids: number[], target: string): { uidvalidity: numbe Copies messages to another mailbox, in UID order like COPY. The copies keep the flags and internal date and get new UIDs in the target. Returns the UIDVALIDITY of the target and the new UID of every message, the same information COPYUID carries. -**What sessions see:** sessions that have the target selected get `EXISTS`. +**What sessions see:** sessions that have the target selected get `EXISTS` and `RECENT`. ```javascript server.control.copyMessages('INBOX', [2, 1], 'Archive/2024'); @@ -248,7 +248,7 @@ moveMessages(path: string, uids: number[], target: string): { uidvalidity: numbe Copies the messages like `copyMessages()` and then expunges them from the source. The return value has the same shape. It does not need the MOVE plugin. -**What sessions see:** `EXISTS` in the target, `EXPUNGE` (or `VANISHED`) and `EXISTS` in the source. +**What sessions see:** `EXISTS` and `RECENT` in the target, `EXPUNGE` (or `VANISHED`) and `EXISTS` in the source. ### replaceMessage(path, uid, message) @@ -258,10 +258,11 @@ replaceMessage(path: string, uid: number, message: { raw: string | Uint8Array; f Replaces a message with a new source. The content of a UID never changes ([RFC 9051 section 2.3.1.1](https://www.rfc-editor.org/rfc/rfc9051#section-2.3.1.1)), so this adds the new message and expunges the old one, and the new message gets a new UID. Flags and internal date of the old message are kept unless `message` gives new ones. Returns the new UID. -**What sessions see:** `EXISTS` for the new message, then `EXPUNGE` (or `VANISHED`) and `EXISTS` for the old one: +**What sessions see:** `EXISTS` and `RECENT` for the new message, then `EXPUNGE` (or `VANISHED`) and `EXISTS` for the old one: ``` * 4 EXISTS +* 1 RECENT * 1 EXPUNGE * 3 EXISTS ``` diff --git a/docs/docs/control-api/overview.md b/docs/docs/control-api/overview.md index 4e45e3b..1f993ae 100644 --- a/docs/docs/control-api/overview.md +++ b/docs/docs/control-api/overview.md @@ -27,15 +27,15 @@ The same operations are available over HTTP through the [REST API](../rest-api/o The control API changes the shared store the same way an IMAP command from another session would, so the connected sessions learn about it through the normal protocol: -| Change | What a session that has the mailbox selected sees | -| ----------------------------------------- | ------------------------------------------------------------------------------ | -| a new message (`addMessage`, copy, move) | `* n EXISTS`, and the message is `\Recent` for the first read-write session | -| removed messages (expunge, move, replace) | `* n EXPUNGE` and a new `EXISTS`, or `* VANISHED` after `ENABLE QRESYNC` | -| changed flags (`setFlags`) | `* n FETCH (UID u FLAGS (...))`, with `MODSEQ` after `ENABLE CONDSTORE` | -| the mailbox is deleted | `* BYE Selected mailbox was deleted`, and the connection closes | -| new UIDVALIDITY (`resetUidValidity`) | `* BYE UIDVALIDITY of the selected mailbox changed`, and the connection closes | +| Change | What a session that has the mailbox selected sees | +| ----------------------------------------- | ---------------------------------------------------------------------------------------- | +| a new message (`addMessage`, copy, move) | `* n EXISTS` and `* n RECENT`, the message is `\Recent` for the first read-write session | +| removed messages (expunge, move, replace) | `* n EXPUNGE` and a new `EXISTS`, or `* VANISHED` after `ENABLE QRESYNC` | +| changed flags (`setFlags`) | `* n FETCH (UID u FLAGS (...))`, with `MODSEQ` after `ENABLE CONDSTORE` | +| the mailbox is deleted | `* BYE Selected mailbox was deleted`, and the connection closes | +| new UIDVALIDITY (`resetUidValidity`) | `* BYE UIDVALIDITY of the selected mailbox changed`, and the connection closes | -A session in IDLE gets these responses right away. Any other session gets them before the tagged response of its next command (`NOOP` is the usual way to ask), following the same rules as changes from a real second session: FETCH, STORE, SEARCH, SORT and THREAD hold back EXPUNGE responses (RFC 3501 section 7.4.1), and messages expunged while a session can not be told stay in its view until it can. See [Multiple sessions](../guides/multiple-sessions.md) for the details. Sessions that use NOTIFY or CONTEXT=SEARCH get their updates too. +A session in IDLE gets these responses right away. Any other session gets them before the tagged response of its next command (`NOOP` is the usual way to ask), following the same rules as changes from a real second session: a command that refers to messages (FETCH, STORE, SEARCH ...) reports new messages and flag changes before its own responses (RFC 3501 section 5.2), FETCH, STORE, SEARCH, SORT and THREAD hold back EXPUNGE responses (RFC 3501 section 7.4.1), and messages expunged while a session can not be told stay in its view until it can. IMAP4rev1 sessions get a RECENT response after the EXISTS of new messages (RFC 3501 section 7.3.2). See [Multiple sessions](../guides/multiple-sessions.md) for the details. Sessions that use NOTIFY or CONTEXT=SEARCH get their updates too. Every change has no session as its **origin**. Inside the server, a change made by a command carries the session that ran it, so that, for example, NOTIFY does not report a session's own changes back to it. A control API change has `origin: null`, which means every session is told, including one that is running a command at that moment. The `mailbox`, `expunge` and `flags` [events](./events.md) carry the same `origin` value, `null` for the control API. This also holds when you call the control API from inside an event listener that fires during a command. @@ -48,11 +48,11 @@ sequenceDiagram participant B as Session 2 (INBOX selected) T->>C: addMessage('INBOX', { raw }) C->>S: append, origin null - S-->>A: * 1 EXISTS (right away) + S-->>A: * 1 EXISTS, * 1 RECENT (right away) S-->>B: queued C-->>T: { uid: 1, uidvalidity: 1 } B->>S: A5 NOOP - S-->>B: * 1 EXISTS + S-->>B: * 1 EXISTS, * 0 RECENT S-->>B: A5 OK ``` diff --git a/docs/docs/extensions/imap4rev2.md b/docs/docs/extensions/imap4rev2.md index 774731b..e99bf92 100644 --- a/docs/docs/extensions/imap4rev2.md +++ b/docs/docs/extensions/imap4rev2.md @@ -56,16 +56,16 @@ S: A4 OK SEARCH completed `ENABLE IMAP4rev2` must come before the first SELECT or EXAMINE (RFC 5161 section 3.1), as for every ENABLE. After it, the session follows RFC 9051: -| Area | IMAP4rev2 behavior | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| SELECT, EXAMINE | No `RECENT` response and no `[UNSEEN n]` code. An untagged `LIST` response for the selected mailbox, with its special-use attributes. `* OK [CLOSED]` when a mailbox was selected before (section 6.3.2) | -| `\Recent` | Not sent in FLAGS of FETCH responses (section 2.3.2) | -| SEARCH | Answers with an `ESEARCH` response, `RETURN (ALL)` when no result option is given (section 6.4.4). UTF-8 is assumed without `CHARSET`, and `CHARSET` is still allowed | -| STATUS | `DELETED` is allowed (section 6.3.11), `RECENT` is not | -| Strings, mailbox names | UTF-8 in quoted strings and mailbox names, which must be Net-Unicode in Normalization Form C (section 5.1) | -| APPEND | A message with 8-bit header fields can be appended (section 6.3.12) | -| message/global | Described in BODYSTRUCTURE and numbered in sections like message/rfc822 | -| Numbers | Partial FETCH ranges and LARGER/SMALLER take 63-bit numbers (number64) | +| Area | IMAP4rev2 behavior | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| SELECT, EXAMINE | No `RECENT` response (nor after the EXISTS of new messages) and no `[UNSEEN n]` code. An untagged `LIST` response for the selected mailbox, with its special-use attributes. `* OK [CLOSED]` when a mailbox was selected before (section 6.3.2) | +| `\Recent` | Not sent in FLAGS of FETCH responses (section 2.3.2) | +| SEARCH | Answers with an `ESEARCH` response, `RETURN (ALL)` when no result option is given (section 6.4.4). UTF-8 is assumed without `CHARSET`, and `CHARSET` is still allowed | +| STATUS | `DELETED` is allowed (section 6.3.11), `RECENT` is not | +| Strings, mailbox names | UTF-8 in quoted strings and mailbox names, which must be Net-Unicode in Normalization Form C (section 5.1) | +| APPEND | A message with 8-bit header fields can be appended (section 6.3.12) | +| message/global | Described in BODYSTRUCTURE and numbered in sections like message/rfc822 | +| Numbers | Partial FETCH ranges and LARGER/SMALLER take 63-bit numbers (number64) | Items that RFC 9051 removed (Appendix E) are refused with `BAD`: `CHECK` (use NOOP), `LSUB` (use `LIST (SUBSCRIBED)`), the `RFC822`, `RFC822.HEADER` and `RFC822.TEXT` FETCH items (use `BODY[]`, `BODY.PEEK[HEADER]`, `BODY[TEXT]`), the `NEW`, `OLD` and `RECENT` SEARCH keys and the `RECENT` STATUS item. diff --git a/docs/docs/extensions/messages.md b/docs/docs/extensions/messages.md index 2ed7474..aa0cac0 100644 --- a/docs/docs/extensions/messages.md +++ b/docs/docs/extensions/messages.md @@ -118,7 +118,7 @@ With ACL loaded, URLs of mailboxes the user can not read are refused with `NO [B Adds `REPLACE` and `UID REPLACE`: append a new version of a message and expunge the old one in one step (RFC 8508). The target can be the selected mailbox or another one. - Only the replaced message is expunged, not every `\Deleted` message. If the new message can not be appended, nothing changes. -- With UIDPLUS, `APPENDUID` comes in an untagged OK before the EXPUNGE. When the target is the selected mailbox, EXISTS comes before EXPUNGE, like the RFC 8508 section 3.2 example. +- With UIDPLUS, `APPENDUID` comes in an untagged OK before the EXPUNGE. When the target is the selected mailbox, EXISTS (and RECENT in IMAP4rev1) comes before EXPUNGE, like the RFC 8508 section 3.2 example. - REPLACE takes a single message even with MULTIAPPEND. It works with CATENATE and, with BINARY, with a literal8 message. - A message number past the end is `BAD`, a UID that does not exist is `NO`, and REPLACE in a read-only mailbox is `NO`. When the message to replace is known to be invalid, the literal is refused before the client sends it. - With QUOTA only the net usage counts. @@ -131,6 +131,7 @@ C: C: hello S: * OK [APPENDUID 1 5] Replacement message saved S: * 5 EXISTS +S: * 1 RECENT S: * 2 EXPUNGE S: A3 OK UID REPLACE completed C: A4 REPLACE 9 INBOX {20} diff --git a/docs/docs/extensions/synchronization.md b/docs/docs/extensions/synchronization.md index 1a27c84..0103643 100644 --- a/docs/docs/extensions/synchronization.md +++ b/docs/docs/extensions/synchronization.md @@ -57,6 +57,7 @@ B C: Subject: hi B C: B C: hello A S: * 5 EXISTS +A S: * 1 RECENT B S: B2 OK APPEND Completed A C: DONE A S: A3 OK IDLE terminated @@ -214,7 +215,7 @@ Adds `NOTIFY SET [STATUS] (filter (events)) ...` and `NOTIFY NONE` (RFC 5465). How events are delivered: -- Events are sent as soon as they happen, also between commands, except EXPUNGE (or VANISHED) with `selected-delayed`, and except during FETCH, STORE and SEARCH. +- Events are sent as soon as they happen, also between commands. While a command runs they wait for its tagged response, EXPUNGE (or VANISHED) waits longer with `selected-delayed` and during FETCH, STORE and SEARCH. For a new message in the selected mailbox an IMAP4rev1 session gets EXISTS, the requested FETCH and then RECENT (RFC 5465 section 5.2 allows the RECENT response). - After the first NOTIFY a session only hears about the events it asked for, also for the selected mailbox. Changes made by the session itself are not reported. - Other mailboxes are reported with STATUS: MESSAGES and UIDNEXT, UNSEEN when the `\Seen` count changed, and HIGHESTMODSEQ when CONDSTORE is enabled. With ACL, only mailboxes with the `l` and `r` rights are reported, and granting or revoking `l` counts as MailboxName. - Fetch attributes of MessageNew never set `\Seen`. diff --git a/docs/docs/guides/multiple-sessions.md b/docs/docs/guides/multiple-sessions.md index 00168bd..6c5c0f1 100644 --- a/docs/docs/guides/multiple-sessions.md +++ b/docs/docs/guides/multiple-sessions.md @@ -1,12 +1,12 @@ --- title: Multiple Sessions sidebar_position: 5 -description: How changes made by one session reach the others, when EXPUNGE responses may be sent, the RFC 2180 strategies for expunged messages, \Recent ownership, DELETE, RENAME and IDLE. +description: How changes made by one session reach the others, when EXISTS, RECENT and EXPUNGE responses may be sent, the RFC 2180 strategies for expunged messages, \Recent ownership, DELETE, RENAME and IDLE. --- # Multiple sessions -Any number of clients can connect to one ImapKit server, and they all work on the same mailbox tree (see [Authentication](./authentication.md)). When one session changes a mailbox, the other sessions that have it selected learn about it through unsolicited responses: `EXISTS` for new messages, `EXPUNGE` for removed ones, and `FETCH` with the new flags. +Any number of clients can connect to one ImapKit server, and they all work on the same mailbox tree (see [Authentication](./authentication.md)). When one session changes a mailbox, the other sessions that have it selected learn about it through unsolicited responses: `EXISTS` (and `RECENT`) for new messages, `EXPUNGE` for removed ones, and `FETCH` with the new flags. Changes made with the [control API](../control-api/overview.md) reach the sessions the same way, as if another client had made them. @@ -14,7 +14,7 @@ Changes made with the [control API](../control-api/overview.md) reach the sessio ## When notifications arrive -Notifications from other sessions are queued and sent before the tagged response of the next command the session runs. NOOP is the usual way for a client to collect them. +Notifications from other sessions are queued and sent before the tagged response of the next command the session runs, and before the responses of a command that refers to messages. NOOP is the usual way for a client to collect them. ```mermaid sequenceDiagram @@ -31,10 +31,10 @@ sequenceDiagram S-->>B: B5 OK Note over S: A is not running a command, its notifications wait A->>S: A3 FETCH 1:3 (UID FLAGS) + S-->>A: * 1 FETCH (UID 1 FLAGS (\Flagged)) S-->>A: three FETCH responses, message 2 still there S-->>A: A3 OK [EXPUNGEISSUED] A->>S: A4 NOOP - S-->>A: * 1 FETCH (UID 1 FLAGS (\Flagged)) S-->>A: * 2 EXPUNGE, * 2 EXISTS S-->>A: A4 OK ``` @@ -42,8 +42,10 @@ sequenceDiagram The rules: - Nothing is sent while the session has no command in progress ([RFC 3501 section 5.3](https://www.rfc-editor.org/rfc/rfc3501#section-5.3)). The one exception is IDLE, see [IDLE](#idle). -- EXPUNGE responses are not sent during FETCH, STORE and SEARCH ([RFC 3501 section 7.4.1](https://www.rfc-editor.org/rfc/rfc3501#section-7.4.1)), nor during the commands that extensions add to that list, such as SORT and THREAD. The other notifications wait with them, so the session's message numbers stay the same until the command completes. When an EXPUNGE is pending, the tagged OK of such a command carries `[EXPUNGEISSUED]`, which tells the client to send NOOP soon ([RFC 5530 section 3](https://www.rfc-editor.org/rfc/rfc5530#section-3)). -- UID commands report the pending EXPUNGE responses before they run, since EXPUNGE is allowed during UID commands. UID SEARCH with message numbers in its criteria is the exception: it runs on the old numbers and the EXPUNGE waits. +- A command that refers to messages (FETCH, STORE, SEARCH, COPY, MOVE, SORT, THREAD and the UID commands) first reports new messages and flag changes, before it resolves its message numbers. [RFC 3501 section 5.2](https://www.rfc-editor.org/rfc/rfc3501#section-5.2): "A server MUST send mailbox size updates automatically if a mailbox size change is observed during the processing of a command". So the client has been told about every message number that a response of the command uses, and a number above the count it was told about is answered with BAD ([RFC 3501 section 9](https://www.rfc-editor.org/rfc/rfc3501#section-9), seq-number). +- EXPUNGE responses are not sent during FETCH, STORE and SEARCH ([RFC 3501 section 7.4.1](https://www.rfc-editor.org/rfc/rfc3501#section-7.4.1)), nor during the commands that extensions add to that list, such as SORT and THREAD. Only the notifications queued before a pending EXPUNGE are sent, the ones queued after it wait with it, so the session's message numbers stay the same until the command completes. That includes a message that arrived after the expunge: its EXISTS response can only follow the EXPUNGE. When an EXPUNGE is pending, the tagged OK of such a command carries `[EXPUNGEISSUED]`, which tells the client to send NOOP soon ([RFC 5530 section 3](https://www.rfc-editor.org/rfc/rfc5530#section-3)). +- UID commands report the pending EXPUNGE responses before they run, since EXPUNGE is allowed during UID commands. UID SEARCH (and UID SORT and UID THREAD) with message numbers in its criteria is the exception: it runs on the old numbers, the EXPUNGE waits and the tagged OK carries `[EXPUNGEISSUED]`. +- The EXISTS responses of new messages are followed by a RECENT response with the number of messages that are `\Recent` in the session ([RFC 3501 section 7.3.2](https://www.rfc-editor.org/rfc/rfc3501#section-7.3.2): it "occurs as a result of a SELECT or EXAMINE command, and if the size of the mailbox changes (e.g., new messages)"), one for several EXISTS responses in a row. Also for the session's own APPEND, COPY or MOVE into its selected mailbox. Not after `ENABLE IMAP4rev2`, which removed the RECENT response. - Unsolicited flag updates always include the UID: `* 1 FETCH (UID 2 FLAGS (\Seen))`. [RFC 9051 section 7.5.2](https://www.rfc-editor.org/rfc/rfc9051#section-7.5.2) requires it, and it is valid in IMAP4rev1 too. A message changed several times is reported once, with its current flags. - The session that made a change gets the usual responses of its own command, never a notification of it. - A session that has the mailbox selected gets an EXISTS with the new count after EXPUNGE responses caused by another session. @@ -82,13 +84,13 @@ B S: * 2 EXPUNGE B S: B5 OK EXPUNGE Completed A C: A3 FETCH 1:3 (UID FLAGS) A S: * 1 FETCH (UID 1 FLAGS (\Flagged)) +A S: * 1 FETCH (UID 1 FLAGS (\Flagged)) A S: * 2 FETCH (UID 2 FLAGS (\Deleted)) A S: * 3 FETCH (UID 3 FLAGS ()) A S: A3 OK [EXPUNGEISSUED] FETCH Completed A C: A4 STORE 2 +FLAGS (\Seen) A S: A4 NO [EXPUNGEISSUED] Some of the messages no longer exist A C: A5 COPY 1:3 Archive -A S: * 1 FETCH (UID 1 FLAGS (\Flagged)) A S: * 2 EXPUNGE A S: * 2 EXISTS A S: A5 NO [EXPUNGEISSUED] Some of the requested messages no longer exist @@ -100,7 +102,7 @@ A S: * 2 FETCH (UID 3 FLAGS ()) A S: A7 OK FETCH Completed ``` -FETCH `A3` still returns message 2 and shows the new flags of message 1, STORE `A4` touches only the expunged message and fails, and COPY `A5` is the first command that may report the EXPUNGE, so it does, copies nothing and fails. After that, message 2 is UID 3. +FETCH `A3` first reports the flag change of B (the first `* 1 FETCH`, unsolicited, with the UID), then returns its own responses, message 2 still among them. STORE `A4` touches only the expunged message and fails, and COPY `A5` is the first command that may report the EXPUNGE, so it does, copies nothing and fails. After that, message 2 is UID 3. A UID command reports the EXPUNGE before it runs: @@ -118,6 +120,56 @@ A S: * 2 FETCH (FLAGS () UID 3) A S: A3 OK UID FETCH Completed ``` +A new message is reported before the responses of a command that refers to it, so the client knows its number. One that arrived after a pending expunge has to wait for the EXPUNGE, until then its number is not valid: + +```text +B C: B3 APPEND INBOX {18} +B S: + Go ahead +B C: Subject: new +B C: +B C: Hi +B S: * 4 EXISTS +B S: * 0 RECENT +B S: B3 OK [APPENDUID 1 4] APPEND Completed +A C: A3 FETCH 4 (UID FLAGS) +A S: * 4 EXISTS +A S: * 1 RECENT +A S: * 4 FETCH (UID 4 FLAGS (\Recent)) +A S: A3 OK FETCH Completed +B C: B4 STORE 1 +FLAGS.SILENT (\Deleted) +B S: B4 OK STORE completed +B C: B5 EXPUNGE +B S: * 1 EXPUNGE +B S: B5 OK EXPUNGE Completed +B C: B6 APPEND INBOX {18} +B S: + Go ahead +B C: Subject: new +B C: +B C: Hi +B S: * 4 EXISTS +B S: * 0 RECENT +B S: B6 OK [APPENDUID 1 5] APPEND Completed +A C: A4 FETCH 1:* (UID) +A S: * 1 FETCH (UID 1) +A S: * 2 FETCH (UID 2) +A S: * 3 FETCH (UID 3) +A S: * 4 FETCH (UID 4) +A S: A4 OK [EXPUNGEISSUED] FETCH Completed +A C: A5 FETCH 5 (UID) +A S: A5 BAD Message sequence number 5 is greater than the number of messages (4) +A C: A6 UID SEARCH 1:4 +A S: * SEARCH 1 2 3 4 +A S: A6 OK [EXPUNGEISSUED] UID SEARCH completed +A C: A7 NOOP +A S: * 1 EXPUNGE +A S: * 3 EXISTS +A S: * 4 EXISTS +A S: * 2 RECENT +A S: A7 OK Completed +``` + +The new message is `\Recent` in A, the first session that selected INBOX read-write, so B gets `* 0 RECENT` for its own APPEND. + ## IDLE During IDLE ([RFC 2177](https://www.rfc-editor.org/rfc/rfc2177)) notifications are sent right away, without waiting for DONE. Continuing the session above, A starts IDLE while B appends a message, sets a flag and expunges another message: @@ -131,6 +183,7 @@ B C: Subject: new B C: B C: Hi B S: * 3 EXISTS +B S: * 0 RECENT B S: B5 OK APPEND Completed B C: B6 STORE 1 +FLAGS (\Seen) B S: * 1 FETCH (FLAGS (\Seen)) @@ -141,6 +194,7 @@ B C: B8 EXPUNGE B S: * 2 EXPUNGE B S: B8 OK EXPUNGE Completed A S: * 3 EXISTS +A S: * 1 RECENT A S: * 1 FETCH (UID 2 FLAGS (\Seen)) A S: * 2 FETCH (UID 3 FLAGS (\Deleted)) A S: * 2 EXPUNGE @@ -157,7 +211,7 @@ Only changes to the selected mailbox are reported. Anything other than `DONE` wh - The first session that selects the mailbox read-write takes the `\Recent` flags. Later sessions see `0 RECENT`. - EXAMINE shows the `\Recent` flags but does not take them ([RFC 3501 section 6.3.2](https://www.rfc-editor.org/rfc/rfc3501#section-6.3.2)). -- A new message is `\Recent` in one session that has the mailbox selected read-write, or, when there is none, for the next session that selects it. +- A new message is `\Recent` in one session that has the mailbox selected read-write, or, when there is none, for the next session that selects it. The sessions that have the mailbox selected get a RECENT response with their new count after its EXISTS response. - STATUS counts every message that is `\Recent` in any session, and does not take the flag. Messages from the storage are `\Recent` only with `"recent": true`, see [Storage](./storage.md#recent). With one such message (some untagged responses left out): @@ -208,6 +262,7 @@ B C: Hi B S: B3 OK APPEND Completed A C: A3 NOOP A S: * 2 EXISTS +A S: * 1 RECENT A S: A3 OK Completed A C: A4 FETCH 1:* (UID) A S: * 1 FETCH (UID 1) diff --git a/docs/docs/reference/custom-plugins.md b/docs/docs/reference/custom-plugins.md index 4735b8b..c993986 100644 --- a/docs/docs/reference/custom-plugins.md +++ b/docs/docs/reference/custom-plugins.md @@ -99,7 +99,7 @@ The server checks the options before the handler runs, so the handler only sees | `astringArguments` | `number[]` | Positions of other astring arguments (user names, identifiers). In these, in mailbox names and in search criteria an atom `NIL` reaches the handler as an atom, not as `null`. | | `searchCriteria` | `number` | Position where SEARCH style criteria start. | | `sequenceSet` | `number` | Position of an argument with message sequence numbers. With `searchCriteria`, used for the RFC 3501 section 5.5 pipelining check, and by UIDONLY. | -| `noExpunge` | `boolean` | EXPUNGE responses are held back while the command runs, as for FETCH, STORE and SEARCH (RFC 3501 section 7.4.1). | +| `noExpunge` | `boolean` | EXPUNGE responses are held back while the command runs, as for FETCH, STORE and SEARCH (RFC 3501 section 7.4.1). The notifications queued before them are still sent. | | `literal8` | `boolean` or `string` | The command accepts `~{n}` literals ([RFC 3516](https://www.rfc-editor.org/rfc/rfc3516)). A string names the capability that allows them. | | `noPipelining` | `boolean` | The command is refused with `BAD` when the client sent more input after it, and so are the commands sent with it (STARTTLS and COMPRESS work this way). | | `appendMessage` | `boolean` | The command takes a message after its mailbox argument, like APPEND. A message literal to a missing mailbox is refused with `NO [TRYCREATE]` before it is sent. | From 99d50c28b6fb0a22a8c29b3d77f29f4437a1a8f9 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:45:39 +0300 Subject: [PATCH 15/17] fix: LIST treats # as a break out character that overrides the reference RFC 3501 section 6.3.8 and RFC 9051 section 6.3.9: with the namespace convention "#" is a break out character "and must be treated as such", so LIST "Work/" "#news.*" lists the #news. namespace instead of looking for "Work/#news.*". CLAUDE.md describes the new notification timing. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 +- README.md | 2 +- docs/docs/extensions/mailboxes.md | 2 +- src/server.ts | 6 ++++-- test/list.test.ts | 15 +++++++++++++++ 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1409b49..0d3c881 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,7 +46,7 @@ Always check RFC text against the real source document at `https://www.rfc-edito Almost everything lives in `src/server.ts`, which defines two classes: - **`IMAPServer`**: holds the shared single-user storage, registered capabilities, command handlers, and the plugin extension arrays. Builds `folderCache` (path to mailbox object) via `indexFolders()` / `processMailbox()` from the namespace-keyed `storage` object (keys like `"INBOX"`, `""`, `"INBOX."`, each with `separator`, `type`, nested `folders`, `messages`). Subscriptions are names in `server.subscriptions`, not part of the mailbox objects (`mailbox.subscribed` is an accessor for it, see `trackSubscription()`), so they survive DELETE and stay with the old name on RENAME; `getSubscriptionTree()` gives LSUB and `LIST (SUBSCRIBED)` the subscribed names, with `\Noselect` stand-ins for names that are not mailboxes. Cross-connection updates go through `server.notify()`, which emits a `notify` event that every connection listens to; `server.notifyFilters` (`filter(connection, notification)`) lets a plugin keep a notification from some connections (ACL does for METADATA). Every notification carries `origin`, the session whose command caused it (`server.activeConnection` while a command handler runs, null for SMTP), and the EXISTS of a new message carries the `message`. -- **`IMAPConnection`**: one per socket. Parses lines and literals with `imap-handler`, queues commands (`scheduleCommand` / `processQueue`, strictly one at a time), refuses commands in the wrong state, with arguments when they take none, and mailbox name arguments that are not valid modified UTF-7 (options from `src/command-states.ts` or `setCommandHandler`, checked centrally in `processQueue`, not per handler), refuses ambiguous pipelining (RFC 3501 5.5), tracks `state` (`"Not Authenticated"`, `"Authenticated"`, `"Selected"`), `username` (set by LOGIN and the AUTHENTICATE plugins) and `selectedMailbox`, and buffers notifications from other connections, flushing them before tagged responses (but not during FETCH/STORE/SEARCH) and before a UID command runs. Messages expunged by another session stay in the session's view (`getSessionMessages()`, marked `ghost`) until the EXPUNGE is reported; README "Multiple sessions" lists the RFC 2180 strategies FETCH, STORE, SEARCH, COPY, MOVE, DELETE and RENAME follow. `connection.inputHandler` lets a plugin (e.g. IDLE, AUTHENTICATE) take over raw input lines, and a plugin can override `connection.canSetSeen()` (FETCH sets `\Seen`) and `connection.canExpunge()` (CLOSE expunges), both `!readOnly` by default, as ACL does. All output goes through `connection.write()` (also raw `+` continuations) and `connection.end()` closes after the output is written, never `connection.socket.write/end`: `connection.transport` is an optional layer between the protocol and the socket with `write`, `receive`, `end(callback)` and `destroy`, which passes data on with `connection.writeRaw()` and `connection.onData()`, so it is always above TLS (COMPRESS uses `src/deflate-layer.ts`, which the mock client, the session helper and the compare tool share). `connection.resetSession()` returns to the Not Authenticated state (UNAUTHENTICATE, RFC 8437), `connection.discardInput()` drops unprocessed input. LITERAL+ and LITERAL- set `server.literalPlus` and `server.nonSyncLiteralLimit`. +- **`IMAPConnection`**: one per socket. Parses lines and literals with `imap-handler`, queues commands (`scheduleCommand` / `processQueue`, strictly one at a time), refuses commands in the wrong state, with arguments when they take none, and mailbox name arguments that are not valid modified UTF-7 (options from `src/command-states.ts` or `setCommandHandler`, checked centrally in `processQueue`, not per handler), refuses ambiguous pipelining (RFC 3501 5.5), tracks `state` (`"Not Authenticated"`, `"Authenticated"`, `"Selected"`), `username` (set by LOGIN and the AUTHENTICATE plugins) and `selectedMailbox`, and buffers notifications from other connections, flushing them before tagged responses and, for commands that refer to messages, before the command runs (`processNotifications(data, beforeCommand)`): new-message EXISTS (followed by RECENT for IMAP4rev1 sessions) and flag updates go out first, EXPUNGE waits during FETCH/STORE/SEARCH (`noExpunge`, RFC 3501 section 7.4.1) together with everything queued after it, and the tagged OK then carries `[EXPUNGEISSUED]`. Messages expunged by another session stay in the session's view (`getSessionMessages()`, marked `ghost`) until the EXPUNGE is reported; README "Multiple sessions" lists the RFC 2180 strategies FETCH, STORE, SEARCH, COPY, MOVE, DELETE and RENAME follow. `connection.inputHandler` lets a plugin (e.g. IDLE, AUTHENTICATE) take over raw input lines, and a plugin can override `connection.canSetSeen()` (FETCH sets `\Seen`) and `connection.canExpunge()` (CLOSE expunges), both `!readOnly` by default, as ACL does. All output goes through `connection.write()` (also raw `+` continuations) and `connection.end()` closes after the output is written, never `connection.socket.write/end`: `connection.transport` is an optional layer between the protocol and the socket with `write`, `receive`, `end(callback)` and `destroy`, which passes data on with `connection.writeRaw()` and `connection.onData()`, so it is always above TLS (COMPRESS uses `src/deflate-layer.ts`, which the mock client, the session helper and the compare tool share). `connection.resetSession()` returns to the Not Authenticated state (UNAUTHENTICATE, RFC 8437), `connection.discardInput()` drops unprocessed input. LITERAL+ and LITERAL- set `server.literalPlus` and `server.nonSyncLiteralLimit`. **Store operations and control API**: `src/store-operations.ts` holds the changes that commands and the control API share: `expungeMessages` (`connection.expungeSpecificMessages` calls it), `notifyFlagChanges`, `changeFlags` (flag changes outside STORE, emits the `flags` event `(mailbox, messages, origin)` that CONDSTORE uses to bump MODSEQ; STORE keeps its own path because of UNCHANGEDSINCE), and `deleteMailbox` (with BYE for other sessions), `renameMailbox`, `subscribeMailbox`, `unsubscribeMailbox`, which the commands of the same name call. They throw `ImapKitError` (`storeError()`, also used by `server.createMailbox` and `server.deleteMailbox`) with a RFC 5530 `code` or `INVALID`, and take the origin from `server.activeConnection`, which `server.withOrigin(origin, fn)` scopes (`processQueue` runs a handler with its session, the control API with null). `src/control.ts` is `server.control` (README "Control API"): it checks every argument and runs each change through `withOrigin(null, ...)`, so sessions see it like a change from another session. The `command` event and the `session` events `login` and `logout` go out from `send()` when the tagged response of the command goes out (`commandCompleted`, comparing with `connection._commandStart`), `select` from SELECT, `unselect` from `connection.closeMailbox()`, `waiting` from `processQueue` when a command takes over input (IDLE, AUTHENTICATE), `open` and `close` from the connection. Plugins add control operations and REST routes with `server.control.register(name, fn, routes)` (ACL, QUOTA, METADATA) and put non-JSON data into `snapshot()` with `server.control.snapshotHandlers` (ACL); `server.control` exists before the plugins load. Plugin data that follows mailbox changes (ACL and annotations on DELETE, ACL inheritance on CREATE through the `created` list of the event, annotations on RENAME INBOX) is handled in `mailbox` event listeners, never in command wrappers, so the control API gets the same behavior. Script rules also have the `quiet` event (`quietFor`, timers in `connection.watchQuiet()`, every input and output calls `connection.touch()`), `chance` (random numbers of `server.script.random`, seeded with `scriptSeed`, see `src/random.ts`) and `chunkDelay: 'tick'`. `src/quirks.ts` holds the quirk presets of the `quirks` option (script rules and plugins to leave out). `server.now()` (the `now` option) is the clock of the dates the server sets. `src/storage-schema.ts` checks the `storage` option in `loadStorage()` (types of the known keys, typos of them; plugin keys stay allowed, add a new plugin storage key to `PLUGIN_KEYS` there) and exports the JSON Schema. `connection.describe()` is the session description of `sessions()` and the events, `connection.isOpen()` tells if output can still be sent. `server.start()` / `server.stop()` are the promise forms of `listen()` / `close()`, `start()` also starts the SMTP listener of the `smtp` option. smtp-server is an optional peer dependency (and a devDependency for the tests): only `src/smtp-listener.ts` imports it, loaded with a dynamic `import()` by `server.loadSmtpListener()` and by the CLI for `--smtpPort`, so `require('imapkit')` never loads it (`test/package.test.ts` checks this). `src/rest.ts` is the REST API (README "REST API", `rest` option, `--rest-port`): a route table over `server.control` on `node:http`, which also generates `GET /v1/openapi.json`. `GET /v1/events` streams server events as Server-Sent Events (a route handler can return `{ stream(res, req) }`). Its access rules are deliberate: loopback by default, a token for any other address, a loopback Host header without a token (DNS rebinding), JSON bodies only and no CORS headers (no cross-origin requests from browsers). diff --git a/README.md b/README.md index 209c532..be71e7f 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,7 @@ The XTOYBIRD commands map to the [control API](#control-api): # Known issues - **anonymous namespaces** are not supported -- **LIST** does not insert a hierarchy delimiter between a reference without one and the mailbox name (RFC 2683 section 3.4.9 recommends it), the two are concatenated as RFC 9051 section 6.3.9 describes, like Dovecot does +- **LIST** does not insert a hierarchy delimiter between a reference without one and the mailbox name (RFC 2683 section 3.4.9 recommends it), the two are concatenated as RFC 9051 section 6.3.9 describes, like Dovecot does. A pattern that starts with the `#` break out character ignores the reference - **CHARSET** values other than US-ASCII and UTF-8 are not supported # Running tests diff --git a/docs/docs/extensions/mailboxes.md b/docs/docs/extensions/mailboxes.md index 93b5aac..8458977 100644 --- a/docs/docs/extensions/mailboxes.md +++ b/docs/docs/extensions/mailboxes.md @@ -41,7 +41,7 @@ Without plugins, LIST and LSUB follow RFC 3501. `\HasChildren` and `\HasNoChildr - DELETE does not unsubscribe, so LSUB keeps listing the name until UNSUBSCRIBE, and a mailbox created again under that name is subscribed. RENAME leaves the subscription with the old name. - SUBSCRIBE refuses names that are not mailboxes, UNSUBSCRIBE accepts any name. - LSUB sends the LIST attributes of an existing mailbox, without `\Noselect`, and `()` for a subscribed name that is no longer a mailbox. -- LIST concatenates the reference and the pattern as they are, without inserting a hierarchy delimiter (RFC 9051 section 6.3.9). +- LIST concatenates the reference and the pattern as they are, without inserting a hierarchy delimiter (RFC 9051 section 6.3.9). A pattern that starts with `#` is a break out character of the namespace convention and ignores the reference: `LIST "Work/" "#news.*"` lists the `#news.` namespace. ```text C: A2 LIST "" "*" diff --git a/src/server.ts b/src/server.ts index 1df2b54..e06531a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1597,8 +1597,10 @@ class IMAPServer extends Stream { // RFC 3501 section 6.3.8 (RFC 9051 section 6.3.9): "An empty ("" string) reference name argument // indicates that the mailbox name is interpreted as by SELECT", and without break out characters // "the canonical form is normally the reference name appended with the mailbox name". The pattern - // matches full mailbox names, also in a personal namespace with a prefix such as "INBOX." - const lookup = (referenceName || '') + match; + // matches full mailbox names, also in a personal namespace with a prefix such as "INBOX.". With the + // namespace convention "#" is a break out character "and must be treated as such" (RFC 3501 section 6.3.8, + // RFC 9051 section 6.3.9): a pattern that starts with it is a name of its own, the reference is ignored + const lookup = match.charAt(0) === '#' ? match : (referenceName || '') + match; // "%" does not match the hierarchy delimiter, which is the one of the namespace a name belongs to const queries = new Map(); diff --git a/test/list.test.ts b/test/list.test.ts index 78580df..412be9f 100644 --- a/test/list.test.ts +++ b/test/list.test.ts @@ -112,6 +112,21 @@ describe('ImapKit tests', () => { }); }); + // RFC 3501 section 6.3.8: with the namespace convention "#" is a break out character "and must be treated as + // such", a mailbox name that starts with it overrides the reference + it('LIST ignores the reference for a name that starts with #', (t, done) => { + const cmds = ['A1 LOGIN testuser testpass', 'A2 LIST "Test/" "#news.*"', 'A3 LIST "Test/" "%"', 'ZZ LOGOUT']; + + ctx.run(cmds, resp => { + resp = resp.toString(); + assert.match(resp, /^\* LIST \(\\HasNoChildren\) "\." "#news\.world"\r\n/m); + assert.match(resp, /^A2 OK/m); + // without a break out character the reference still applies + assert.doesNotMatch(resp.slice(resp.indexOf('A2 OK')), /#news/); + done(); + }); + }); + it('LIST #news namespace', (t, done) => { const cmds = ['A1 LOGIN testuser testpass', 'A2 CAPABILITY', 'A3 LIST "#news." "*"', 'ZZ LOGOUT']; From 8691cffefdf5b4c263bbcaf4cbe641541a1275a8 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:51:34 +0300 Subject: [PATCH 16/17] refactor: simplify the review bug fixes - Unknown AUTHENTICATE mechanisms run through the command queue with a fallback handler, so the central checks (state, script rules, commandChecks) apply as for every command - Reuse isAtom for EXISTS, EXPUNGE and CREATE-SPECIAL-USE checks, the ATOM-CHAR grammar of imap-handler for use-attr, and getNamespace for LIST with an empty mailbox name - One RECENT helper that counts the session's \Recent set, one snapshot lookup per flush, the sequence number check cached per command, one pattern per namespace in matchFolders - One permanent flag list for PERMANENTFLAGS and STORE, one storage date check, one user credentials type, the quirk requires check inside the plugin loader Co-Authored-By: Claude Opus 5.5 --- src/commands/list.ts | 17 ++- src/control.ts | 15 +-- src/load-plugins.ts | 29 ++--- src/plugins/create-special-use.ts | 9 +- src/quirks.ts | 5 +- src/server.ts | 203 ++++++++++++++++-------------- src/storage-schema.ts | 3 - test/storage-schema.test.ts | 6 +- 8 files changed, 149 insertions(+), 138 deletions(-) diff --git a/src/commands/list.ts b/src/commands/list.ts index 92ced26..83fc35c 100644 --- a/src/commands/list.ts +++ b/src/commands/list.ts @@ -35,19 +35,16 @@ export default function listCommand(connection: IMAPConnection, parsed: ParsedCo // MAY be the empty string if the reference is non-rooted or is an empty string." const server = connection.server; const reference: string = parsed.attributes[0].value || ''; - // the namespace of the reference, the one with the longest matching prefix + // the namespace of the reference, the personal one for a reference that is not a valid name let key = server.referenceNamespace; - let prefix = ''; - for (const name of Object.keys(server.storage)) { - const exported = connection.exportMailboxName(name); - if (name !== 'INBOX' && exported.length > prefix.length && reference.substr(0, exported.length) === exported) { - key = name; - prefix = exported; - } + try { + key = server.getNamespace(connection.importMailboxName(reference)) || key; + } catch { + // not a mailbox name, so not in another namespace } - const namespace = key !== false ? server.storage[key] : null; + const namespace = key !== false ? server.storage[key] : undefined; // the root of a name in another namespace is the prefix of that namespace, like "#news." in the RFC example - const root = key !== server.referenceNamespace ? prefix : ''; + const root = key !== false && key !== server.referenceNamespace ? connection.exportMailboxName(key) : ''; if (namespace) { connection.send( { diff --git a/src/control.ts b/src/control.ts index a9bc243..0713b79 100644 --- a/src/control.ts +++ b/src/control.ts @@ -86,19 +86,8 @@ interface NewMessage { internaldate?: Date | string | undefined; } -interface UserOptions { - password?: string | undefined; - xoauth2?: - | { - accessToken?: string | undefined; - /** - * @deprecated kept from hoodiecrow and ignored: access tokens never expire. Change the token with - * `control.updateUser()` to test a client against an expired one - */ - sessionTimeout?: number | undefined; - } - | undefined; -} +/** The credentials of a user, see UserData */ +type UserOptions = Pick; /** What a REST route handler gets, see src/rest.ts */ interface RouteRequest { diff --git a/src/load-plugins.ts b/src/load-plugins.ts index b3ae721..a3f4d63 100644 --- a/src/load-plugins.ts +++ b/src/load-plugins.ts @@ -65,7 +65,8 @@ function loadPlugins(server: IMAPServer, plugins: (string | Plugin)[] | string | } }); - const load = (entry: string | Plugin) => { + // `requiredBy` names the plugin whose `requires` lists this one + const load = (entry: string | Plugin, requiredBy?: string) => { let plugin = entry; if (typeof plugin === 'string') { const name = resolvePlugin(plugin); @@ -84,27 +85,27 @@ function loadPlugins(server: IMAPServer, plugins: (string | Plugin)[] | string | throw new TypeError('Invalid plugin, expecting a plugin name or a function'); } - if (loaded.has(plugin) || excluded.has(plugin)) { + const quirk = excluded.get(plugin); + if (quirk) { + if (requiredBy) { + // the plugin can not work without it (IMAP4rev2 folds in MOVE and UIDPLUS, RFC 9051 Appendix E), so + // advertising it without the required plugin would break its RFC + throw new Error(requiredBy + ' requires ' + entry + ', which the "' + quirk + '" quirk removes'); + } + return; + } + if (loaded.has(plugin)) { return; } loaded.add(plugin); - const requires = ([] as string[]).concat(plugin.requires || []); - requires.forEach(name => { - const required = resolvePlugin(name); - const quirk = required && excluded.get(builtinPlugins[required]); - if (quirk) { - // the plugin can not work without it (IMAP4rev2 folds in MOVE and UIDPLUS, RFC 9051 Appendix E), so - // advertising it without the required plugin would break its RFC - throw new Error((typeof entry === 'string' ? entry.trim() : plugin.name) + ' requires ' + name + ', which the "' + quirk + '" quirk removes'); - } - }); - requires.forEach(load); + const name = typeof entry === 'string' ? entry.trim() : plugin.name; + ([] as string[]).concat(plugin.requires || []).forEach(required => load(required, name)); plugin(server); }; - ([] as (string | Plugin)[]).concat(plugins || []).forEach(load); + ([] as (string | Plugin)[]).concat(plugins || []).forEach(entry => load(entry)); server.emit('pluginsLoaded'); } diff --git a/src/plugins/create-special-use.ts b/src/plugins/create-special-use.ts index d8101be..6c6f3e2 100644 --- a/src/plugins/create-special-use.ts +++ b/src/plugins/create-special-use.ts @@ -1,3 +1,5 @@ +import formalSyntax from 'imap-handler/lib/formal'; +import { isAtom } from '../arguments.js'; import type { Attribute, Callback, CommandHandler, IMAPConnection, IMAPResponse, IMAPServer, Mailbox, ParsedCommand } from '../types.js'; /** @@ -6,9 +8,8 @@ import type { Attribute, Callback, CommandHandler, IMAPConnection, IMAPResponse, * @help option "special-use" */ -// use-attr-ext = "\" atom (RFC 6154 section 6), ATOM-CHAR is any CHAR except atom-specials (RFC 3501 section 9) -// eslint-disable-next-line no-control-regex -const USE_ATTR = /^\\[^\x00-\x20\x7f-\xff(){%*"\\\]]+$/; +// use-attr-ext = "\" atom (RFC 6154 section 6), with the ATOM-CHAR of the imap-handler grammar (RFC 3501 section 9) +const USE_ATTR = new RegExp('^\\\\[' + formalSyntax['ATOM-CHAR']().replace(/[\\\]^-]/g, '\\$&') + ']+$'); export default function createSpecialUsePlugin(server: IMAPServer) { // Register capability @@ -47,7 +48,7 @@ export default function createSpecialUsePlugin(server: IMAPServer) { // RFC 6154 section 6: use-attr-ext = "\" atom, so every entry is an atom that starts with a backslash, // not NIL, a string, a literal, a number or a list const entry = specialUseList[i]; - if (!entry || Array.isArray(entry) || entry.type !== 'ATOM' || typeof entry.value !== 'string' || !USE_ATTR.test(entry.value)) { + if (!isAtom(entry) || !USE_ATTR.test(entry.value)) { connection.send( { tag: parsed.tag, diff --git a/src/quirks.ts b/src/quirks.ts index 05704bd..5733244 100644 --- a/src/quirks.ts +++ b/src/quirks.ts @@ -130,12 +130,13 @@ export function resolveQuirks(names: string[] | string | null | undefined): { ru const rules: ScriptRule[] = []; const removePlugins = new Map(); ([] as string[]).concat(names || []).forEach(name => { - const quirk = Object.hasOwn(quirks, String(name).toLowerCase()) ? quirks[String(name).toLowerCase()] : null; + const key = String(name).toLowerCase(); + const quirk = Object.hasOwn(quirks, key) ? quirks[key] : null; if (!quirk) { throw new Error('Unknown quirk "' + name + '". Available quirks: ' + Object.keys(quirks).join(', ')); } rules.push(...(quirk.rules || [])); - (quirk.removePlugins || []).forEach(plugin => removePlugins.set(plugin, String(name).toLowerCase())); + (quirk.removePlugins || []).forEach(plugin => removePlugins.set(plugin, key)); }); return { rules, removePlugins }; } diff --git a/src/server.ts b/src/server.ts index e06531a..398d751 100644 --- a/src/server.ts +++ b/src/server.ts @@ -13,7 +13,7 @@ import { MONTHS, monthIndex, isDateTime } from './dates.js'; import fetchHandlers from './commands/handlers/fetch.js'; import { hasSequenceSetKey } from './commands/handlers/search.js'; import { isSequenceSet } from './numbers.js'; -import { restoreNilAtoms } from './arguments.js'; +import { restoreNilAtoms, isAtom } from './arguments.js'; import { refuseMissingTarget } from './commands/append.js'; import { DEFAULT_SESSION_TIMEOUT, storeError, expungeMessages, notifyFlagChanges } from './store-operations.js'; import { Control } from './control.js'; @@ -150,7 +150,7 @@ function stateError(command: string, state: string): string { * @return {Boolean} true if the notification must wait while EXPUNGE responses are not allowed */ function isPendingExpunge(notification: Notification): boolean { - return !!notification.mailboxCopy || (!!notification.attributes && (notification.attributes[1] || {}).value === 'EXPUNGE'); + return !!notification.mailboxCopy || isAtom(notification.attributes && notification.attributes[1], 'EXPUNGE'); } /** @@ -160,10 +160,22 @@ function isPendingExpunge(notification: Notification): boolean { * @return {Boolean} true for an EXISTS response */ function isExists(notification?: Notification): boolean { - const name = notification && !notification.command && notification.attributes && notification.attributes[1]; - return !!name && name.type === 'ATOM' && name.value === 'EXISTS'; + return !!notification && !notification.command && isAtom(notification.attributes && notification.attributes[1], 'EXISTS'); } +/** + * Answers AUTHENTICATE with a mechanism no plugin supports: NO, not a syntax error (RFC 3501 section 6.2.2). It runs + * through the command queue, so the state check comes first (AUTHENTICATE is only valid in the Not Authenticated + * state, RFC 3501 section 6.2) + */ +// usesSequenceNumbers() of each running command +const sequenceNumberUse = new WeakMap(); + +const unsupportedMechanism: CommandHandler = (connection, parsed, data, callback) => { + connection.sendStatus(parsed, data, 'NO', 'Unsupported authentication mechanism', false, 'UNKNOWN COMMAND'); + callback(); +}; + /** * Creates a new IMAP server, call `listen()` on it to start accepting connections * @@ -692,8 +704,7 @@ class IMAPServer extends Stream { let seen = 0; let unseen = 0; // flags stay defined in the mailbox once a message had them, see rememberFlags - const permanentFlags = ([] as string[]).concat(mailbox.permanentFlags || []); - (mailbox.knownFlags || []).forEach(flag => this.ensureFlag(permanentFlags, flag)); + const permanentFlags = this.permanentFlagList(mailbox); let recent = 0; // \Recent sets of the sessions that have this mailbox selected @@ -1248,7 +1259,23 @@ class IMAPServer extends Stream { * @return {Boolean} true if the flag is a permanent flag of the mailbox */ isPermanentFlag(mailbox: Mailbox, flag: string): boolean { - return mailbox.allowPermanentFlags || mailbox.permanentFlags.indexOf(flag) >= 0 || (mailbox.knownFlags || []).indexOf(flag) >= 0; + // the lists of permanentFlagList(), without building it: this runs for every flag STORE and APPEND set + return ( + mailbox.allowPermanentFlags || (mailbox.permanentFlags || []).indexOf(flag) >= 0 || (!!mailbox.knownFlags && mailbox.knownFlags.indexOf(flag) >= 0) + ); + } + + /** + * The flags of the PERMANENTFLAGS list (without `\*`): the `permanentFlags` of the mailbox and every flag its + * messages have or had (`knownFlags`, see rememberFlags) + * + * @param {Object} mailbox Mailbox object + * @return {Array} flags, a new list + */ + permanentFlagList(mailbox: Mailbox): string[] { + const list = ([] as string[]).concat(mailbox.permanentFlags || []); + (mailbox.knownFlags || []).forEach(flag => this.ensureFlag(list, flag)); + return list; } /** @@ -1603,51 +1630,46 @@ class IMAPServer extends Stream { const lookup = match.charAt(0) === '#' ? match : (referenceName || '') + match; // "%" does not match the hierarchy delimiter, which is the one of the namespace a name belongs to - const queries = new Map(); - const getQuery = (separator: string, flags = '') => { - const key = separator + '/' + flags; - let query = queries.get(key); - if (!query) { - const pattern = lookup - // escape regex symbols - .replace(/([\\^$+?!.():=[\]{}|,-])/g, '\\$1') - .replace(/[*]/g, '.*') - .replace(/[%]/g, '[^' + separator.replace(/([\\^$+*?!.():=[\]{}|,-])/g, '\\$1') + ']*'); - query = new RegExp('^' + pattern + '$', flags); - queries.set(key, query); - } - return query; - }; + const toRegExp = (separator: string, flags = '') => + new RegExp( + '^' + + lookup + // escape regex symbols + .replace(/([\\^$+?!.():=[\]{}|,-])/g, '\\$1') + .replace(/[*]/g, '.*') + .replace(/[%]/g, '[^' + separator.replace(/([\\^$+*?!.():=[\]{}|,-])/g, '\\$1') + ']*') + + '$', + flags + ); // RFC 3501 section 6.3.8 allows to "hide" otherwise accessible mailboxes from the wildcards: the // mailboxes of namespaces other than the personal one are only matched when the pattern names - // the prefix of the namespace before any wildcard (LIST "" "user.%", LIST "#news." "*") + // the prefix of the namespace before any wildcard (LIST "" "user.%", LIST "#news." "*"). The pattern + // of every namespace that is matched, null for a hidden one const fixedPrefix = lookup.replace(/[*%].*$/, ''); - const visible = new Map(); - const isVisible = (key: string) => { - if (!visible.has(key)) { - const name = toName(key); - visible.set(key, key === this.referenceNamespace || fixedPrefix.substr(0, name.length) === name); - } - return visible.get(key); - }; + const queries = new Map(); + Object.keys(this.storage).forEach(key => { + const name = toName(key); + const visible = key === this.referenceNamespace || fixedPrefix.substr(0, name.length) === name; + queries.set(key, visible ? toRegExp(this.storage[key].separator) : null); + }); const result: ListedMailbox[] = []; // "The special name INBOX is included in the output from LIST, if [...] the uppercase string "INBOX" // matches the interpreted reference and mailbox name arguments", INBOX is case-insensitive - if (source.INBOX && getQuery((this.storage.INBOX && this.storage.INBOX.separator) || '/', 'i').test('INBOX')) { + if (source.INBOX && toRegExp((this.storage.INBOX && this.storage.INBOX.separator) || '/', 'i').test('INBOX')) { result.push(source.INBOX); } Object.keys(source).forEach(path => { const folder = source[path]; - const nsKey = folder.namespace; - if (path === 'INBOX' || nsKey === false || nsKey === 'INBOX' || !this.storage[nsKey] || !isVisible(nsKey)) { + const query = path !== 'INBOX' && folder.namespace !== false && folder.namespace !== 'INBOX' ? queries.get(folder.namespace) : null; + if (!query) { return; } const name = toName(path); - if (getQuery(this.storage[nsKey].separator).test(name) && (folder.flags.indexOf('\\NonExistent') < 0 || name === lookup)) { + if (query.test(name) && (folder.flags.indexOf('\\NonExistent') < 0 || name === lookup)) { result.push(folder); } }); @@ -1742,6 +1764,8 @@ interface QueuedCommand { data: string; /** a command line that a script rule handles instead of the parser and the command handler */ script?: { rule: ScriptRule; context: ScriptContext } | undefined; + /** handler for a command that has no registered one (AUTHENTICATE with an unknown mechanism) */ + handler?: CommandHandler | undefined; } class IMAPConnection { @@ -2923,6 +2947,9 @@ class IMAPConnection { } let queue = this.notificationQueue; + // Flag updates use the sequence numbers this session knows: before the EXPUNGE responses of + // the snapshot are sent, the snapshot, afterwards the current message list + const snapshot = queue.find(notification => notification.mailboxCopy); let held: Notification[] = []; if (data && (this.holdsExpunge(data) || (beforeCommand && this.usesSequenceNumbers(data)))) { const first = queue.findIndex(isPendingExpunge); @@ -2935,9 +2962,6 @@ class IMAPConnection { } } - // Flag updates use the sequence numbers this session knows: before the EXPUNGE responses of - // the snapshot are sent, the snapshot, afterwards the current message list - const snapshot = queue.concat(held).find(notification => notification.mailboxCopy); this.notificationQueue = held; queue = this.prepareNotifications(queue); @@ -2955,7 +2979,8 @@ class IMAPConnection { // a message changed several times is reported once, its FETCH response carries the current flags const reported = new Set(); - // before the snapshot (or with all of it still held back) the session knows the old list + // before the snapshot (or with all of it still held back, then the split at the first pending expunge leaves + // no snapshot in the queue) the session knows the old list const sessionList = (i: number) => (snapshot && (snapshotIndex < 0 || i < snapshotIndex) ? (snapshot.mailboxCopy as Message[]) : current); // the last EXISTS response of a run of EXISTS responses, and if the run announces new messages @@ -2973,34 +2998,41 @@ class IMAPConnection { newMessages = newMessages || !!notification.message; } const next = queue[i + 1]; - if (lastExists >= 0 && !isExists(next) && !(next && next.fetchedMessage)) { - const announced = newMessages; - const existsIndex = lastExists; - lastExists = -1; - newMessages = false; - if (!announced) { - // after expunges, the EXPUNGE responses report the change (RFC 3501 section 7.4.1) - return; - } - // RFC 3501 section 7.3.2: the RECENT response "occurs as a result of a SELECT or EXAMINE command, and if - // the size of the mailbox changes (e.g., new messages)". One for consecutive EXISTS responses that announce - // new messages, with the number of \Recent messages among those the client was told about. It goes after - // the FETCH responses that NOTIFY sends for new messages, RFC 5465 section 5.2: "an unsolicited EXISTS - // response, followed by an unsolicited FETCH response [...] The server MAY also send a RECENT response" - const count = Number(queue[existsIndex].attributes[0]); - const known = sessionList(existsIndex).slice(0, count); - this.send( - { - tag: '*', - notification: true, - attributes: [known.filter(message => this.isRecent(message)).length, { type: 'ATOM', value: 'RECENT' }] - }, - 'RECENT NOTIFICATION' - ); + if (lastExists < 0 || isExists(next) || (next && next.fetchedMessage)) { + // not the end of a run of EXISTS responses (and the FETCH responses NOTIFY sends after them) + return; } + // RFC 3501 section 7.3.2: the RECENT response "occurs as a result of a SELECT or EXAMINE command, and if the + // size of the mailbox changes (e.g., new messages)". One for consecutive EXISTS responses that announce new + // messages, with the number of \Recent messages among those the client was told about. After expunges the + // EXPUNGE responses report the change (RFC 3501 section 7.4.1). It goes after the FETCH responses that NOTIFY + // sends for new messages, RFC 5465 section 5.2: "an unsolicited EXISTS response, followed by an unsolicited + // FETCH response [...] The server MAY also send a RECENT response" + if (newMessages) { + this.sendRecent(Number(queue[lastExists].attributes[0]), getSequence(sessionList(lastExists))); + } + lastExists = -1; + newMessages = false; }); } + /** + * Sends an untagged RECENT response with the number of \Recent messages among the first `count` messages + * + * @param {Number} count Number of messages the client was told about + * @param {Map} sequence Message to the sequence number this session knows it by + */ + sendRecent(count: number, sequence: Map): void { + let recent = 0; + (this.recent || new Set()).forEach(message => { + const seq = sequence.get(message); + if (seq && seq <= count) { + recent++; + } + }); + this.send({ tag: '*', notification: true, attributes: [recent, { type: 'ATOM', value: 'RECENT' }] }, 'RECENT NOTIFICATION'); + } + /** * Compile a command object to a response string and write it to socket. * If the command object has a skipResponse property, the command is @@ -3233,6 +3265,16 @@ class IMAPConnection { * @return {Boolean} true if the command uses message sequence numbers */ usesSequenceNumbers(parsed: CommandContext): boolean { + // the notification checks ask several times per command, the answer does not change + let cached = sequenceNumberUse.get(parsed); + if (cached === undefined) { + cached = this.findSequenceNumbers(parsed); + sequenceNumberUse.set(parsed, cached); + } + return cached; + } + + findSequenceNumbers(parsed: CommandContext): boolean { const { sequenceSet, searchCriteria } = this.server.getCommandOptions(parsed.command); if (sequenceSet !== false) { // other forms of sequence sets, like "$" of SEARCHRES (RFC 5182 section 2.3), do not use numbers @@ -3455,12 +3497,14 @@ class IMAPConnection { return; } - if (this.server.getCommandHandler(parsed.command)) { + // an unknown SASL mechanism is answered by its own handler, after the central checks (state, script rules) + const fallback = /^AUTHENTICATE /i.test(parsed.command) && !this.server.getCommandHandler(parsed.command) ? unsupportedMechanism : undefined; + if (fallback || this.server.getCommandHandler(parsed.command)) { if (this.isAmbiguous(parsed)) { this.sendStatus(parsed, data, 'BAD', 'Commands with message sequence numbers must wait for the completion of earlier commands'); return; } - const element = { parsed, data }; + const element: QueuedCommand = { parsed, data, handler: fallback }; if (scripted) { // processQueue runs it once the script rule released the queue this._commandQueue.unshift(element); @@ -3468,30 +3512,6 @@ class IMAPConnection { this._commandQueue.push(element); } this.processQueue(); - } else if (/^AUTHENTICATE /i.test(parsed.command)) { - // AUTHENTICATE is only valid in the not authenticated state (RFC 3501 section 6.2), which is checked - // first, as for a supported mechanism (processQueue), so the answer does not depend on the mechanism - const states = this.server.getCommandOptions(parsed.command).states; - if (states && states.indexOf(this.state) < 0) { - this.sendStatus(parsed, data, 'BAD', stateError(parsed.command.toUpperCase(), this.state)); - return; - } - // an unsupported mechanism is a NO, not a syntax error (RFC 3501 section 6.2.2) - this.send( - { - tag: parsed.tag, - command: 'NO', - attributes: [ - { - type: 'TEXT', - value: 'Unsupported authentication mechanism' - } - ] - }, - 'UNKNOWN COMMAND', - parsed, - data - ); } else { this.send( { @@ -3689,7 +3709,8 @@ class IMAPConnection { this.state === 'Selected' && (command.startsWith('UID ') || options.noExpunge || options.sequenceSet !== false || options.searchCriteria !== false) ) { - // A command that refers to messages runs on the message list the client was told about. RFC 3501 section 5.2: + // A command that refers to messages runs on the message list the client was told about. Only these: CLOSE + // must not send pending EXPUNGE responses (RFC 3501 section 6.4.2) and SELECT leaves the old mailbox. RFC 3501 section 5.2: // "A server MUST send mailbox size updates automatically if a mailbox size change is observed during the // processing of a command", so the new messages and flag changes of other sessions are reported first, // before the command resolves its sequence set or search criteria. @@ -3705,7 +3726,7 @@ class IMAPConnection { this.server.withOrigin(this, () => { try { const inputHandler = this.inputHandler; - (this.server.getCommandHandler(element.parsed.command) as CommandHandler)(this, element.parsed, element.data, next); + (element.handler || (this.server.getCommandHandler(element.parsed.command) as CommandHandler))(this, element.parsed, element.data, next); if (this.inputHandler && this.inputHandler !== inputHandler) { // the command reads the lines that follow (IDLE, AUTHENTICATE), script rules match them with it this.inputCommand = { tag: element.parsed.tag, command: element.parsed.command }; diff --git a/src/storage-schema.ts b/src/storage-schema.ts index f12e4f2..74e76a1 100644 --- a/src/storage-schema.ts +++ b/src/storage-schema.ts @@ -165,9 +165,6 @@ export function validateStorage(storage: unknown): void { if (value === undefined || value === false || (key === 'SAVEDATE' && value === null)) { continue; } - if (typeof value !== 'string' && !(value instanceof Date)) { - fail(path + '.' + key, 'must be a date-time string or a Date'); - } if (value instanceof Date ? isNaN(value.getTime()) : !isDateTime(value)) { fail(path + '.' + key, 'must be a date-time string like "14-Sep-2013 21:22:28 -0300" or a Date, not ' + JSON.stringify(value)); } diff --git a/test/storage-schema.test.ts b/test/storage-schema.test.ts index ab515c8..c972352 100644 --- a/test/storage-schema.test.ts +++ b/test/storage-schema.test.ts @@ -24,7 +24,11 @@ describe('storage validation', () => { ['a message that is a number', { INBOX: { messages: [5] } }, /messages\[0\]: a message is a string or an object/], ['a UID of 0', { INBOX: { messages: [{ raw: 'x', uid: 0 }] } }, /messages\[0\]\.uid: must be an integer/], ['flags that are not strings', { INBOX: { messages: [{ raw: 'x', flags: [1] }] } }, /\.flags: must be a flag or a list of flags/], - ['a numeric internal date', { INBOX: { messages: [{ raw: 'x', internaldate: 5 }] } }, /\.internaldate: must be a date-time string or a Date/], + [ + 'a numeric internal date', + { INBOX: { messages: [{ raw: 'x', internaldate: 5 }] } }, + /\.internaldate: must be a date-time string like "14-Sep-2013 21:22:28 -0300" or a Date, not 5/ + ], // RFC 3501 section 9: date-time = DQUOTE date-day-fixed "-" date-month "-" date-year SP time SP zone DQUOTE [ 'an RFC 5322 date as the internal date', From f08389ce1e3fe66e2362958893c089fb50ee12c8 Mon Sep 17 00:00:00 2001 From: Andris Reinman Date: Thu, 8 Oct 2026 19:52:40 +0300 Subject: [PATCH 17/17] test: LIST lists the child mailboxes of INBOX Co-Authored-By: Claude Opus 5.5 --- test/list.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/list.test.ts b/test/list.test.ts index 412be9f..3ff8de0 100644 --- a/test/list.test.ts +++ b/test/list.test.ts @@ -265,3 +265,19 @@ describe('LIST with a prefixed personal namespace', () => { }); }); }); + +describe('LIST with child mailboxes of INBOX', () => { + const ctx = setupServer(() => ({ storage: { INBOX: { folders: { Child: {} } }, '': { folders: { Other: {} } } } })); + + it('lists them with "*" and with "INBOX/%"', (t, done) => { + ctx.run(['A1 LOGIN testuser testpass', 'A2 LIST "" "*"', 'A3 LIST "" "INBOX/%"', 'ZZ LOGOUT'], resp => { + resp = resp.toString(); + const a2 = resp.slice(resp.indexOf('A1 OK'), resp.indexOf('A2 OK')); + assert.match(a2, /^\* LIST \(\\HasChildren\) "\/" "?INBOX"?\r$/m); + assert.match(a2, /^\* LIST \(\\HasNoChildren\) "\/" "INBOX\/Child"\r$/m); + assert.match(a2, /^\* LIST \(\\HasNoChildren\) "\/" "?Other"?\r$/m); + assert.match(resp.slice(resp.indexOf('A2 OK')), /^\* LIST \(\\HasNoChildren\) "\/" "INBOX\/Child"\r\nA3 OK/m); + done(); + }); + }); +});