From 3ab3aec7faeadf9577d3e1662d9edea0c026d842 Mon Sep 17 00:00:00 2001 From: LEN5010 <1649211052@qq.com> Date: Sun, 15 Nov 2026 06:09:26 -0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E6=8F=92=E4=BB=B6=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E4=B8=8D=E5=86=8D=E5=85=A8=E8=A1=A8=E6=89=AB?= =?UTF-8?q?=E6=8F=8F=E8=B0=83=E7=94=A8=E8=AE=B0=E5=BD=95=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E6=95=B4=E7=82=B9=E4=BB=BB=E5=8A=A1=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E7=9A=84=E4=BA=8B=E4=BB=B6=E5=BE=AA=E7=8E=AF=E5=8D=A1=E9=A1=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 估算模型请求输入量时,request_estimate 会通过 last_mind_usage 查询本群最近一次聊天调用作为用量参照。这条查询没有可用索引,只能扫描整张 model_calls;scene 列排在 request、response 两个大字段后面,逐行判断时要读溢出页。插件调用记录的 plugin 有值,没有聊天回合的群永远匹配不到,每次都会扫完整张表。线上 8.7 GB 的库在容器内存不足时,每次查询在事件循环上同步阻塞约 20 秒:group_digest 的 hourly-notes 每个整点卡一次,daily-summary 连续调用模型时会连续卡好几分钟。 - 插件发起的模型调用(无论 role)不再查询聊天用量参照,插件 prompt 与聊天会话前缀不会匹配。 - 业务数据库格式升为 3,新增部分索引 model_call_mind_usage ON model_calls(scene,role,id) WHERE plugin IS NULL,聊天请求的这条查询改走索引。upgrade 接受格式 1、2,由补建缺失索引的循环建立新索引。 - 补充测试:插件请求不触发 last_mind_usage;查询计划使用新索引;格式 2 升级建立索引且可重复执行;格式 3 结构指纹。同步 CONTRIBUTING 格式表。 --- CONTRIBUTING.en.md | 4 +- CONTRIBUTING.md | 4 +- src/len_bot/next/maintenance/upgrade.py | 5 ++- src/len_bot/next/models/request.py | 9 +++-- src/len_bot/next/storage/schema.py | 1 + src/len_bot/next/storage/store.py | 2 +- tests/test_format_fingerprint.py | 3 +- tests/test_jargon_migration.py | 25 +++++++++++- tests/test_mind_usage_anchor.py | 54 +++++++++++++++++++++++++ tests/test_next_store_format.py | 6 +-- 10 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 tests/test_mind_usage_anchor.py diff --git a/CONTRIBUTING.en.md b/CONTRIBUTING.en.md index b1dfe27f9..74259a403 100644 --- a/CONTRIBUTING.en.md +++ b/CONTRIBUTING.en.md @@ -109,13 +109,15 @@ Before the first release candidate (RC), all six format and interface numbers we |---|---|---| | Plugin interface | `1` | `INTERFACE` in `src/len_bot/plugin.py` | | Root configuration | `6` | `CONFIG_VERSION` in `src/len_bot/next/config.py` | -| Business database | `2` | `FORMAT_VERSION` in `src/len_bot/next/storage/store.py` | +| Business database | `3` | `FORMAT_VERSION` in `src/len_bot/next/storage/store.py` | | Memory job database | `1` | `FORMAT_VERSION` in `src/len_bot/next/memory/jobs.py` | | Local memory index | `1` | `FORMAT_VERSION` in `src/len_bot/next/memory/local.py` | | Plugin key-value store | `1` | `FORMAT_VERSION` in `src/len_bot/next/plugins/kv.py` | Business database format `2` adds structured jargon judgments and the last evaluated message position. After updating the code, stop the instance and run `uv run --no-sync python -m len_bot.next.maintenance.upgrade run --snapshot /path/to/snapshot`. The migration preserves terms, examples, fixed meanings and disabled decisions. Existing automatic meanings become available after reassessment. +Business database format `3` adds the `model_call_mind_usage` index on `model_calls`, so looking up the usage of a scene's last chat call no longer scans the whole table. The same `upgrade` command builds the index; on a large database the first build reads most of the file and can take several minutes. + The four SQLite databases above record their file type in `PRAGMA application_id` and their format version in `PRAGMA user_version`; the host opens only files matching both expected values. The reference retrieval index (`reference/library.py`) stores derived data only and rebuilds its tables automatically from source documents whenever its schema changes. The structure fingerprint test (`tests/test_format_fingerprint.py`) records one fingerprint per format number for the `CREATE TABLE` and `CREATE INDEX` statements of the four databases above and for the field structure of the root configuration model. It fails when the structure changed but the number did not. From the first RC on, bump the matching number, add the migration in `maintenance/upgrade.py` and record the new fingerprint for the new number in the test; the fingerprint of a released number never changes. Before the RC the numbers stay at `1`, and the fingerprint recorded for `1` is replaced together with the structure change. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0633371ed..66a948858 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,13 +109,15 @@ uv run --no-sync ruff check src/len_bot scripts deploy/package tests |---|---|---| | 插件接口版本 | `1` | `src/len_bot/plugin.py` 的 `INTERFACE` | | 根配置格式 | `6` | `src/len_bot/next/config.py` 的 `CONFIG_VERSION` | -| 业务数据库 | `2` | `src/len_bot/next/storage/store.py` 的 `FORMAT_VERSION` | +| 业务数据库 | `3` | `src/len_bot/next/storage/store.py` 的 `FORMAT_VERSION` | | 记忆处理库 | `1` | `src/len_bot/next/memory/jobs.py` 的 `FORMAT_VERSION` | | 本地记忆索引 | `1` | `src/len_bot/next/memory/local.py` 的 `FORMAT_VERSION` | | 插件键值存储 | `1` | `src/len_bot/next/plugins/kv.py` 的 `FORMAT_VERSION` | 业务数据库格式 `2` 增加黑话结构化判定与最后评估位置。更新代码后,停止实例并执行 `uv run --no-sync python -m len_bot.next.maintenance.upgrade run --snapshot /path/to/snapshot`,迁移保留原有词条、样本、人工固定和禁用决定;旧自动解释在新判定完成后恢复使用。 +业务数据库格式 `3` 为 `model_calls` 增加 `model_call_mind_usage` 索引,聊天请求查询本群最近一次聊天调用的用量时不再扫描整张表。升级时由同一条 `upgrade` 命令建立索引;数据库较大时首次建立要读取大部分文件,可能需要数分钟。 + 上述四个 SQLite 数据库通过 `PRAGMA application_id` 标识文件类型,通过 `PRAGMA user_version` 记录格式编号;宿主只打开这两个值与当前代码一致的数据库文件。资料检索索引库(`reference/library.py`)仅保存可由源文档重建的派生数据,表结构变化时自动重建。 结构指纹测试(`tests/test_format_fingerprint.py`)为上述四个数据库的 `CREATE TABLE` 与 `CREATE INDEX` 语句以及根配置模型的字段结构各记录一个指纹,并按格式编号保存。结构变了而编号没有变时测试失败:自首个 RC 起,须递增对应编号,在 `maintenance/upgrade.py` 补上迁移,并在测试中为新编号记录新指纹,已发布编号的指纹保持不变;RC 之前编号固定为 `1`,随结构改动一并更新 `1` 对应的指纹。 diff --git a/src/len_bot/next/maintenance/upgrade.py b/src/len_bot/next/maintenance/upgrade.py index 9f6365154..2e3f30191 100644 --- a/src/len_bot/next/maintenance/upgrade.py +++ b/src/len_bot/next/maintenance/upgrade.py @@ -122,7 +122,7 @@ def migrate_business_database(path: Path) -> bool: version = db.execute('PRAGMA user_version').fetchone()[0] if application != 0x4C424E31: raise ValueError(f'{path} 不是 LenBot 业务数据库(application_id={application})') - if version not in {1, FORMAT_VERSION}: + if version not in range(1, FORMAT_VERSION + 1): raise ValueError(f'{path} 的格式是 {version},升级支持从格式 1 到 {FORMAT_VERSION}') with closing(sqlite3.connect(':memory:')) as expected: expected.executescript(business_script()) @@ -136,6 +136,9 @@ def migrate_business_database(path: Path) -> bool: if version == 1: db.execute('ALTER TABLE jargon ADD COLUMN judgment TEXT') db.execute('ALTER TABLE jargon ADD COLUMN last_evaluated_seq INTEGER NOT NULL DEFAULT 0') + # Format 3 adds model_call_mind_usage, which the loop below creates. Its first build reads every + # model_calls row, and scene and plugin sit behind the large request and response columns, so on a + # multi-gigabyte database it reads most of the file and can take minutes inside this transaction. for sql in missing: db.execute(sql) db.execute(f'PRAGMA user_version={FORMAT_VERSION}') diff --git a/src/len_bot/next/models/request.py b/src/len_bot/next/models/request.py index 4d292f5b8..4d4ed3022 100644 --- a/src/len_bot/next/models/request.py +++ b/src/len_bot/next/models/request.py @@ -34,7 +34,8 @@ def __call__(self, turn_id: str, role: Literal["mind", "recap", "vision"], def request_estimate(config: SharedConfig, store: Store, model: ChatModel, *, scene: str, role: str, - messages: list[dict], tools: list[dict], output_tokens: int) -> tuple[int, str]: + messages: list[dict], tools: list[dict], output_tokens: int, + plugin: str | None = None) -> tuple[int, str]: """Actual prior input plus changed-material estimates, never cached-token discounts.""" binding = config.binding(role) messages = project_messages(messages, binding.history_policy) @@ -43,7 +44,9 @@ def request_estimate(config: SharedConfig, store: Store, model: ChatModel, *, sc block['type'] == 'image_url' for block in message['content']) for message in messages) if images: return estimated, 'text_estimate_images_unknown' - previous = store.last_mind_usage(scene) if role == 'mind' else None + # The usage anchor is the scene's last chat call. A plugin prompt never extends the chat session's + # prefix, so a plugin request, whatever role it uses, skips the lookup. + previous = store.last_mind_usage(scene) if role == 'mind' and plugin is None else None if previous is not None: request, usage = previous tokens = parse_token_usage(usage) @@ -85,7 +88,7 @@ async def request_model(config: SharedConfig, store: Store, model: ChatModel, .encode()).hexdigest() if input_estimate is None: estimated, method = request_estimate(config, store, model, scene=scene, role=role, - messages=messages, tools=tools, output_tokens=tokens) + messages=messages, tools=tools, output_tokens=tokens, plugin=plugin) else: input_tokens, method = input_estimate estimated = input_tokens + tokens diff --git a/src/len_bot/next/storage/schema.py b/src/len_bot/next/storage/schema.py index 4b3c86eaf..9cb177f9d 100644 --- a/src/len_bot/next/storage/schema.py +++ b/src/len_bot/next/storage/schema.py @@ -101,6 +101,7 @@ CREATE INDEX message_scene_time ON messages(scene,json_extract(body,'$.time'),seq); CREATE INDEX message_sender_time ON messages(scene,json_extract(body,'$.sender.uid'),json_extract(body,'$.time'),seq); CREATE INDEX model_call_expiry ON model_calls(ended,id); +CREATE INDEX model_call_mind_usage ON model_calls(scene,role,id) WHERE plugin IS NULL; CREATE INDEX turns_scene_time ON turns(scene,started); CREATE INDEX turns_expiry ON turns(ended,id); """ diff --git a/src/len_bot/next/storage/store.py b/src/len_bot/next/storage/store.py index c616dd087..37905cb3f 100644 --- a/src/len_bot/next/storage/store.py +++ b/src/len_bot/next/storage/store.py @@ -29,7 +29,7 @@ from ..chat.schedule_store import ScheduleStore -FORMAT_VERSION = 2 +FORMAT_VERSION = 3 FORMAT_UPGRADE = '请停止实例后执行 uv run --no-sync python -m len_bot.next.maintenance.upgrade run --snapshot /path/to/snapshot。' MAX_SEARCH_TERMS = 8 diff --git a/tests/test_format_fingerprint.py b/tests/test_format_fingerprint.py index 2f4cfbf83..542a8bd2e 100644 --- a/tests/test_format_fingerprint.py +++ b/tests/test_format_fingerprint.py @@ -22,7 +22,8 @@ 5: '3e91cc135803e42bba80ff913aa6e6c3188e5255ee56957bbeea752072d5c5d7', 6: '409ab84ed1934d18c71b9182f5de845a4accb15e1217d3855a3e1f02723a5dd2'}, 'business': {1: 'b214769fd5591fa83037f0c29331f6f60853b1fc674f82d06a3bbcbf11e06b80', - 2: 'c721eaa639ab12a1fe45721726959778be77dfb794584cf33b7fa4b8d6edb47f'}, + 2: 'c721eaa639ab12a1fe45721726959778be77dfb794584cf33b7fa4b8d6edb47f', + 3: '4225ec2f881df920e362df37244cbe2f56104378e2f25273b21c094bfad4d2e1'}, 'memory_jobs': {1: 'ee52e1599807d9cc2e5191d2dee75d18c740ca80a6f0960fe7ec4c5479814151'}, 'local_memory_index': {1: '18837e0b06ca13a8d74f2ebe4fc9ea7377c401f15dc26bf31b8d03a107b7b8f1'}, 'plugin_kv': {1: 'bdf97df854e3bb90efd78dfd3e7c342a9729ea1f69931a88e191d54c64cdd56a'}, diff --git a/tests/test_jargon_migration.py b/tests/test_jargon_migration.py index 9e325a05c..ff62e9d2d 100644 --- a/tests/test_jargon_migration.py +++ b/tests/test_jargon_migration.py @@ -5,7 +5,7 @@ from len_bot.next.maintenance.upgrade import migrate_business_database from len_bot.next.learning.jargon_store import JargonStore from len_bot.next.storage.schema import business_script -from len_bot.next.storage.store import Store +from len_bot.next.storage.store import FORMAT_VERSION, Store def test_current_format_upgrade_restores_missing_message_time_indexes(tmp_path): @@ -28,6 +28,29 @@ def test_current_format_upgrade_restores_missing_message_time_indexes(tmp_path): assert all('TEMP B-TREE' not in row[3] for row in plan) + +def test_format_2_upgrade_builds_the_mind_usage_index(tmp_path): + path = tmp_path / 'business.db' + index = 'CREATE INDEX model_call_mind_usage ON model_calls(scene,role,id) WHERE plugin IS NULL;\n' + assert index in business_script() + with closing(sqlite3.connect(path)) as db: + db.executescript('PRAGMA application_id=1279413809; PRAGMA user_version=2;\n' + + business_script().replace(index, '')) + with db: + for request, plugin in (('{"from": "chat"}', None), ('{"from": "plugin"}', 'group_digest')): + db.execute("INSERT INTO model_calls(role,started,ended,request,usage,scene,plugin) " + "VALUES ('mind',1,2,?,'{}','test:group:1',?)", (request, plugin)) + assert migrate_business_database(path) + assert not migrate_business_database(path) + with closing(sqlite3.connect(path)) as db: + assert db.execute('PRAGMA user_version').fetchone()[0] == FORMAT_VERSION + plan = [row[3] for row in db.execute( + "EXPLAIN QUERY PLAN SELECT request FROM model_calls WHERE scene=? AND role='mind' " + "AND plugin IS NULL ORDER BY id DESC LIMIT 1", ('test:group:1',))] + assert any('USING INDEX model_call_mind_usage' in step for step in plan), plan + with Store(path) as store: + assert store.last_mind_usage('test:group:1') == ({'from': 'chat'}, {}) + def test_jargon_upgrade_preserves_manual_controls_and_queues_old_records(tmp_path): path = tmp_path / 'business.db' old = business_script().replace(' judgment TEXT, last_evaluated_seq INTEGER NOT NULL DEFAULT 0,\n', '') diff --git a/tests/test_mind_usage_anchor.py b/tests/test_mind_usage_anchor.py new file mode 100644 index 000000000..c60336ab5 --- /dev/null +++ b/tests/test_mind_usage_anchor.py @@ -0,0 +1,54 @@ +"""The usage anchor of a chat request comes from the scene's last chat call, found by index, never for plugins.""" +from contextlib import closing +import json +import sqlite3 + +import pytest + +from len_bot.next.models.client import ChatModel +from len_bot.next.models.request import request_model +from len_bot.next.storage.schema import business_script +from len_bot.next.storage.store import Store +from test_next_model import recorded_text_response +from test_next_native_models import peer_server +from test_next_persona_permissions import _config, _package + +LAST_MIND_USAGE = ("SELECT request,usage,error,ended FROM model_calls WHERE scene=? AND role='mind' " + "AND plugin IS NULL ORDER BY id DESC LIMIT 1") + + +@pytest.mark.asyncio +async def test_plugin_request_skips_the_chat_usage_anchor(tmp_path, monkeypatch): + config = _config(tmp_path / 'instance', _package(tmp_path, 'plain', tools=[], documents={})) + body = json.dumps(recorded_text_response()).encode() + server = await peer_server([(200, 'application/json', body)] * 3, []) + async with server: + config.models.providers['synthetic'].base_url = f'http://127.0.0.1:{server.sockets[0].getsockname()[1]}/v1' + with Store(config.database) as store: + lookups = [] + original = store.last_mind_usage + + def counted(scene): + lookups.append(scene) + return original(scene) + + monkeypatch.setattr(store, 'last_mind_usage', counted) + chat = [{'role': 'user', 'content': 'hello'}] + async with ChatModel(config.model_settings('mind')) as model: + first = await request_model(config, store, model, chat, [], scene=config.scene, role='mind') + await request_model(config, store, model, [{'role': 'user', 'content': 'hourly notes'}], [], + scene=config.scene, role='mind', plugin='fixture') + await request_model(config, store, model, [*chat, first.message, {'role': 'user', 'content': 'again'}], + [], scene=config.scene, role='mind') + plugins = [row[0] for row in store.db.execute('SELECT plugin FROM model_calls ORDER BY id')] + assert plugins == [None, 'fixture', None] + # Only the two chat requests look up the anchor; the plugin request in between does not. + assert lookups == [config.scene, config.scene] + + +def test_last_mind_usage_searches_its_index(): + with closing(sqlite3.connect(':memory:')) as db: + db.executescript(business_script()) + plan = [row[3] for row in db.execute('EXPLAIN QUERY PLAN ' + LAST_MIND_USAGE, ('onebot:group:80001',))] + assert any('USING INDEX model_call_mind_usage' in step for step in plan), plan + assert not any(step.startswith('SCAN') or 'TEMP B-TREE' in step for step in plan), plan diff --git a/tests/test_next_store_format.py b/tests/test_next_store_format.py index 293e9a536..f0718bb12 100644 --- a/tests/test_next_store_format.py +++ b/tests/test_next_store_format.py @@ -19,8 +19,8 @@ def test_a_newly_created_database_holds_every_table_and_opens_again(tmp_path): @pytest.mark.parametrize('application_id,version,message', [ - (0x4C424E31, 7, '格式是 7,当前版本使用格式 2。请停止实例后执行'), - (0x4C424E31, FORMAT_VERSION, '缺少数据表 .* 等 .* 张,属于格式 2 的早期结构。请停止实例后执行'), + (0x4C424E31, 7, f'格式是 7,当前版本使用格式 {FORMAT_VERSION}。请停止实例后执行'), + (0x4C424E31, FORMAT_VERSION, f'缺少数据表 .* 等 .* 张,属于格式 {FORMAT_VERSION} 的早期结构。请停止实例后执行'), (0, 0, '带有数据表但没有 LenBot 的格式标识(application_id=0).*请停止实例后执行'), ]) def test_another_format_is_refused_with_the_way_forward(tmp_path, application_id, version, message): @@ -35,5 +35,5 @@ def test_one_missing_table_is_named(tmp_path): Store(path).__exit__() with sqlite3.connect(path) as db: db.execute('DROP TABLE usage') - with pytest.raises(ValueError, match='缺少数据表 usage,属于格式 2 的早期结构'): + with pytest.raises(ValueError, match=f'缺少数据表 usage,属于格式 {FORMAT_VERSION} 的早期结构'): Store(path)