diff --git a/docs/issue-166-validation.md b/docs/issue-166-validation.md new file mode 100644 index 00000000..0864b085 --- /dev/null +++ b/docs/issue-166-validation.md @@ -0,0 +1,124 @@ +# Issue #166: offline decryption validation + +Validated on 2026-09-28 against upstream `1b516cf` plus this fix, on macOS arm64 +(macOS 26.3.1, Python 3.11.15). No private databases, keys, account identifiers, +message content, or raw application logs are included in this report. + +## Real WeChat data + +A stable, private copy of an existing local account was used. The project's +normal scanner selected **20 business databases**; search indexes and key-info +files were excluded by the existing database filter. Main files plus sidecars +copied for these databases totalled **1,332,165,592 bytes**. + +No original WeChat files were modified. Each database/sidecar pair was copied +only when its before/after size, modification time and inode were unchanged. +Input-copy SHA-256 checksums were also unchanged after each test. Runtime output +and key persistence used a separate private temporary directory. + +| Check | Result | +| --- | --- | +| Production `GET /api/decrypt_stream` router through FastAPI TestClient | HTTP 200; completed; 20 successful, 0 failed | +| Production `POST /api/decrypt` router over localhost HTTP | HTTP 200; completed; 20 successful, 0 failed | +| Key persistence through the actual router | Successful on both routes | +| Independent `PRAGMA integrity_check` on every output | 20/20 passed after each route | +| `session.db` | Authenticated, decrypted and passed integrity checking; 7 tables | +| WAL replay observed | 1 database, 4 committed frames, 3 applied pages | +| Production chat sessions route with `source=decrypted` | HTTP 200; returned 5 sessions | +| Production chat messages route with `source=decrypted` | HTTP 200; returned 5 messages | + +The other nonempty WAL files did not contain current committed frames selected +by recovery; file size alone does not establish outstanding transactions. The +unmodified real account did not require index rebuilding. Its successful export +therefore does **not** independently reproduce the reporter's original index +corruption. That failure and recovery are covered by the encrypted regression +fixtures below. + +A second local account was discovered but not decrypted because none of its +stored keys passed cross-database authentication. This is an untested account, +not a successful test or a decryption regression. + +## Desktop startup follow-up + +The original expired-runtime blocker was resolved upstream in `1b516cf`, merged +into this branch. The new official macOS source runtime is +`macos-source-runtime-20260928-1790594550`, with expiry +`2026-11-12T11:22:30Z`. The standard `npm run dev` entry point downloaded it, +verified its pinned artifacts, accepted the `source-public` profile, and launched +Nuxt and Electron. A subsequent launch verified and reused the cache. No expiry, +signature or native-runtime checks were bypassed. + +The first launch exceeded the desktop's default 30-second readiness timeout +while setting up a fresh Python environment. After dependencies were installed, +a retry with a longer desktop startup timeout exposed a separate local wait: +the native broker remained inside macOS `SecItemCopyMatching`, called by +`load_or_create_device_identity`. The backend did not reach its health endpoint +before the broker startup timeout. A process sample established this wait; +it is not evidence that the renewed runtime is expired or incompatible. + +The user completed the macOS keychain prompt and the application's first-use +agreement. The standard Electron-launched backend then returned HTTP 200 with +`status=healthy`, and the actual Electron window displayed the application and +existing chat data. + +### Issue-specific desktop backend regression + +The existing encrypted regression fixture generator was reused for two +snapshots with the exact named-index entry-count error from #166. The same +input bytes were tested against the unfixed code and the running desktop +backend. The baseline checkout is `b70da36`; its decrypt implementation, +SQLite diagnostics and decrypt router are unchanged through upstream `1b516cf`. + +| Same-input comparison | Unfixed code | Fixed desktop backend | +| --- | --- | --- | +| Page authentication | 3/3 pages, no HMAC warnings | Authenticated | +| Named-index mismatch without WAL | `wrong # of entries in index SessionUnreadListTable_1_NameId_CreateTime`; session fails | Rebuilds only the affected index; full integrity check passes | +| Same mismatch with a correcting committed WAL page | Same index failure; WAL omitted | 1 committed frame replayed; full integrity check passes without REINDEX | +| Key persistence | Rejected because session did not verify | Saved after session/message authentication | +| Records in repaired session fixture | Output rejected | Both expected records present | + +Each fixture included an authenticated message database. Both POST requests to +the actual desktop backend completed with 2 successes and 0 failures. The WAL +case also passed the actual SSE endpoint used by the decrypt page, ending in +`complete`, 2 successes, 0 failures and `db_key_persisted=true`. Input hashes +were unchanged. These are encrypted SQLite fixtures, not the reporter's +unavailable 734-page original database. + +The already prepared real-account snapshot was then tested through this same +Electron-launched backend: **20/20 successful, 0 failures, 20/20 independent +integrity checks passed, key persisted, and source-copy hashes unchanged**. +The POST request took 6.82 seconds. Explicit `source=decrypted` chat requests +returned 5 sessions and 5 messages from these outputs. The visible Electron +chat page was also inspected; private content and screenshots are not published. + +This is targeted regression testing against the complete desktop application's +backend plus a visible desktop startup/chat check. It does not claim automated +click-through of the entire first-use/key-capture/decrypt wizard. The earlier +minimal-router HTTP tests remain supplementary evidence. No new account-key +capture or modification of original WeChat databases was needed. + +The frontend production static build (`npm run generate`) passed, generating +34 routes. After merging `1b516cf`, all 76 focused Python tests and 3 frontend +feedback tests passed again. + +## Regression tests + +- 76 focused Python tests passed across offline WAL recovery, key modes, SSE, + decryption UI contracts, decrypted fallback and account validation. +- 3 frontend feedback tests passed (`node --test frontend/tests/decrypt-feedback.test.mjs`). +- `git diff --check` passed. + +`tests/test_offline_wal_recovery.py` uses real SQLite pages and encrypted +fixtures, without mocking integrity diagnostics. Coverage includes: + +- The same named-index entry-count error reported in #166, with all encrypted + pages authenticating; recovery both from a committed WAL page and by rebuilding + only the affected indexes on a disposable output. +- Little- and big-endian WAL checksums, repeated page writes, committed versus + uncommitted/partial tails, recycled salts, growth, truncation and VACUUM. +- Invalid WAL headers, frame checksums, page HMACs and missing growth pages. +- Refusing table corruption, changing sources and source/output aliases. +- Preserving the prior output when validation or atomic publication fails, and + preventing an old output WAL from overriding the new snapshot. +- Keeping key authentication independent of output integrity without accepting + a key that authenticates only one of the required database roles. diff --git a/frontend/pages/decrypt.vue b/frontend/pages/decrypt.vue index 9edf61aa..5d799ab3 100644 --- a/frontend/pages/decrypt.vue +++ b/frontend/pages/decrypt.vue @@ -1026,6 +1026,16 @@ +
+ 数据库解密详情（{{ databaseFailures.length }} 个未完成） + +
+ @@ -1117,7 +1127,7 @@ const { const loading = ref(false) const error = ref('') const warning = ref('') // 警告，用于密钥提示 -const warningIsError = computed(() => /失败|错误|异常|中断/.test(String(warning.value || ''))) +const warningIsError = computed(() => !String(warning.value || '').startsWith('解密部分成功：') && /失败|错误|异常|中断/.test(String(warning.value || ''))) const currentStep = ref(0) const mediaAccount = ref('') const activeKeyAccount = ref('') @@ -1144,7 +1154,7 @@ const imageKeyMemoryScanNote = computed(() => String( || platformCapabilities.value?.image_key_memory_scan_note || '图片密钥扫描原生资源缺失或安装不完整，请重新安装完整发行包。' )) -const DB_KEY_PERSISTENCE_WARNING = '数据库密钥未通过完整实时库校验或无法安全保存；请重新获取并确认主要数据库解密成功，仍失败请检查数据目录权限。' +const DB_KEY_PERSISTENCE_WARNING = '数据库密钥未通过 session/message 跨库认证或保存失败；请查看失败详情，确认账号密钥及数据目录写入权限。' const guideDialog = reactive({ open: false, eyebrow: '操作提示', @@ -2012,8 +2022,15 @@ const cancelDbKeyAcquisition = () => { } const showDbKeyPersistenceWarning = (result) => { + if (result?.failure_count > 0 && result?.success_count > 0) { + warning.value = `解密部分成功：${result.success_count}/${result.total_databases} 个数据库可用；失败文件请查看解密详情，已成功的数据可继续使用。` + } + const repaired = Object.values(result?.account_results || {}).flatMap(account => + Object.values(account.db_diagnostics || {}).filter(db => db.success && db.index_repair?.success).map(db => db.db_name) + ) + if (repaired.length) warning.value = [warning.value, `已在解密输出副本中重建索引并通过完整性检查：${repaired.join('、')}。`].filter(Boolean).join(' ') if (result?.db_key_persisted !== false) return - warning.value = DB_KEY_PERSISTENCE_WARNING + warning.value = [warning.value, DB_KEY_PERSISTENCE_WARNING].filter(Boolean).join(" ") logDecryptDebug('decrypt:db-key-persistence-warning', { error_count: Array.isArray(result?.db_key_persistence_errors) ? result.db_key_persistence_errors.length @@ -2525,6 +2542,12 @@ const getMediaDecryptConcurrency = () => { // 解密结果存储 const decryptResult = ref(null) +const databaseFailures = computed(() => Object.entries(decryptResult.value?.account_results || {}).flatMap(([account, result]) => + Object.entries(result.db_diagnostics || {}).filter(([, db]) => db.success === false).map(([name, db]) => ({ + id: `${account}/${name}`, name, authenticated: db.key_authenticated === true, + error: db.error === 'key_mismatch' ? '密钥与此数据库不匹配' : (db.error || '数据库完整性检查未通过') + })) +)) // 验证表单 const validateForm = () => { @@ -2657,6 +2680,7 @@ const handleDecrypt = async () => { db_key_length: String(formData.key || '').trim().length }) loading.value = true + decryptResult.value = null error.value = '' warning.value = '' diff --git a/frontend/tests/decrypt-feedback.test.mjs b/frontend/tests/decrypt-feedback.test.mjs new file mode 100644 index 00000000..b8f9a2f0 --- /dev/null +++ b/frontend/tests/decrypt-feedback.test.mjs @@ -0,0 +1,29 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import vm from 'node:vm' + +const source = readFileSync(new URL('../pages/decrypt.vue', import.meta.url), 'utf8') +function feedback(result) { + const start = source.indexOf('const showDbKeyPersistenceWarning =') + const end = source.indexOf('\nconst runMacosLldbFallback', start) + const context = { result, warning: { value: '' }, DB_KEY_PERSISTENCE_WARNING: 'KEY_SAVE_WARNING', logDecryptDebug: () => {} } + vm.runInNewContext(`${source.slice(start, end)}\nshowDbKeyPersistenceWarning(result)`, context) + return context.warning.value +} + +test('partial success with an authenticated key does not ask for key recapture', () => { + const message = feedback({ success_count:27, failure_count:1, total_databases:28, db_key_persisted:true }) + assert.match(message, /解密部分成功：27\/28/) + assert.doesNotMatch(message, /KEY_SAVE_WARNING/) +}) + +test('recovered indexes are disclosed and key saving failures stay visible', () => { + const message = feedback({ db_key_persisted:false, account_results:{ account:{ db_diagnostics:{ session:{ db_name:'session.db', success:true, index_repair:{ success:true } } } } } }) + assert.match(message, /重建索引.*session\.db/) + assert.match(message, /KEY_SAVE_WARNING/) +}) + +test('normal success has no warning', () => { + assert.equal(feedback({ db_key_persisted:true, success_count:28, failure_count:0 }), '') +}) diff --git a/src/wechat_decrypt_tool/routers/decrypt.py b/src/wechat_decrypt_tool/routers/decrypt.py index afb12f4c..59553967 100644 --- a/src/wechat_decrypt_tool/routers/decrypt.py +++ b/src/wechat_decrypt_tool/routers/decrypt.py @@ -211,12 +211,17 @@ def _database_diagnostic_role(diagnostic: dict[str, Any]) -> str: def _database_diagnostic_verified(diagnostic: dict[str, Any]) -> bool: return bool( - diagnostic.get("success") is True - and not bool(diagnostic.get("copied_as_sqlite")) + not bool(diagnostic.get("copied_as_sqlite")) and str(diagnostic.get("key_mode") or "").strip() in {"raw_enc_key", "sqlcipher_passphrase"} - and int(diagnostic.get("failed_pages") or 0) == 0 - and str(diagnostic.get("diagnostic_status") or "").strip() == "ok" + # Key authentication is independent of WAL/page/index integrity. + # Retain compatibility with results produced before key_authenticated. + and (diagnostic.get("key_authenticated") is True or ( + "key_authenticated" not in diagnostic + and diagnostic.get("success") is True + and int(diagnostic.get("failed_pages") or 0) == 0 + and str(diagnostic.get("diagnostic_status") or "").strip() == "ok" + )) ) diff --git a/src/wechat_decrypt_tool/sqlite_diagnostics.py b/src/wechat_decrypt_tool/sqlite_diagnostics.py index 17005962..872d136b 100644 --- a/src/wechat_decrypt_tool/sqlite_diagnostics.py +++ b/src/wechat_decrypt_tool/sqlite_diagnostics.py @@ -182,3 +182,41 @@ def format_sqlite_diagnostics(diagnostics: Mapping[str, Any]) -> str: continue compact[str(key)] = value return json.dumps(compact, ensure_ascii=False, sort_keys=True) + + +def repair_sqlite_indexes(path: str | Path) -> dict[str, Any]: + """Repair index-only damage on a disposable output, never on the source. + + Do not attempt salvage of table/page corruption. A full integrity check must + pass after rebuilding every affected named index before output is accepted. + """ + import re + + result: dict[str, Any] = {"attempted": False, "success": False} + conn = None + try: + conn = sqlite3.connect(str(path)) + conn.execute("PRAGMA trusted_schema=OFF") + errors = [str(row[0]) for row in conn.execute("PRAGMA integrity_check(1000)")] + if not errors or errors == ["ok"] or len(errors) >= 1000: + return result + indexes = set() + for error in errors: + match = re.fullmatch(r"(?:wrong # of entries in index|row \d+ missing from index) (.+)", error) + if match is None: + return result + indexes.add(match.group(1)) + names = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='index'")} + if not indexes or not indexes <= names: + return result + result.update(attempted=True, indexes=sorted(indexes), original_errors=errors[:5]) + for name in sorted(indexes): + conn.execute(f"REINDEX {_quote_ident(name)}") + conn.commit() + result['success'] = conn.execute("PRAGMA integrity_check").fetchall() == [('ok',)] + except sqlite3.Error as exc: + result['error'] = _clean_error(exc) + finally: + if conn is not None: + conn.close() + return result diff --git a/src/wechat_decrypt_tool/sqlite_wal.py b/src/wechat_decrypt_tool/sqlite_wal.py new file mode 100644 index 00000000..5f65281d --- /dev/null +++ b/src/wechat_decrypt_tool/sqlite_wal.py @@ -0,0 +1,89 @@ +"""Recover a stable SQLite/SQLCipher WAL snapshot without touching source files. + +Format: https://sqlite.org/fileformat2.html#walformat. SQLCipher computes WAL +checksums over the encrypted page payload, before writing the frame to disk. +""" +from __future__ import annotations + +import struct +from collections.abc import Callable + + +def _checksum(data: bytes, endian: str, state: tuple[int, int] = (0, 0)) -> tuple[int, int]: + a, b = state + for x, y in struct.iter_unpack(endian + 'II', data): + a = (a + x + b) & 0xffffffff + b = (b + y + a) & 0xffffffff + return a, b + + +def merge_wal_snapshot( + database: bytes, + wal: bytes, + page_size: int, + *, + verify_page: Callable[[bytes, int], bool] | None = None, +) -> tuple[bytes, dict]: + """Apply only checksum-valid committed frames, respecting database truncation. + + A salt mismatch marks the recycled tail of a WAL. Complete frames with a + bad checksum are rejected rather than silently exporting an older snapshot. + Incomplete/uncommitted trailing frames never become part of the output. + """ + info = {'wal_bytes': len(wal), 'committed_frames': 0, 'applied_pages': 0, + 'ignored_tail_bytes': 0, 'database_pages': len(database) // page_size} + if not wal: + return database, info + if len(wal) < 32: + raise ValueError('WAL 文件头不完整，请退出微信后重试') + magic, version, wal_page_size = struct.unpack('>III', wal[:12]) + if magic not in (0x377f0682, 0x377f0683) or version != 3007000: + raise ValueError('WAL 文件头或版本无效') + if wal_page_size != page_size or len(database) % page_size: + raise ValueError('WAL 与数据库页大小不匹配，或主数据库页不完整') + endian = '<' if magic == 0x377f0682 else '>' + state = _checksum(wal[:24], endian) + if state != struct.unpack('>II', wal[24:32]): + raise ValueError('WAL 文件头校验失败') + salt = wal[16:24] + frame_size = 24 + page_size + pending: dict[int, bytes] = {} + committed: dict[int, bytes] = {} + base_pages = len(database) // page_size + size = base_pages + commit_end = 32 + for offset in range(32, len(wal) - frame_size + 1, frame_size): + header = wal[offset:offset + 24] + if header[8:16] != salt: + break # old frames left after WAL reset / preallocated zero tail + pgno, db_size = struct.unpack('>II', header[:8]) + if pgno == 0 or pgno > 0xfffffffe: + raise ValueError('WAL 页号无效') + page = wal[offset + 24:offset + frame_size] + state = _checksum(page, endian, _checksum(header[:8], endian, state)) + if state != struct.unpack('>II', header[16:24]): + raise ValueError('WAL 帧校验失败，请重新获取稳定的数据库副本') + pending[pgno] = page + if db_size: + committed.update(pending) + pending.clear() + committed = {n: p for n, p in committed.items() if n <= db_size} + base_pages = min(base_pages, db_size) + # Check before allocating: malformed sizes must not cause huge allocations. + extension = sum(n > base_pages for n in committed) + if db_size - base_pages != extension: + raise ValueError('WAL 提交缺少数据库扩展页') + size = db_size + commit_end = offset + frame_size + info['committed_frames'] = (commit_end - 32) // frame_size + info.update(applied_pages=len(committed), ignored_tail_bytes=len(wal) - commit_end, + database_pages=size) + if not info['committed_frames']: + return database, info + output = bytearray(database[:base_pages * page_size]) + output.extend(b'\0' * (size * page_size - len(output))) + for pgno, page in committed.items(): + if verify_page is not None and not verify_page(page, pgno): + raise ValueError(f'WAL 第 {pgno} 页 HMAC 校验失败') + output[(pgno - 1) * page_size:pgno * page_size] = page + return bytes(output), info diff --git a/src/wechat_decrypt_tool/wechat_decrypt.py b/src/wechat_decrypt_tool/wechat_decrypt.py index b7438746..510fce7e 100644 --- a/src/wechat_decrypt_tool/wechat_decrypt.py +++ b/src/wechat_decrypt_tool/wechat_decrypt.py @@ -15,6 +15,7 @@ import json import struct import time +import tempfile from pathlib import Path from typing import Any @@ -23,7 +24,8 @@ from .app_paths import get_output_databases_dir from .database_filters import should_skip_source_database -from .sqlite_diagnostics import collect_sqlite_diagnostics, sqlite_diagnostics_status +from .sqlite_diagnostics import collect_sqlite_diagnostics, sqlite_diagnostics_status, repair_sqlite_indexes +from .sqlite_wal import merge_wal_snapshot # 注意：不再支持默认密钥，所有密钥必须通过参数传入 @@ -101,6 +103,7 @@ def _safe_file_snapshot(path: str | Path) -> dict[str, Any]: { "exists": True, "size": int(st.st_size), + "inode": int(st.st_ino), "mtime_ns": int(getattr(st, "st_mtime_ns", int(st.st_mtime * 1_000_000_000))), } ) @@ -115,6 +118,7 @@ def _safe_file_snapshot(path: str | Path) -> dict[str, Any]: siblings[suffix] = { "exists": True, "size": int(st.st_size), + "inode": int(st.st_ino), "mtime_ns": int(getattr(st, "st_mtime_ns", int(st.st_mtime * 1_000_000_000))), } except FileNotFoundError: @@ -483,7 +487,7 @@ def build_decrypt_summary_message(*, success_count: int, total_databases: int, d if success_count <= 0: if diagnostic_warning_count > 0: - return "解密失败：数据库校验未通过，密钥可能不匹配当前账号。" + return "解密失败：数据库校验未通过，请查看各文件的密钥认证、WAL 或完整性诊断。" return "解密失败：未能成功解密任何数据库。" if success_count < total_databases: @@ -721,6 +725,9 @@ def decrypt_database(self, db_path: str, output_path: str) -> bool: "source_changed_during_read": False, "read_ms": 0, "key_mode": "", + "key_authenticated": False, + "wal": {}, + "index_repair": {}, "input_layout": {}, "expected_output_size": 0, "output_header_debug": {}, @@ -729,6 +736,20 @@ def decrypt_database(self, db_path: str, output_path: str) -> bool: "error": "", } self.last_result = result + working_output: Path | None = None + + def _write_output(data: bytes) -> None: + nonlocal working_output + with tempfile.NamedTemporaryFile(dir=Path(output_path).parent, prefix=".decrypt-", suffix=".db", delete=False) as stream: + working_output = Path(stream.name) + # A materialized snapshot is standalone; do not leave WAL mode + # enabled and let readers create new sidecars beside the export. + if data.startswith(SQLITE_HEADER) and data[18:20] == b"\x02\x02": + stream.write(memoryview(data)[:18]) + stream.write(b"\x01\x01") + stream.write(memoryview(data)[20:]) + else: + stream.write(data) def _append_failed_page(page_num: int, reason: str, error: str = "") -> None: result["failure_reasons"][reason] = int(result["failure_reasons"].get(reason) or 0) + 1 @@ -754,17 +775,26 @@ def _finalize(success: bool, error: str = "") -> bool: if error: result["error"] = " ".join(str(error).split()).strip() - output_file = Path(str(output_path)) - if output_file.exists(): + output_file = working_output + if output_file is not None and output_file.exists(): try: result["output_size"] = int(output_file.stat().st_size) except Exception: pass diagnostics = collect_sqlite_diagnostics(output_file, quick_check=True) + if (normalized_success and diagnostics.get("quick_check_ok") is False + and result["key_authenticated"] and result["failed_pages"] == 0 + and result["hmac_warning_pages"] == 0): + result["index_repair"] = repair_sqlite_indexes(output_file) + if result["index_repair"].get("success"): + diagnostics = collect_sqlite_diagnostics(output_file, quick_check=True) + result["output_size"] = output_file.stat().st_size + diagnostics["path"] = str(output_path) result["diagnostics"] = diagnostics result["diagnostic_status"] = sqlite_diagnostics_status(diagnostics) result["output_header_debug"] = _read_plain_sqlite_header_debug(output_file) + result["output_header_debug"]["path"] = str(output_path) if normalized_success: failure_message = _build_decrypt_failure_message(result) @@ -773,13 +803,21 @@ def _finalize(success: bool, error: str = "") -> bool: result["success"] = False if not result["error"]: result["error"] = failure_message - if output_file.exists(): + if output_file is not None and output_file.exists(): try: output_file.unlink() except Exception as exc: logger.warning("删除无效解密输出失败: %s, 错误: %s", output_file, exc) + if normalized_success and output_file is not None: + # Never let an old output WAL override a freshly decrypted main file. + if any(Path(str(output_path) + suffix).exists() for suffix in ("-wal", "-shm", "-journal")): + raise ValueError("输出数据库正在使用或遗留日志文件，请关闭读取连接后重试") + os.replace(output_file, output_path) payload = { + "key_authenticated": result["key_authenticated"], + "wal": result["wal"], + "index_repair": result["index_repair"], "db_name": result["db_name"], "db_path": result["db_path"], "output_path": result["output_path"], @@ -823,6 +861,10 @@ def _finalize(success: bool, error: str = "") -> bool: logger.info(f"开始解密数据库: {db_path}") try: + if Path(db_path).resolve() == Path(output_path).resolve() or ( + Path(output_path).exists() and os.path.samefile(db_path, output_path) + ): + raise ValueError("解密输出不能覆盖源数据库") source_snapshot_before = _safe_file_snapshot(db_path) result["source_snapshot_before"] = source_snapshot_before logger.info( @@ -840,6 +882,18 @@ def _finalize(success: bool, error: str = "") -> bool: read_t0 = time.perf_counter() with open(db_path, 'rb') as f: encrypted_data = f.read() + wal_path = Path(str(db_path) + "-wal") + try: + wal_data = wal_path.read_bytes() + except FileNotFoundError: + wal_data = b"" + journal_path = Path(str(db_path) + "-journal") + try: + with journal_path.open('rb') as journal: + if journal.read(8) == bytes.fromhex('d9d505f920a163d7'): + raise ValueError("源数据库存在待恢复的回滚日志，请先通过微信完成恢复") + except FileNotFoundError: + pass result["read_ms"] = round((time.perf_counter() - read_t0) * 1000.0, 1) source_snapshot_after = _safe_file_snapshot(db_path) @@ -848,7 +902,13 @@ def _finalize(success: bool, error: str = "") -> bool: after_size = int(source_snapshot_after.get("size") or 0) before_mtime = int(source_snapshot_before.get("mtime_ns") or 0) after_mtime = int(source_snapshot_after.get("mtime_ns") or 0) - source_changed = bool(before_size != after_size or before_mtime != after_mtime) + source_changed = bool( + before_size != after_size or before_mtime != after_mtime + or source_snapshot_before.get("inode") != source_snapshot_after.get("inode") + or any(source_snapshot_before.get("siblings", {}).get(suffix) != + source_snapshot_after.get("siblings", {}).get(suffix) + for suffix in ("-wal", "-journal")) + ) result["source_changed_during_read"] = source_changed logger.info( "[decrypt.pipeline] source_snapshot_after %s", @@ -873,6 +933,9 @@ def _finalize(success: bool, error: str = "") -> bool: after_mtime, ) + if source_changed: + return _finalize(False, "源数据库或 WAL 在读取期间发生变化，请退出微信后重试") + logger.info(f"读取文件大小: {len(encrypted_data)} bytes") result["input_size"] = int(len(encrypted_data)) result["input_layout"] = { @@ -906,8 +969,17 @@ def _finalize(success: bool, error: str = "") -> bool: # 检查是否已经是解密的数据库 if encrypted_data.startswith(SQLITE_HEADER): logger.info(f"文件已是SQLite格式，直接复制: {db_path}") - with open(output_path, 'wb') as f: - f.write(encrypted_data) + plain_page_size = int.from_bytes(encrypted_data[16:18], 'big') + if plain_page_size == 1: + plain_page_size = 65536 + if plain_page_size < 512 or plain_page_size > 65536 or plain_page_size & (plain_page_size - 1): + raise ValueError("SQLite 页大小无效") + encrypted_data, result["wal"] = merge_wal_snapshot(encrypted_data, wal_data, plain_page_size) + if result["wal"]["committed_frames"]: + merged = bytearray(encrypted_data) + merged[28:32] = result["wal"]["database_pages"].to_bytes(4, "big") + encrypted_data = bytes(merged) + _write_output(encrypted_data) result["copied_as_sqlite"] = True return _finalize(True) @@ -922,6 +994,7 @@ def _finalize(success: bool, error: str = "") -> bool: enc_key, mac_key, key_mode = resolved_key_material result["key_mode"] = key_mode + result["key_authenticated"] = True logger.info("Page 1 HMAC verification passed: mode=%s path=%s", key_mode, db_path) logger.info( "[decrypt.pipeline] key_material_resolved %s", @@ -938,6 +1011,11 @@ def _finalize(success: bool, error: str = "") -> bool: ), ) + encrypted_data, result["wal"] = merge_wal_snapshot( + encrypted_data, wal_data, PAGE_SIZE, + verify_page=lambda page, pgno: hmac.compare_digest( + page[-HMAC_SIZE:], _compute_page_hmac(mac_key, page, pgno)), + ) decrypted_data = bytearray() total_pages = (len(encrypted_data) + PAGE_SIZE - 1) // PAGE_SIZE successful_pages = 0 @@ -1042,9 +1120,11 @@ def _finalize(success: bool, error: str = "") -> bool: result["successful_pages"] = int(successful_pages) result["failed_pages"] = int(failed_pages) - # 写入解密后的文件 - with open(output_path, 'wb') as f: - f.write(decrypted_data) + # WAL commit size is authoritative even if page 1 was not in the WAL. + if result["wal"]["committed_frames"]: + decrypted_data[28:32] = result["wal"]["database_pages"].to_bytes(4, "big") + # 写入待校验的临时文件，验证成功后原子替换输出。 + _write_output(decrypted_data) logger.info(f"解密文件大小: {len(decrypted_data)} bytes") if int(len(decrypted_data)) != int(result["expected_output_size"]): @@ -1077,6 +1157,13 @@ def _finalize(success: bool, error: str = "") -> bool: except Exception as e: logger.error(f"解密失败: {db_path}, 错误: {e}") return _finalize(False, str(e)) + finally: + if working_output is not None: + for suffix in ("", "-wal", "-shm", "-journal"): + try: + Path(str(working_output) + suffix).unlink(missing_ok=True) + except OSError as exc: + logger.warning("清理解密临时文件失败: %s", exc) def decrypt_wechat_databases(db_storage_path: str = None, key: str = None) -> dict: """ diff --git a/tests/test_decrypt_image_keys_frontend.py b/tests/test_decrypt_image_keys_frontend.py index b720ac29..eeb4198c 100644 --- a/tests/test_decrypt_image_keys_frontend.py +++ b/tests/test_decrypt_image_keys_frontend.py @@ -197,7 +197,7 @@ def test_saved_database_key_prefill_remains_available_before_macos_recapture(): def test_db_key_persistence_failure_warns_without_blocking_image_key_step(): source = read_decrypt_page() - warning = "数据库密钥未通过完整实时库校验或无法安全保存；请重新获取并确认主要数据库解密成功，仍失败请检查数据目录权限。" + warning = "数据库密钥未通过 session/message 跨库认证或保存失败；请查看失败详情，确认账号密钥及数据目录写入权限。" assert warning in source assert "if (result?.db_key_persisted !== false) return" in source assert source.count("showDbKeyPersistenceWarning(") == 2 diff --git a/tests/test_decrypt_stream_sse.py b/tests/test_decrypt_stream_sse.py index 16dfeb84..00b8e8e4 100644 --- a/tests/test_decrypt_stream_sse.py +++ b/tests/test_decrypt_stream_sse.py @@ -491,7 +491,10 @@ def test_decrypt_stream_marks_invalid_output_as_failed(self): self.assertEqual(events[-1].get("status"), "failed") self.assertEqual(events[-1].get("success_count"), 0) self.assertEqual(events[-1].get("failure_count"), 1) - self.assertIn("密钥可能不匹配", str(events[-1].get("message") or "")) + self.assertIn("数据库校验未通过", str(events[-1].get("message") or "")) + diagnostic = next(iter(events[-1]["account_results"].values()))["db_diagnostics"]["MSG0.db"] + self.assertEqual(diagnostic["error"], "key_mismatch") + self.assertIs(diagnostic["key_authenticated"], False) upsert_mock.assert_not_called() out = root / "output" / "databases" / "wxid_bad" / "MSG0.db" diff --git a/tests/test_offline_wal_recovery.py b/tests/test_offline_wal_recovery.py new file mode 100644 index 00000000..dcc39812 --- /dev/null +++ b/tests/test_offline_wal_recovery.py @@ -0,0 +1,321 @@ +"""Real SQLite pages and encrypted fixtures for issue #166 (no mocked diagnostics).""" +import hashlib +from contextlib import closing +import hmac +import sqlite3 +import struct +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +import wechat_decrypt_tool.wechat_decrypt as decrypt +from wechat_decrypt_tool.sqlite_wal import merge_wal_snapshot +from wechat_decrypt_tool.routers.decrypt import _db_key_persistence_rejection + +KEY = bytes(range(32)) +SALT = bytes(range(16)) +INDEX = 'SessionUnreadListTable_1_NameId_CreateTime' + + +def checksum(data, state=(0, 0), endian='<'): + values = struct.unpack(endian + 'I' * (len(data) // 4), data) + a, b = state + for i in range(0, len(values), 2): + a = (a + values[i] + b) % 2**32 + b = (b + values[i + 1] + a) % 2**32 + return a, b + + +def wal_bytes(frames, endian='<'): + header = struct.pack('>IIIIII', 0x377f0682 if endian == '<' else 0x377f0683, + 3007000, 4096, 0, 123, 456) + state = checksum(header, endian=endian) + output = header + struct.pack('>II', *state) + for pgno, size, page in frames: + fields = struct.pack('>IIII', pgno, size, 123, 456) + state = checksum(fields[:8] + page, state, endian) + output += fields + struct.pack('>II', *state) + page + return output + + +def plain_database(path): + with closing(sqlite3.connect(path)) as c: + c.execute('PRAGMA page_size=4096') + c.execute('VACUUM') + # Reserve SQLCipher's 80-byte IV/MAC region before creating any records. + raw = bytearray(path.read_bytes()) + raw[20] = 80 + raw[105:107] = (4096 - 80).to_bytes(2, 'big') + path.write_bytes(raw) + c = sqlite3.connect(path) + c.execute('CREATE TABLE records(id INTEGER PRIMARY KEY, name TEXT, time INT)') + c.execute(f'CREATE INDEX {INDEX} ON records(name,time)') + c.execute("INSERT INTO records VALUES(1,'before',1)") + c.commit() + return c + + +@pytest.fixture(scope='module') +def keys(): + enc = decrypt._derive_sqlcipher_enc_key(KEY, SALT) + mac = decrypt._derive_mac_key(enc, SALT) + return enc, mac + + +def encrypt_page(page, pgno, keys): + enc, mac = keys + iv = bytes(range(16, 32)) + prefix = SALT if pgno == 1 else b'' + start = 16 if pgno == 1 else 0 + cipher = Cipher(algorithms.AES(enc), modes.CBC(iv)).encryptor() + data = prefix + cipher.update(page[start:4016]) + cipher.finalize() + iv + digest = hmac.new(mac, data[start:] + pgno.to_bytes(4, 'little'), hashlib.sha512).digest() + return data + digest + + +def encrypt_database(data, keys): + return b''.join(encrypt_page(data[i:i+4096], i//4096+1, keys) for i in range(0, len(data), 4096)) + + +def run_decrypt(src, dst): + worker = decrypt.WeChatDatabaseDecryptor(KEY.hex()) + ok = worker.decrypt_database(str(src), str(dst)) + return ok, worker.last_result + + +@pytest.mark.parametrize('encrypted', [False, True]) +def test_real_sqlite_wal_commits_are_exported(tmp_path, keys, encrypted): + src, out = tmp_path/'source.db', tmp_path/'output.db' + c = plain_database(src) + c.execute('PRAGMA journal_mode=WAL') + c.execute('PRAGMA wal_autocheckpoint=0') + c.execute("UPDATE records SET name='first commit'") + c.commit() + c.execute("UPDATE records SET name='latest commit'") + c.execute("INSERT INTO records VALUES(2,'second row',2)") + c.commit() + main, wal = src.read_bytes(), Path(str(src)+'-wal').read_bytes() + c.close() + if encrypted: + frames = [] + for offset in range(32, len(wal), 4120): + pgno, size = struct.unpack('>II', wal[offset:offset+8]) + frames.append((pgno, size, encrypt_page(wal[offset+24:offset+4120], pgno, keys))) + wal = wal_bytes(frames) + main = encrypt_database(main, keys) + src.write_bytes(main) + Path(str(src)+'-wal').write_bytes(wal) + ok, result = run_decrypt(src, out) + assert ok, result + assert result['wal']['committed_frames'] > 0 + assert result['key_authenticated'] is encrypted + assert out.read_bytes()[18:20] == b'\x01\x01' + with closing(sqlite3.connect(out)) as conn: + assert conn.execute('SELECT name FROM records ORDER BY id').fetchall() == [('latest commit',), ('second row',)] + assert conn.execute('PRAGMA integrity_check').fetchall() == [('ok',)] + assert src.read_bytes() == main + assert Path(str(src)+'-wal').read_bytes() == wal + + +def broken_index(tmp_path): + path = tmp_path/'fixture.db' + c = plain_database(path) + before = path.read_bytes() + index_page = c.execute("SELECT rootpage FROM sqlite_master WHERE type='index'").fetchone()[0] + c.execute("INSERT INTO records VALUES(2,'new',2)") + c.commit() + c.close() + after = path.read_bytes() + mixed = bytearray(after) + offset = (index_page-1)*4096 + mixed[offset:offset+4096] = before[offset:offset+4096] + return bytes(mixed), after, index_page + + +@pytest.mark.parametrize('with_wal', [False, True]) +def test_issue166_authenticated_index_mismatch_recovers(tmp_path, keys, with_wal): + mixed, good, pgno = broken_index(tmp_path) + src, out = tmp_path/'session.db', tmp_path/'out.db' + encrypted = encrypt_database(mixed, keys) + src.write_bytes(encrypted) + if with_wal: + page = good[(pgno-1)*4096:pgno*4096] + Path(str(src)+'-wal').write_bytes(wal_bytes([(pgno,len(good)//4096,encrypt_page(page,pgno,keys))])) + ok, result = run_decrypt(src, out) + assert ok, result + assert result['failed_pages'] == result['hmac_warning_pages'] == 0 + assert result['key_authenticated'] + if with_wal: + assert not result['index_repair'] # WAL fixed it, no REINDEX required. + else: + assert result['index_repair']['success'] + assert result['index_repair']['indexes'] == [INDEX] + with closing(sqlite3.connect(out)) as c: + assert c.execute('SELECT COUNT(*) FROM records').fetchone() == (2,) + assert c.execute('PRAGMA integrity_check').fetchone() == ('ok',) + assert src.read_bytes() == encrypted + + +@pytest.mark.parametrize('endian', ['<', '>']) +def test_commit_boundaries_recycled_tail_and_truncation(endian): + a, b, c, d = [bytes([n])*4096 for n in range(4)] + wal = wal_bytes([(2,2,b), (3,3,c), (2,2,d), (2,0,a)], endian) + actual, info = merge_wal_snapshot(a+a+a, wal+b'\0'*4120, 4096) + assert actual == a+d + assert info['committed_frames'] == 3 + assert info['ignored_tail_bytes'] == 8240 + + +@pytest.mark.parametrize('damage', ['header', 'frame', 'page_size', 'version', 'short', 'growth']) +def test_invalid_wal_is_not_silently_ignored(damage): + data = b'0'*4096 + wal = bytearray(wal_bytes([(1,1,data)])) + if damage == 'header': wal[24] ^= 1 + if damage == 'frame': wal[-1] ^= 1 + if damage == 'page_size': wal[8:12] = (8192).to_bytes(4,'big') + if damage == 'version': wal[4:8] = (0).to_bytes(4,'big') + if damage == 'short': wal = wal[:16] + if damage == 'growth': wal = wal_bytes([(1,0xfffffffe,data)]) + with pytest.raises(ValueError): merge_wal_snapshot(data, bytes(wal), 4096) + + +def test_uncommitted_and_partial_frames_do_not_change_snapshot(): + a, b = b'a'*4096, b'b'*4096 + actual, info = merge_wal_snapshot(a, wal_bytes([(1,1,b), (1,0,a)])[:-100], 4096) + assert actual == b + assert info['ignored_tail_bytes'] == 4020 + assert merge_wal_snapshot(a, wal_bytes([(1,0,b)]),4096)[0] == a + + +def test_valid_wal_checksum_does_not_bypass_page_hmac(tmp_path, keys): + src, out = tmp_path/'session.db', tmp_path/'out.db' + conn = plain_database(src); conn.close() + data = encrypt_database(src.read_bytes(), keys); src.write_bytes(data) + bad = bytearray(data[4096:8192]); bad[-1] ^= 1 + Path(str(src)+'-wal').write_bytes(wal_bytes([(2,3,bytes(bad))])) + out.write_bytes(b'previous good output') + ok, result = run_decrypt(src, out) + assert not ok and 'HMAC' in result['error'] + assert out.read_bytes() == b'previous good output' + assert not list(tmp_path.glob('.decrypt-*')) + + +def test_changing_wal_is_rejected(tmp_path, monkeypatch): + src = tmp_path/'source.db'; conn = plain_database(src); conn.close() + original = decrypt._safe_file_snapshot + calls = 0 + def snapshot(path): + nonlocal calls + result = original(path); calls += 1 + if calls == 2: result['siblings']['-wal'] = {'exists': True, 'size': 1} + return result + monkeypatch.setattr(decrypt, '_safe_file_snapshot', snapshot) + ok, result = run_decrypt(src,tmp_path/'out.db') + assert not ok and result['source_changed_during_read'] + assert not (tmp_path/'out.db').exists() + + +def test_key_authentication_is_independent_of_output_integrity(): + diagnostics = {name:{'db_name':name, 'key_authenticated':True, + 'key_mode':'sqlcipher_passphrase', 'success':False, 'diagnostic_status':'quick_check_failed'} + for name in ['session.db','message_0.db']} + assert _db_key_persistence_rejection({'db_diagnostics':diagnostics}) == '' + diagnostics['session.db']['key_authenticated'] = False + assert 'session' in _db_key_persistence_rejection({'db_diagnostics':diagnostics}) + + +def test_table_corruption_is_not_repaired_or_published(tmp_path, keys): + src = tmp_path/'session.db'; c = plain_database(src); c.close() + data = bytearray(src.read_bytes()); data[4096] = 0xff + src.write_bytes(encrypt_database(data,keys)) + ok, result = run_decrypt(src,tmp_path/'out.db') + assert not ok + assert not result['index_repair'].get('attempted') + assert not (tmp_path/'out.db').exists() + + +def test_partial_output_still_saves_cross_database_authenticated_key(monkeypatch): + import wechat_decrypt_tool.routers.decrypt as router + import wechat_decrypt_tool.wcdb_realtime as realtime + saved = [] + monkeypatch.setattr(router, 'upsert_account_keys_in_store', lambda *a, **kw: saved.append((a,kw))) + monkeypatch.setattr(realtime.WCDB_REALTIME, 'disconnect', lambda account: None) + diags = {name: {'db_name':name, 'key_mode':'sqlcipher_passphrase', 'key_authenticated':True, + 'success':name != 'session.db', 'diagnostic_status':'quick_check_failed' if name == 'session.db' else 'ok'} + for name in ['session.db','message_0.db']} + assert router._persist_db_keys({'test_account':{'success':1,'db_diagnostics':diags}},KEY.hex()) == (True,[]) + assert len(saved) == 1 + diags['session.db']['key_authenticated'] = False + assert router._persist_db_keys({'test_account':{'success':1,'db_diagnostics':diags}},KEY.hex()) == (False,['test_account']) + assert len(saved) == 1 + + +def test_failed_publication_keeps_previous_output(tmp_path, monkeypatch): + src, out = tmp_path/'source.db', tmp_path/'output.db' + c = plain_database(src); c.close() + out.write_bytes(b'previous good output') + def fail(*args): raise PermissionError('output busy') + monkeypatch.setattr(decrypt.os,'replace',fail) + ok, result = run_decrypt(src,out) + assert not ok and 'output busy' in result['error'] + assert out.read_bytes() == b'previous good output' + assert not list(tmp_path.glob('.decrypt-*')) + + +@pytest.mark.parametrize('alias', ['same_path', 'hardlink', 'symlink']) +def test_source_is_never_overwritten(tmp_path, alias): + src = tmp_path/'source.db'; c = plain_database(src); c.close() + original = src.read_bytes(); out = tmp_path/'out.db' + if alias == 'hardlink': out.hardlink_to(src) + elif alias == 'symlink': + try: + out.symlink_to(src) + except OSError: + pytest.skip('Creating symlinks is unavailable on this platform') + else: out = src + ok, result = run_decrypt(src,out) + assert not ok and '源数据库' in result['error'] + assert src.read_bytes() == original + + +def test_growth_after_truncation_cannot_reuse_discarded_pages(): + a,b,c = b'a'*4096,b'b'*4096,b'c'*4096 + with pytest.raises(ValueError, match='扩展页'): + merge_wal_snapshot(a+b+c,wal_bytes([(1,1,a),(3,3,c)]),4096) + result, info = merge_wal_snapshot(a+b+c,wal_bytes([(1,1,a),(2,0,c),(3,3,b)]),4096) + assert result == a+c+b + + +def test_old_output_wal_cannot_override_new_output(tmp_path): + src, out = tmp_path/'source.db',tmp_path/'out.db' + c = plain_database(src); c.close() + out.write_bytes(b'previous good output') + Path(str(out)+'-wal').write_bytes(b'existing log') + ok, result = run_decrypt(src,out) + assert not ok and '输出数据库' in result['error'] + assert out.read_bytes() == b'previous good output' + assert Path(str(out)+'-wal').read_bytes() == b'existing log' + + +def test_real_sqlite_wal_growth_and_vacuum(tmp_path): + src, out = tmp_path/'source.db',tmp_path/'out.db' + c = plain_database(src) + c.execute('PRAGMA journal_mode=WAL') + c.execute('PRAGMA wal_autocheckpoint=0') + c.executemany('INSERT INTO records VALUES(?,?,?)', [(i, 'x'*1000, i) for i in range(2, 100)]) + c.commit() + expected = c.execute('SELECT COUNT(*) FROM records').fetchone() + ok, result = run_decrypt(src,out) + assert ok, result + with closing(sqlite3.connect(out)) as reader: + assert reader.execute('SELECT COUNT(*) FROM records').fetchone() == expected + assert reader.execute('PRAGMA integrity_check').fetchone() == ('ok',) + c.execute('DELETE FROM records WHERE id > 2'); c.commit() + c.execute('VACUUM') + ok, result = run_decrypt(src,out) + assert ok, result + with closing(sqlite3.connect(out)) as reader: + assert reader.execute('SELECT COUNT(*) FROM records').fetchone() == (2,) + assert reader.execute('PRAGMA integrity_check').fetchone() == ('ok',) + c.close()