From ca4cfb2ab05f6035ce5d408a0c52a1882dbb0fd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?= <277378677+maiphucgiang@users.noreply.github.com> Date: Tue, 22 Sep 2026 04:13:30 +0800 Subject: [PATCH 1/4] Persist gateway state and default API keys in SQLite Load optional native .env settings and retain a generated default key across restarts, with one-time terminal disclosure. Migrate runtime snapshots into the control database, keep audit logs separate, and retire text log output. --- .env.example | 7 +- README.md | 2 +- README.zh-CN.md | 2 +- app/admin_auth.py | 29 ++- app/audit_store.py | 2 +- app/control_store.py | 11 +- app/credential_cooldowns.py | 41 ++-- app/credits.py | 21 +- app/model_blocks.py | 15 +- app/runtime_management.py | 19 +- app/safe_logging.py | 2 +- app/settings.py | 6 +- app/startup.py | 100 +++++++++ app/state_store.py | 276 +++++++++++++++++++++++++ app/trial_rewards.py | 12 +- app/usage_snapshots.py | 39 ++-- converter.py | 70 ++----- docs/advanced.md | 12 +- docs/advanced.zh-CN.md | 12 +- docs/deployment.md | 16 +- docs/deployment.zh-CN.md | 16 +- docs/webui.md | 8 +- docs/webui.zh-CN.md | 8 +- pyproject.toml | 1 + requirements.in | 1 + requirements.txt | 4 +- tests/test_control_store.py | 2 +- tests/test_environment_config.py | 1 + tests/test_identity_sync.py | 7 +- tests/test_login.py | 1 + tests/test_runtime_endpoints.py | 67 +++--- tests/test_sqlite_state.py | 341 +++++++++++++++++++++++++++++++ tests/webui_fixture.py | 8 +- uv.lock | 2 + web/src/pages/Settings.tsx | 2 + 35 files changed, 965 insertions(+), 198 deletions(-) create mode 100644 app/startup.py create mode 100644 app/state_store.py create mode 100644 tests/test_sqlite_state.py diff --git a/.env.example b/.env.example index 1d84684..33534d6 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # Copy to .env for first use; preserve existing settings when updating. -# Compose loads .env automatically; native launches use uv run --env-file .env. +# Compose and native launches read .env; native zero-configuration startup can omit it. # Native listener and Compose host mapping; explicit --host/--port override native values. CODEBUDDY2API_BIND=127.0.0.1 @@ -15,6 +15,7 @@ CODEBUDDY2API_AUTH_PATH=./auth # CODEBUDDY_IMPORT_DIR=./auth/imports # Empty locks management; set a random key before exposing inference beyond loopback. +# Native: omit/comment the variable to use the saved default key (first local start displays it once). CODEBUDDY2API_KEY= # Native-only unsafe opt-in; Compose uses its host mapping to control exposure. # CODEBUDDY2API_ALLOW_OPEN_NOAUTH=false @@ -48,9 +49,9 @@ CODEBUDDY2API_MAX_CONCURRENT=64 # Disable only declared model-capability preflight; other safeguards and Intl image merging remain enabled. # CODEBUDDY2API_MODEL_CAPABILITY_GUARD=true -# SQLite audit defaults to the data directory; this optional path enables a separate text log. +# Retired: this setting only warns; all persistent logs use logs.sqlite3. CODEBUDDY2API_LOG= -# Redacted text preview bytes; zero logs metadata only. +# Legacy text-preview limit; text output is retired. SQLite diagnostics have a separate budget. CODEBUDDY2API_LOG_BODY_LIMIT=65536 # International trial credits remain manual-only in the WebUI. diff --git a/README.md b/README.md index 54ca005..fd97942 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The template binds to localhost only. Configure HTTPS and restrict network acces ## FAQ - **WebUI sign-in fails behind an HTTPS domain/reverse proxy?** Trust your public origin via `admin_allowed_origins` — see [Management Origin checks](docs/advanced.md#management-origin--csrf-switch). -- **No Docker on the server?** Run the terminal login and the gateway with Python directly — see [Local Python setup](docs/deployment.md#local-python-setup). +- **No Docker?** After installing dependencies and the WebUI, run `uv run converter.py` or `python3 converter.py` without `.env`. First local startup saves a default key and displays it once — see [Local Python setup](docs/deployment.md#local-python-setup). - **Where is my data?** Everything lives in `auth/` (or `/data/auth` in Docker): credentials, settings and log databases — see [Data and backups](docs/webui.md#data-and-backups). - **Which image tag should I use?** `latest` follows stable releases, `edge` follows main, version tags pin one release — see [Published images](docs/deployment.md#use-published-images). diff --git a/README.zh-CN.md b/README.zh-CN.md index 621a835..506bf57 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -82,7 +82,7 @@ docker compose up -d --no-build ## 常见问题 - **绑定域名后经 HTTPS 反代无法登录 WebUI？** 将对外来源加入 `admin_allowed_origins` 信任列表——见[管理 Origin 校验](docs/advanced.zh-CN.md#管理-origin--csrf-开关)。 -- **服务器没有 Docker？** 可用 Python 直接完成终端登录并运行网关——见[本地 Python 运行](docs/deployment.zh-CN.md#本地-python-运行)。 +- **没有 Docker？** 准备依赖和 WebUI 后，直接 `uv run converter.py` 或 `python3 converter.py`，无需 `.env`；首次本地启动保存默认 key 并仅在终端显示一次——见[本地 Python 运行](docs/deployment.zh-CN.md#本地-python-运行)。 - **数据在哪里？** 全部位于 `auth/`（Docker 中为 `/data/auth`）：凭证、设置与日志数据库——见[数据与备份](docs/webui.zh-CN.md#数据与备份)。 - **镜像标签怎么选？** `latest` 跟随稳定版，`edge` 跟随 main，版本标签固定某一发行版——见[使用已发布镜像](docs/deployment.zh-CN.md#使用已发布镜像)。 diff --git a/app/admin_auth.py b/app/admin_auth.py index 79d3945..b09d05d 100644 --- a/app/admin_auth.py +++ b/app/admin_auth.py @@ -10,6 +10,7 @@ from pathlib import Path import re import secrets +import sqlite3 import stat import tempfile import threading @@ -132,7 +133,8 @@ def __init__(self, config, *, clock=time.monotonic, wall_clock=time.time): self.failures = OrderedDict() self._configured_key = None self._identity = None - self._path = _session_path(config.get("session_path")) + self._store = config.get("state_store") + self._path = self._store.path if self._store is not None else _session_path(config.get("session_path")) self._storage_error = None # Set when the snapshot could not be written or cleared. @staticmethod @@ -179,6 +181,14 @@ def storage(self): def _revoke(self): """Revoke the persisted snapshot; returns False when it could not be cleared.""" + if self._store is not None: + try: + self._store.delete("sessions") + except (OSError, ValueError, sqlite3.Error) as error: + self._storage_failed(error) + return False + self._storage_ok() + return True if self._path is None: return True try: @@ -209,6 +219,12 @@ def _restore(self, key): Returns False when a snapshot exists that cannot be trusted or validated, so the caller revokes it instead of leaving it available to a later start. """ + if self._store is not None: + try: + document = self._store.get("sessions") + return True if document is None else self._adopt(document, key) + except (OSError, ValueError, sqlite3.Error): + return False if self._path is None: return True try: @@ -237,6 +253,9 @@ def _restore(self, key): except (ValueError, UnicodeDecodeError, RecursionError): # RecursionError: size-bounded but deeply nested JSON still exhausts the parser. return False + return self._adopt(document, key) + + def _adopt(self, document, key): if (not isinstance(document, dict) or set(document) != {"version", "fingerprint", "sessions"} or type(document["version"]) is not int or document["version"] != SESSION_FILE_VERSION): return False @@ -280,6 +299,14 @@ def _persist(self): document = {"version": SESSION_FILE_VERSION, "fingerprint": self._fingerprint(self._configured_key), "sessions": {sid: {"csrf_token": item["csrf_token"], "expires": item["expires"]} for sid, item in self.sessions.items()}} + if self._store is not None: + try: + self._store.put("sessions", document) + self._storage_ok() + return True + except (OSError, ValueError, sqlite3.Error) as error: + self._storage_failed(error) + return self._revoke() try: content = json.dumps(document, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") diff --git a/app/audit_store.py b/app/audit_store.py index 3f294a9..7a608a9 100644 --- a/app/audit_store.py +++ b/app/audit_store.py @@ -21,7 +21,7 @@ METRICS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_creation_tokens", "reasoning_tokens", "total_tokens", "credit") _IDENTIFIER = re.compile(r"^[A-Za-z0-9_.:/@-]{1,160}$") -_SECRET = re.compile(r"(?i)(bearer|sk-|access[_-]?token|refresh[_-]?token|api[_-]?key|eyJ|://)") +_SECRET = re.compile(r"(?i)(bearer|sk-|cb-|access[_-]?token|refresh[_-]?token|api[_-]?key|eyJ|://)") def safe_label(value: Any, limit: int = 160) -> str | None: diff --git a/app/control_store.py b/app/control_store.py index 038d899..2da74f8 100644 --- a/app/control_store.py +++ b/app/control_store.py @@ -70,12 +70,13 @@ def validate_model(source, rule, models=None, known_models=(), *, legacy_scopes= class ControlStore: - SCHEMA_VERSION = 1 + SCHEMA_VERSION = 2 def __init__(self, path): path = Path(path) existed = path.exists() path = _secure_path(path) + self.path = path self._lock = threading.RLock() self._db = sqlite3.connect(path, check_same_thread=False, isolation_level=None, timeout=1) try: @@ -88,7 +89,7 @@ def __init__(self, path): self._db.execute("CREATE TABLE control (id INTEGER PRIMARY KEY CHECK(id=1), revision INTEGER NOT NULL, payload TEXT NOT NULL)") self._db.execute("INSERT INTO control VALUES (1,0,?)", (json.dumps({"settings": {}, "models": {}, "credentials": {}}),)) self._db.execute("PRAGMA user_version=1") - elif version != self.SCHEMA_VERSION: + elif version not in (1, self.SCHEMA_VERSION): raise ValueError("管理数据库 schema 不受支持或已有库为空，未执行初始化") self._snapshot = self._load() self._db.execute("CREATE TABLE IF NOT EXISTS buddy_bootstrap (" @@ -106,12 +107,18 @@ def __init__(self, path): "accept_started INTEGER NOT NULL DEFAULT 0, chat_started INTEGER NOT NULL DEFAULT 0, " "completed INTEGER NOT NULL DEFAULT 0, model TEXT, chat_state TEXT NOT NULL DEFAULT 'pending', " "total_tokens INTEGER, updated_at REAL NOT NULL)") + self._db.execute("CREATE TABLE IF NOT EXISTS runtime_state (name TEXT PRIMARY KEY, payload TEXT NOT NULL)") + self._db.execute("CREATE TABLE IF NOT EXISTS state_imports (name TEXT PRIMARY KEY, imported INTEGER NOT NULL, migrated_at REAL NOT NULL)") + self._db.execute("CREATE TABLE IF NOT EXISTS gateway_secrets (name TEXT PRIMARY KEY, value TEXT NOT NULL, announced INTEGER NOT NULL DEFAULT 0 CHECK(announced IN (0,1)))") + self._db.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}") self._db.execute("COMMIT") except Exception: if self._db.in_transaction: self._db.execute("ROLLBACK") self._db.close() raise + from .state_store import StateStore + self.state = StateStore(self) def _load(self): row = self._db.execute("SELECT revision,payload FROM control WHERE id=1").fetchone() diff --git a/app/credential_cooldowns.py b/app/credential_cooldowns.py index bac7f40..2ba2490 100644 --- a/app/credential_cooldowns.py +++ b/app/credential_cooldowns.py @@ -17,7 +17,7 @@ import json import os import re -import stat +import sqlite3 import tempfile import threading import time @@ -95,8 +95,9 @@ class CredentialCooldowns: """Thread-safe, optionally persisted cooldown table keyed by validated account identity.""" def __init__(self, path=None, *, auth_ceiling_s: float = AUTH_CEILING_S, - model_ceiling_s: float = MODEL_CEILING_S): - self.path = str(path) if path else None + model_ceiling_s: float = MODEL_CEILING_S, store=None): + self._store = store + self.path = str(store.path) if store is not None else (str(path) if path else None) self.auth_ceiling_s = _bounded_ceiling(auth_ceiling_s, AUTH_CEILING_S) self.model_ceiling_s = _bounded_ceiling(model_ceiling_s, MODEL_CEILING_S) self._lock = threading.RLock() @@ -113,28 +114,8 @@ def __init__(self, path=None, *, auth_ceiling_s: float = AUTH_CEILING_S, def _load(self): """Adopt only a fully valid snapshot; anything else leaves the table empty.""" - try: - # Reject a symlink or FIFO *before* opening, so a device cannot block the read. - # This is best effort on Windows and is backed up by the fstat check below. - if not stat.S_ISREG(os.lstat(self.path).st_mode): - return - fd = os.open(self.path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - | getattr(os, "O_NONBLOCK", 0)) - except OSError: - return - try: - with os.fdopen(fd, "rb") as stream: - if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): - return - raw = stream.read(MAX_BYTES + 1) - except OSError: - return - if len(raw) > MAX_BYTES: - return - try: - document = _strict_json(raw) - except (ValueError, UnicodeDecodeError, RecursionError): - return + from .state_store import load_document + document = load_document(self.path, self._store, "cooldowns", MAX_BYTES) if (not isinstance(document, dict) or set(document) != {"version", "accounts"} or type(document["version"]) is not int or document["version"] != VERSION): return @@ -204,6 +185,16 @@ def _save_locked(self) -> bool: self.last_error = "serialize" self._dirty = True return False + if self._store is not None: + try: + self._store.put("cooldowns", json.loads(content)) + except (OSError, ValueError, sqlite3.Error) as error: + self.last_error = type(error).__name__ + self._dirty = True + return False + self.last_error = "capacity" if shed else None + self._dirty = shed + return not shed temporary = None try: directory = os.path.dirname(self.path) or "." diff --git a/app/credits.py b/app/credits.py index 0cc7278..0e84217 100644 --- a/app/credits.py +++ b/app/credits.py @@ -630,13 +630,17 @@ def fetch_request_usage(access_token: str, days: int = USAGE_MAX_DAYS, class CreditLedger: """Persist per-credential check-in and balance snapshots for expiry-aware scheduling.""" - def __init__(self, path: Path): - self.path = Path(path) + def __init__(self, path: Path | None = None, *, store=None): + self._store = store + self.path = Path(store.path if store is not None else path) self._lock = threading.Lock() self._data: dict = {"version": 1, "creds": {}} self._load() def _load(self): + if self._store is not None: + self._data = self._store.get("credits") or {"version": 1, "creds": {}} + return try: self._data = json.loads(self.path.read_text(encoding="utf-8")) if "creds" not in self._data: @@ -645,6 +649,9 @@ def _load(self): self._data = {"version": 1, "creds": {}} def _save(self): + if self._store is not None: + self._store.put("credits", self._data) + return try: self.path.parent.mkdir(parents=True, exist_ok=True) tmp = self.path.with_suffix(self.path.suffix + ".tmp") @@ -737,8 +744,9 @@ class ModelCatalogCache: SCHEMA_VERSION = 2 - def __init__(self, path: Path, ttl: float = 6 * 3600): - self.path = Path(path) + def __init__(self, path: Path | None = None, ttl: float = 6 * 3600, *, store=None): + self._store = store + self.path = Path(store.path if store is not None else path) self.ttl = max(60.0, float(ttl or 0)) self._lock = threading.Lock() self._data: dict = {"version": self.SCHEMA_VERSION, "groups": {}} @@ -747,7 +755,7 @@ def __init__(self, path: Path, ttl: float = 6 * 3600): def _load(self): with self._lock: try: - d = json.loads(self.path.read_text(encoding="utf-8")) + d = (self._store.get("catalog") or self._data) if self._store is not None else json.loads(self.path.read_text(encoding="utf-8")) if (not isinstance(d, dict) or d.get("version") not in (1, self.SCHEMA_VERSION) or not isinstance(d.get("groups"), dict)): return @@ -773,6 +781,9 @@ def _load(self): pass # Keep the loaded catalog when disk reads fail. def _save(self): + if self._store is not None: + self._store.put_cache("catalog", self._data) + return try: self.path.parent.mkdir(parents=True, exist_ok=True) tmp = self.path.with_suffix(self.path.suffix + ".tmp") diff --git a/app/model_blocks.py b/app/model_blocks.py index 6b4958b..fd360df 100644 --- a/app/model_blocks.py +++ b/app/model_blocks.py @@ -16,8 +16,9 @@ class ModelBlocks: """Maintain thread-safe per-backend model backoff with optional persistence.""" - def __init__(self, path=None, ttl_s: float = DEFAULT_TTL_S, max_ttl_s: float = MAX_TTL_S): - self.path = str(path) if path else None + def __init__(self, path=None, ttl_s: float = DEFAULT_TTL_S, max_ttl_s: float = MAX_TTL_S, *, store=None): + self._store = store + self.path = str(store.path) if store is not None else (str(path) if path else None) self.ttl_s = max(60.0, float(ttl_s or 0)) self.max_ttl_s = max(self.ttl_s, float(max_ttl_s or 0)) self._lock = threading.Lock() @@ -29,8 +30,11 @@ def __init__(self, path=None, ttl_s: float = DEFAULT_TTL_S, max_ttl_s: float = M def _load(self): try: - with open(self.path, "r", encoding="utf-8") as f: - data = json.load(f) + if self._store is not None: + data = self._store.get("model_blocks") + else: + with open(self.path, "r", encoding="utf-8") as f: + data = json.load(f) except (OSError, ValueError): return if not isinstance(data, dict): @@ -52,6 +56,9 @@ def _load(self): self._data = out def _save_locked(self): + if self._store is not None: + self._store.put_cache("model_blocks", {"version": 1, "blocks": self._data}) + return if not self.path: return try: diff --git a/app/runtime_management.py b/app/runtime_management.py index ba0b1f3..eb98c45 100644 --- a/app/runtime_management.py +++ b/app/runtime_management.py @@ -59,16 +59,19 @@ def close(self): pass -def initialize(gateway, args, argv=None, *, parser=None): +def initialize(gateway, args, argv=None, *, parser=None, dotenv_keys=()): config = gateway.CONFIG config["auto_accept_buddy"] = buddy.auto_accept_from_env(os.environ) config["auto_accept_buddy_source"] = "environment" if "CODEBUDDY2API_AUTO_ACCEPT_BUDDY" in os.environ else "default" root = gateway.managed_auth_dir() control = ControlStore(root / "control.sqlite3") config["control_store"] = control - # Persist management sessions so a restart does not force another API-key login. - # The file stores an HMAC fingerprint of the key epoch, so key rotation still revokes them. - config["session_path"] = root / "admin-sessions.json" + config["state_store"] = control.state + try: + control.state.migrate(root) + except BaseException: + control.close() + raise config.update(vars(args)) config["model_guard"] = not args.no_model_guard aliases = {"log": "log_path", "no_model_guard": "model_guard"} @@ -82,10 +85,16 @@ def initialize(gateway, args, argv=None, *, parser=None): key = options[matches[0]].dest if len(matches) == 1 else flag[2:].replace("-", "_") explicit.add(aliases.get(key, key)) apply_persisted_settings(config, explicit=explicit) + from .startup import resolve_startup_key + try: + resolve_startup_key(config, args, dotenv_keys) + except BaseException: + control.close() + raise for key in SCHEMA: if hasattr(args, key) and key != "api_key": setattr(args, key, config[key]) - config["trial_ledger"] = gateway.trial_rewards.TrialLedger(root / "trial-ledger.json") + config["trial_ledger"] = gateway.trial_rewards.TrialLedger(store=control.state) try: config["audit_store"] = AuditStore(root / "logs.sqlite3", max_bytes=config["audit_max_bytes"], retention_days=config["audit_retention_days"], diff --git a/app/safe_logging.py b/app/safe_logging.py index b58e20e..19cfa37 100644 --- a/app/safe_logging.py +++ b/app/safe_logging.py @@ -47,7 +47,7 @@ # hex strings/UUIDs: those often carry useful request and trace identifiers. _JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]*){0,2}") _API_TOKEN = re.compile( - r"(? 256 * 1024: + raise ValueError(".env 必须是普通文件且不超过 256 KiB") + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + with os.fdopen(fd, "rb") as stream: + opened = os.fstat(stream.fileno()) + if (not stat.S_ISREG(opened.st_mode) + or (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino)): + raise ValueError(".env 文件在读取期间发生变化") + content = stream.read(256 * 1024 + 1) + if len(content) > 256 * 1024: + raise ValueError(".env 文件过大") + text = content.decode("utf-8-sig") + for binding in parse_stream(io.StringIO(text)): + if binding.error: + raise ValueError(f".env 第 {binding.original.line} 行语法错误") + previous = set(os.environ) + load_dotenv(stream=io.StringIO(text), override=False) + return set(os.environ) - previous + + +def resolve_startup_key(config, args, dotenv_keys=()): + sources = config["settings_sources"] + for name, spec in SCHEMA.items(): + if sources.get(name) == "environment" and spec["env"] in dotenv_keys: + sources[name] = "dotenv" + config["announce_default_key"] = False + if sources.get("api_key") in ("cli", "environment", "dotenv"): + return + store = config["state_store"] + key, pending = store.default_key() + if key is None: + if config["host"] not in ("127.0.0.1", "::1", "localhost"): + raise ValueError("非回环监听请显式设置 API key；默认密钥仅在本地首次启动时生成") + if not sys.stderr.isatty(): + raise ValueError("首次生成 API key 需要交互终端；后台运行请显式配置 CODEBUDDY2API_KEY") + key, pending = store.default_key(create=True) + if pending and not sys.stderr.isatty(): + raise ValueError("默认 API key 尚未显示；请先在交互终端启动，或显式配置 CODEBUDDY2API_KEY") + config["api_key"] = args.api_key = key + sources["api_key"] = "generated" + config["announce_default_key"] = pending + + +def announce_default_key(config): + if not config.get("announce_default_key") or not sys.stderr.isatty(): + return + key = config["api_key"] + if config["state_store"].claim_announcement(key): + # This is deliberately outside every logging/audit path. + sys.stderr.write(f"\n默认 API key：{key}\n已保存到 control.sqlite3，仅显示这一次；管理登录与 API 请求共用。\n") + sys.stderr.flush() + config["announce_default_key"] = False + + +def run_server(app, config, *, host, port): + if not config.get("announce_default_key"): + return uvicorn.run(app, host=host, port=port, log_level="warning") + + class FirstStartServer(uvicorn.Server): + async def startup(self, sockets=None): + await super().startup(sockets=sockets) + if self.started and not self.should_exit: + try: + announce_default_key(config) + except BaseException: + # A failed announcement/commit must not leave a hidden-key server running. + self.should_exit = True + await self.shutdown(sockets=sockets) + raise + + server = FirstStartServer(uvicorn.Config(app, host=host, port=port, log_level="warning")) + try: + server.run() + except KeyboardInterrupt: + pass + if not server.started: + raise SystemExit(3) diff --git a/app/state_store.py b/app/state_store.py new file mode 100644 index 0000000..256b157 --- /dev/null +++ b/app/state_store.py @@ -0,0 +1,276 @@ +"""Private SQLite state and one-time imports of legacy runtime snapshots.""" +from __future__ import annotations + +from contextlib import contextmanager +import json +import math +import os +from pathlib import Path +import re +import secrets +import sqlite3 +import stat +import time +import warnings + +MAX_BYTES = 16 * 1024 * 1024 +LEGACY_FILES = { + "sessions": "admin-sessions.json", + "cooldowns": "credential-cooldowns.json", + "model_blocks": "model-site-blocks.json", + "credits": "credits-ledger.json", + "trial": "trial-ledger.json", + "catalog": "model-catalog.json", + "usage": "usage-snapshots.json", +} + + +def strict_json(raw): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("duplicate state field") + result[key] = value + return result + + def invalid(value): + raise ValueError("non-finite state value") + + return json.loads(raw, object_pairs_hook=pairs, parse_constant=invalid) + +def load_document(path, store, name, limit=MAX_BYTES): + """Read SQLite at runtime; retain bounded legacy readers for import validation.""" + if store is not None: + return store.get(name) + try: + metadata = os.lstat(path) + if not stat.S_ISREG(metadata.st_mode): + return None + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + with os.fdopen(fd, "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + return None + raw = stream.read(limit + 1) + return strict_json(raw) if len(raw) <= limit else None + except (OSError, ValueError, RecursionError): + return None + + + +def _check_document(name, value): + if name not in LEGACY_FILES or not isinstance(value, dict): + raise ValueError("Invalid runtime state") + # Never promote an accidentally supplied credential file into SQLite. + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, dict): + if any(str(key).lower().replace("_", "") in {"accesstoken", "refreshtoken", "idtoken"} for key in item): + raise ValueError("Credential material is not runtime state") + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) + field, versions = { + "sessions": ("sessions", (1,)), "cooldowns": ("accounts", (1,)), + "model_blocks": ("blocks", (1,)), "credits": ("creds", (1,)), + "trial": ("accounts", (1,)), "catalog": ("groups", (1, 2)), + "usage": ("accounts", (1,)), + }[name] + if type(value.get("version")) is not int or value["version"] not in versions or not isinstance(value.get(field), dict): + raise ValueError("Unsupported runtime state schema") + if name != "sessions" and set(value) != {"version", field}: + raise ValueError("Unexpected runtime state fields") + if name == "sessions": + from .admin_auth import _TOKEN, _FINGERPRINT, MAX_PERSISTED_SESSIONS + if (set(value) != {"version", "fingerprint", "sessions"} + or not isinstance(value.get("fingerprint"), str) or not _FINGERPRINT.fullmatch(value["fingerprint"]) + or len(value[field]) > MAX_PERSISTED_SESSIONS): + raise ValueError("Invalid session snapshot") + for sid, row in value[field].items(): + if (not isinstance(sid, str) or not _TOKEN.fullmatch(sid) or not isinstance(row, dict) + or set(row) != {"csrf_token", "expires"} + or not isinstance(row.get("csrf_token"), str) or not _TOKEN.fullmatch(row["csrf_token"]) + or type(row.get("expires")) not in (int, float) + or not math.isfinite(row["expires"]) or not 0 < row["expires"] < 1e11): + raise ValueError("Invalid persisted session") + elif name == "trial": + from .trial_rewards import _key, _timestamp, _safe_result, _result, _RECORD_FIELDS, _RESULT_FIELDS, _MAX_ACCOUNTS + if len(value[field]) > _MAX_ACCOUNTS: + raise ValueError("Too many trial accounts") + for key, row in value[field].items(): + _key(key) + if not isinstance(row, dict) or set(row) != _RECORD_FIELDS: + raise ValueError("Invalid trial history") + _timestamp(row["attempted_at"]) + if row["finished_at"] is not None: + _timestamp(row["finished_at"]) + if row["finished_at"] < row["attempted_at"]: + raise ValueError("Invalid trial deadline") + result = {key: row[key] for key in _RESULT_FIELDS} + if (type(result["ok"]) is not bool or type(result["already"]) is not bool + or (result["code"] is not None and type(result["code"]) is not int) + or (result["status"] is not None and type(result["status"]) is not int) + or result != _safe_result(result) + or (row["finished_at"] is None and result != _result())): + raise ValueError("Invalid trial result") + elif name == "usage": + from .usage_snapshots import UsageSnapshots, _FIELDS, MAX_ACCOUNTS + if len(value[field]) > MAX_ACCOUNTS: + raise ValueError("Too many usage snapshots") + for path, row in value[field].items(): + if (not isinstance(path, str) or not path or len(path) > 4096 or not isinstance(row, dict) + or set(row) != _FIELDS or UsageSnapshots._valid_row(row) is None): + raise ValueError("Invalid usage snapshot") + elif name == "credits": + for row in value[field].values(): + if (not isinstance(row, dict) or set(row) - {"identity", "checkin", "credits", "error", "travel"} + or not isinstance(row.get("checkin", {}), dict) or not isinstance(row.get("credits", {}), dict)): + raise ValueError("Invalid credit history") + elif name == "cooldowns": + from .credential_cooldowns import _FIELDS, _valid_identity, _valid_token, _number, MAX_ACCOUNTS, MAX_MODELS + if len(value[field]) > MAX_ACCOUNTS: + raise ValueError("Too many cooldown accounts") + for identity, row in value[field].items(): + if (not _valid_identity(identity) or not isinstance(row, dict) or set(row) != _FIELDS + or not _valid_token(row["profile"]) or not isinstance(row["models"], dict) + or len(row["models"]) > MAX_MODELS + or _number(row["fail_until"]) is None or _number(row["failed_at"]) is None + or any(not _valid_token(model) or _number(until) is None for model, until in row["models"].items())): + raise ValueError("Invalid cooldown snapshot") + elif name == "model_blocks": + for models in value[field].values(): + if not isinstance(models, dict): + raise ValueError("Invalid model blocks") + for row in models.values(): + if (not isinstance(row, dict) or type(row.get("until")) not in (int, float) + or not math.isfinite(row["until"]) or type(row.get("hits", 1)) is not int): + raise ValueError("Invalid model backoff") + elif name == "catalog": + for row in value[field].values(): + if (not isinstance(row, dict) or not isinstance(row.get("models"), list) + or not all(isinstance(model, dict) for model in row["models"]) + or type(row.get("fetched_at")) not in (int, float) + or not math.isfinite(row["fetched_at"])): + raise ValueError("Invalid model catalog") + + +class StateStore: + """Share the control connection and lock without exposing secrets in public snapshots.""" + + def __init__(self, control): + self.control = control + self.path = control.path + self._db = control._db + self._lock = control._lock + + @contextmanager + def transaction(self): + with self._lock: + self._db.execute("BEGIN IMMEDIATE") + try: + yield + self._db.execute("COMMIT") + except BaseException: + if self._db.in_transaction: + self._db.execute("ROLLBACK") + raise + + def _get(self, name): + if name not in LEGACY_FILES: + raise ValueError("Unknown state namespace") + row = self._db.execute("SELECT payload FROM runtime_state WHERE name=?", (name,)).fetchone() + if row is None: + return None + value = strict_json(row[0]) + _check_document(name, value) + return value + + def get(self, name): + with self._lock: + return self._get(name) + + def _put(self, name, value): + content = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + _check_document(name, value) + if len(content.encode()) > MAX_BYTES: + raise ValueError("Runtime state exceeds size limit") + self._db.execute("INSERT INTO runtime_state(name,payload) VALUES(?,?) " + "ON CONFLICT(name) DO UPDATE SET payload=excluded.payload", (name, content)) + + def put(self, name, value): + with self.transaction(): + self._put(name, value) + + def put_cache(self, name, value): + """Cache failures stay visible without replacing the upstream inference outcome.""" + if name not in ("catalog", "model_blocks"): + raise ValueError("Not a rebuildable cache") + try: + self.put(name, value) + return True + except (OSError, ValueError, sqlite3.Error) as error: + warnings.warn(f"SQLite {name} cache write failed ({type(error).__name__}); keeping memory state", + RuntimeWarning, stacklevel=2) + return False + + + def delete(self, name): + if name not in LEGACY_FILES: + raise ValueError("Unknown state namespace") + with self.transaction(): + self._db.execute("DELETE FROM runtime_state WHERE name=?", (name,)) + + + def migrate(self, root): + """Import each legacy source once, including absence, before any maintenance starts.""" + root = Path(root) + with self.transaction(): + for name, filename in LEGACY_FILES.items(): + if self._db.execute("SELECT 1 FROM state_imports WHERE name=?", (name,)).fetchone(): + continue + path = root / filename + try: + metadata = path.lstat() + except FileNotFoundError: + raw = None + else: + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1 or metadata.st_size > MAX_BYTES: + raise ValueError(f"Cannot migrate {filename}: invalid file") + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + with os.fdopen(fd, "rb") as stream: + opened = os.fstat(stream.fileno()) + if (not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino)): + raise ValueError(f"Cannot migrate {filename}: changed file") + raw = stream.read(MAX_BYTES + 1) + if len(raw) > MAX_BYTES: + raise ValueError(f"Cannot migrate {filename}: size limit") + if raw is not None: + try: + value = strict_json(raw) + _check_document(name, value) + if self._get(name) is None: + self._put(name, value) + except (ValueError, TypeError, KeyError, RecursionError): + raise ValueError(f"Cannot migrate {filename}: invalid state; restore a valid backup") from None + self._db.execute("INSERT INTO state_imports(name,imported,migrated_at) VALUES(?,?,?)", + (name, int(raw is not None), time.time())) + + def default_key(self, *, create=False): + with self.transaction(): + row = self._db.execute("SELECT value,announced FROM gateway_secrets WHERE name='default_api_key'").fetchone() + if row is not None: + if not re.fullmatch(r"cb-[0-9a-f]{32}", row[0]) or row[1] not in (0, 1): + raise ValueError("Invalid saved API key; restore the control database") + return row[0], not bool(row[1]) + if not create: + return None, False + key = "cb-" + secrets.token_hex(16) + self._db.execute("INSERT INTO gateway_secrets(name,value,announced) VALUES('default_api_key',?,0)", (key,)) + return key, True + + def claim_announcement(self, key): + """Commit before printing: never repeat a secret after a crash or concurrent startup.""" + with self.transaction(): + return self._db.execute("UPDATE gateway_secrets SET announced=1 WHERE name='default_api_key' " + "AND value=? AND announced=0", (key,)).rowcount == 1 diff --git a/app/trial_rewards.py b/app/trial_rewards.py index ea5f76d..356909c 100644 --- a/app/trial_rewards.py +++ b/app/trial_rewards.py @@ -150,8 +150,9 @@ def __init__(self, result): class TrialLedger: """Persist a bounded JSON ledger under one cross-process read-modify-write lock.""" - def __init__(self, path): - path = Path(path) + def __init__(self, path=None, *, store=None): + self._store = store + path = Path(store.path if store is not None else path) if not path.name or path.name in (".", ".."): raise ValueError("Trial ledger requires a file path") # Resolve the parent without following a symlink at the final filename. @@ -160,9 +161,13 @@ def __init__(self, path): self._lock_name = "trial-" + hashlib.sha256(self.path.name.encode()).hexdigest() + ".info" def _lock(self): + if self._store is not None: + return self._store.transaction() return credential_file_lock(self.path.parent, self._lock_name) def _load(self): + if self._store is not None: + return (self._store.get("trial") or {"accounts": {}})["accounts"] flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) try: fd = os.open(self.path, flags) @@ -208,6 +213,9 @@ def _save(self, accounts): separators=(",", ":"), allow_nan=False).encode() if len(content) > _MAX_BYTES: raise ValueError("Trial ledger exceeds size limit") + if self._store is not None: + self._store._put("trial", {"version": 1, "accounts": accounts}) + return fd, temporary = tempfile.mkstemp(prefix=".trial-", suffix=".tmp", dir=self.path.parent) try: with os.fdopen(fd, "wb") as stream: diff --git a/app/usage_snapshots.py b/app/usage_snapshots.py index b4f840d..864ec53 100644 --- a/app/usage_snapshots.py +++ b/app/usage_snapshots.py @@ -17,7 +17,7 @@ import json import os import re -import stat +import sqlite3 import tempfile import threading import time @@ -89,8 +89,9 @@ def no_constants(name): class UsageSnapshots: """Thread-safe, optionally persisted cache of per-account usage snapshots.""" - def __init__(self, path=None): - self.path = str(path) if path else None + def __init__(self, path=None, *, store=None): + self._store = store + self.path = str(store.path) if store is not None else (str(path) if path else None) self._lock = threading.RLock() self._data: dict[str, dict] = {} self.last_error: str | None = None @@ -103,26 +104,8 @@ def __init__(self, path=None): def _load(self): """Adopt only a fully valid snapshot; anything else leaves the cache empty.""" - try: - if not stat.S_ISREG(os.lstat(self.path).st_mode): - return - fd = os.open(self.path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - | getattr(os, "O_NONBLOCK", 0)) - except OSError: - return - try: - with os.fdopen(fd, "rb") as stream: - if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): - return - raw = stream.read(MAX_BYTES + 1) - except OSError: - return - if len(raw) > MAX_BYTES: - return - try: - document = _strict_json(raw) - except (ValueError, UnicodeDecodeError, RecursionError): - return + from .state_store import load_document + document = load_document(self.path, self._store, "usage", MAX_BYTES) if (not isinstance(document, dict) or set(document) != {"version", "accounts"} or type(document["version"]) is not int or document["version"] != VERSION): return @@ -212,6 +195,16 @@ def _save_locked(self) -> bool: self.last_error = "serialize" self._dirty = True return False + if self._store is not None: + try: + self._store.put("usage", json.loads(content)) + except (OSError, ValueError, sqlite3.Error) as error: + self.last_error = type(error).__name__ + self._dirty = True + return False + self.last_error = "capacity" if shed else None + self._dirty = shed + return not shed temporary = None try: directory = os.path.dirname(self.path) or "." diff --git a/converter.py b/converter.py index f485007..e81ab88 100644 --- a/converter.py +++ b/converter.py @@ -27,7 +27,7 @@ from fastapi.exception_handlers import http_exception_handler as _default_http_exception_handler from fastapi.responses import JSONResponse, Response, StreamingResponse from starlette.concurrency import run_in_threadpool -import uvicorn +import uvicorn as uvicorn # Keep the existing embedding/test hook. try: from app.desensitize import desensitize_body @@ -72,6 +72,7 @@ def desensitize_body(body, roles=("system",), desensitize_harness_user=False, from app.content_filter import ContentFilterDetector, is_filter_error from app.request_limits import ImageLimitError, apply_image_policy from app.safe_logging import format_log_body, sanitize_log_text +from app.startup import load_startup_env, run_server from app.site_routing import (DOMESTIC, INTERNATIONAL, PROFILE_ENDPOINTS, site_for_auth, site_for_headers, profile_for_auth, profile_for_headers, profile_region, profile_product, profile_site, chat_url_for_headers, refresh_url_for_auth) @@ -466,15 +467,15 @@ class CredentialPool: """Manage credential discovery, reloads, sticky sessions, cooldowns and refresh.""" def __init__(self, paths: list[Path] | None = None, scan: bool = False, - blocks_path: Path | None = None, cooldowns_path: Path | None = None): + blocks_path: Path | None = None, cooldowns_path: Path | None = None, *, state_store=None): self._lock = threading.RLock() self._entries: list[dict] = [] # {id, cm, fail_until} self._sticky: "OrderedDict[str, tuple[str, float]]" = OrderedDict() self._model_fail: dict[tuple[str, str], float] = {} # Per-credential/model 429 expiry # Keep unsupported-model backoff isolated by backend and model. - self._blocks = ModelBlocks(blocks_path, ttl_s=MODEL_SITE_BLOCK_S, max_ttl_s=MODEL_SITE_BLOCK_MAX_S) + self._blocks = ModelBlocks(blocks_path, ttl_s=MODEL_SITE_BLOCK_S, max_ttl_s=MODEL_SITE_BLOCK_MAX_S, store=state_store) # Cooldowns outlive a restart so a backend that just refused is not retried immediately. - self._cooldowns = CredentialCooldowns(cooldowns_path) + self._cooldowns = CredentialCooldowns(cooldowns_path, store=state_store) self._storage_warned = 0.0 # Rate limit for persistence-failure warnings self._rr = {None: 0, "cn": 0, "intl": 0} self._ledger = None # Prefer credits expiring sooner. @@ -1795,48 +1796,19 @@ async def _protocol_http_exception(request: Request, exc: HTTPException): # --------------------------------------------------------------------------- -# File logging +# Runtime audit events # --------------------------------------------------------------------------- -_LOG_LOCK = threading.Lock() -LOG_MAX_BYTES = 50 * 1024 * 1024 # Rotation threshold; a single entry may exceed it. -LOG_BACKUPS = 2 # Retain the two most recent rotated logs. def _log(msg: str): - """Write bounded redacted logs with rotation under a shared lock.""" + """Persist allowlisted runtime events in SQLite, never free-form text or secrets.""" audit = CONFIG.get("audit_store") component = re.match(r"\[(cred|credits|models|usage|trial|checkin|housekeeper)\]", msg) if audit is not None and component: # Persist event codes, not free-form lines which may contain upstream data. code = "cooldown" if "熔断" in msg or "冷却" in msg else "failure" if "失败" in msg or "异常" in msg else "updated" audit.event("runtime", component.group(1), {"code": code}) - path = CONFIG.get("log_path") - if not path: - return - budget = min(max(1024, CONFIG.get("log_body_limit", 65536) + 256), max(0, LOG_MAX_BYTES - 256)) - msg = sanitize_log_text(msg, budget) - line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n" - try: - with _LOG_LOCK: - try: - size = os.path.getsize(path) - except FileNotFoundError: - size = 0 - rotated = size > 0 and size + len(line.encode("utf-8")) > LOG_MAX_BYTES - if rotated: - for i in range(LOG_BACKUPS - 1, 0, -1): - old = f"{path}.{i}" - if os.path.exists(old): - os.replace(old, f"{path}.{i + 1}") - os.replace(path, f"{path}.1") - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) - with os.fdopen(fd, "a", encoding="utf-8", newline="\n") as stream: - if rotated: - stream.write(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ==== 日志轮转 ====\n") - stream.write(line) - except OSError: - pass # Logging failures must not interrupt requests. def _log_json(label: str, value): @@ -3777,6 +3749,7 @@ def _boolean_arg(value): def main(): + dotenv_keys = set() if any(arg in ("-h", "--help") for arg in sys.argv[1:]) else load_startup_env() ap = argparse.ArgumentParser(description="CodeBuddy -> OpenAI 兼容转换器（直连后端）") ap.add_argument("command", nargs="?", choices=("serve", "login"), default="serve", help="serve 启动服务（默认）；login 扫码登录、自动轮询并保存账号") @@ -3787,7 +3760,7 @@ def main(): ap.add_argument("--host", default="127.0.0.1", help="监听地址；覆盖 CODEBUDDY2API_BIND") ap.add_argument("--port", type=int, default=8787, help="监听端口；覆盖 CODEBUDDY2API_PORT") ap.add_argument("--api-key", default=os.environ.get("CODEBUDDY2API_KEY", ""), - help="可选：要求客户端携带的 API key（默认不校验）") + help="管理与推理密钥；未配置时首次本地交互启动生成并保存") ap.add_argument("--admin-csrf", type=_boolean_arg, nargs="?", const=True, default=os.environ.get("CODEBUDDY2API_ADMIN_CSRF", "true"), help="管理 Origin/CSRF 校验，默认 true；仅在受信任本地环境设为 false，鉴权仍启用") @@ -3796,8 +3769,7 @@ def main(): help="额外信任的管理页来源（逗号分隔，支持域名或完整来源，裸域名按 https）；" "反代 HTTPS 域名登录报 Origin 校验失败时设置，也可在 WebUI 配置") ap.add_argument("--log", default=None, metavar="PATH", - help="额外写入兼容文本日志（如 --log converter.log 或 --log /tmp/cb.log）。" - "不传仍记录 SQLite 审计，但不输出文本文件。") + help="已停用：日志统一保存到数据目录中的 logs.sqlite3") ap.add_argument("--desensitize", action="store_true", help="适配固定 CLI 模板、压缩运行时提示并零宽脱敏关键词。默认关闭。") ap.add_argument("--no-compact", action="store_true", @@ -3851,7 +3823,7 @@ def main(): help="按账号模型声明预检图片、工具、思考和输出上限；false 仅关闭新增能力预检") ap.add_argument("--log-body-limit", type=_nonnegative_int, metavar="BYTES", default=os.environ.get("CODEBUDDY2API_LOG_BODY_LIMIT", "65536"), - help="每条正文日志的预览字节上限，默认 64 KiB；0 只记录摘要") + help="旧文本预览兼容项；文本输出已停用，SQLite 诊断使用独立预算") ap.add_argument("--tool-call-max-retry", type=_nonnegative_int, metavar="N", default=os.environ.get("CODEBUDDY2API_TOOL_CALL_MAX_RETRY", "3"), help="工具参数损坏时的额外生成上限，默认 3；0 表示不重试（每次额外生成都消耗额度）") @@ -3888,10 +3860,11 @@ def main(): CONFIG["usd_rate"] = args.usd_rate or None CONFIG["credit_price_usd"] = args.credit_price_usd or None CONFIG["model_guard"] = not args.no_model_guard - # File logging is enabled only when a path is configured. - CONFIG["log_path"] = args.log if args.log else os.environ.get("CODEBUDDY2API_LOG") + if args.log is not None or os.environ.get("CODEBUDDY2API_LOG"): + sys.stderr.write("[log] --log / CODEBUDDY2API_LOG 已停用；请在 WebUI 查看 SQLite 日志。\n") + CONFIG["log_path"] = None from app import runtime_management - runtime_management.initialize(sys.modules[__name__], args, parser=ap) + runtime_management.initialize(sys.modules[__name__], args, parser=ap, dotenv_keys=dotenv_keys) # Validate effective binding and authentication before credential scans or background work. if (args.host not in ("127.0.0.1", "::1", "localhost") and not CONFIG.get("api_key") and os.environ.get("CODEBUDDY2API_ALLOW_OPEN_NOAUTH", "").lower() not in ("1", "true", "yes")): @@ -3901,17 +3874,14 @@ def main(): files = [Path(p) for p in args.auth_file] if not files: seed_credentials() # Seed missing desktop credentials into managed storage. - CONFIG["cred_pool"] = CredentialPool(files, scan=not files, - blocks_path=managed_auth_dir() / "model-site-blocks.json", - cooldowns_path=managed_auth_dir() / "credential-cooldowns.json") - CONFIG["usage_snapshots"] = UsageSnapshots(managed_auth_dir() / "usage-snapshots.json") + CONFIG["cred_pool"] = CredentialPool(files, scan=not files, state_store=CONFIG["state_store"]) + CONFIG["usage_snapshots"] = UsageSnapshots(store=CONFIG["state_store"]) CONFIG["cred"] = CONFIG["cred_pool"].first() CONFIG["account_catalogs"] = {} # Disable static fallback before maintenance starts. if credits_mod is not None: - ledger = credits_mod.CreditLedger(managed_auth_dir() / "credits-ledger.json") + ledger = credits_mod.CreditLedger(store=CONFIG["state_store"]) CONFIG["ledger"] = ledger - CONFIG["model_cache"] = credits_mod.ModelCatalogCache( - managed_auth_dir() / "model-catalog.json", ttl=args.model_catalog_ttl) + CONFIG["model_cache"] = credits_mod.ModelCatalogCache(ttl=args.model_catalog_ttl, store=CONFIG["state_store"]) CONFIG["cred_pool"].set_ledger(ledger) # Verify balance ownership before publishing catalogs. _publish_model_cache() # Publish cached usage before maintenance threads start, so the dashboard is populated from @@ -3968,7 +3938,7 @@ def main(): _log(f"==== converter 启动 ====") try: - uvicorn.run(app, host=args.host, port=args.port, log_level="warning") + run_server(app, CONFIG, host=args.host, port=args.port) finally: runtime_management.close(CONFIG) diff --git a/docs/advanced.md b/docs/advanced.md index f1655e0..30bb987 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -6,18 +6,18 @@ Use the [WebUI](webui.md) for everyday management. See [deployment](deployment.m ## Configuration and CLI -Precedence: **explicit CLI flags > environment > persisted WebUI settings > defaults**. Hot settings apply immediately; restart-marked settings require a manual restart. Change locked options in the startup configuration; the WebUI does not edit `.env`. +Precedence: **explicit CLI flags > process environment > `.env` > saved SQLite settings > defaults**. Hot settings apply immediately; restart-marked settings require a manual restart. Change locked options at their source; the WebUI does not edit `.env` or expose API keys. Compose explicitly passes some environment variables and CLI flags, so deleting a line from `.env` may not unlock it. Recreate the container after changing these values; to let the WebUI manage them, also remove the corresponding explicit Compose settings. | Flag | Default | Description | |------|---------|-------------| | `--host` / `--port` | `127.0.0.1` / `8787` | Local listener | -| `--api-key` | none | Shared management and inference key; management is locked without it | +| `--api-key` | saved default | Shared management/inference key; first local interactive startup generates and saves one if absent; explicit empty locks management | | `--admin-csrf [true/false]` | `true` | Startup-only management Origin/CSRF checks; disabling does not bypass API-key or session authentication | | `--admin-allowed-origins` | none | Extra trusted management Origins (comma-separated; bare domains mean HTTPS) for reverse-proxy sign-in; hot and WebUI-editable | | `--auth-file` | scan `auth/` | Explicit credential file, repeatable; disables scanning other files | -| `--log` | none | Additional text logs, 50 MiB rotation and 2 backups; SQLite auditing remains enabled | +| `--log` | retired | Warns without writing a file; use SQLite logs in the WebUI | | `--desensitize` | off | Adapt fixed CLI templates, compact runtime prompts and mask keywords with zero-width characters | | `--no-compact` | off | With desensitization, retain fuller instructions while adapting templates and pruning runtime context; does not disable Responses projection | | `--keep-tool-metadata [true/false]` | `false` | Retain tool descriptions and parameter-schema `description/title`, independently of prompt compaction | @@ -40,7 +40,7 @@ Compose explicitly passes some environment variables and CLI flags, so deleting | `--failover-max` | `0` | Extra credentials tried when a request fails before the first response byte reaches the client; `0` keeps the upstream behaviour of surfacing the failure directly | | `--retry-write-timeout` | `false` | Opt a request-body write timeout into replay (fresh connection and `--failover-max`), accepting that bytes already sent may have been processed | | `--max-request-bytes` | `33554432` | Positive byte limit for the processed upstream JSON | -| `--log-body-limit` | `65536` | Text-log body preview bytes; `0` logs summaries only, not the SQLite diagnostic budget | +| `--log-body-limit` | `65536` | Legacy text-preview option; text output is retired and SQLite diagnostics use their own budget | Environment variables include `CODEBUDDY_AUTH_DIR`, `CODEBUDDY_IMPORT_DIR`, `CODEBUDDY2API_KEY`, `CODEBUDDY2API_ADMIN_CSRF`, `CODEBUDDY2API_ADMIN_ORIGINS`, `CODEBUDDY2API_KEEP_TOOL_METADATA`, `CODEBUDDY2API_LOG`, `CODEBUDDY2API_MAX_IMAGES`, `CODEBUDDY2API_IMAGE_POLICY`, `CODEBUDDY2API_MAX_REQUEST_BYTES`, `CODEBUDDY2API_LOG_BODY_LIMIT`, `CODEBUDDY2API_FAILOVER_MAX` and `CODEBUDDY2API_RETRY_WRITE_TIMEOUT`. See [deployment](deployment.md) for startup examples. @@ -82,7 +82,7 @@ Manual `POST /admin/credentials/{id}/travel` returns `buddy_confirmation` with o Travel claims and departures share an account-scoped write reservation. Uncertain results do not expire or replay; fresh status reads reconcile them without issuing upstream writes. Store failures stop claims and departures, and local readback updates preserve receipt ownership across processes. -Trial credits are manual-only for eligible `intl-work` accounts: use the credential row's claim drawer or `POST /admin/credentials/{id}/trial`. Startup, periodic maintenance and balance sync never claim. Results expose safe error categories, HTTP/business codes and retry time; response bodies are capped at 64 KiB and never returned to the browser. Success/already-claimed records persist in `auth/trial-ledger.json`; failures wait at least 24 hours before another manual attempt. Keep this file when upgrading. +Trial credits are manual-only for eligible `intl-work` accounts through the credential drawer or `POST /admin/credentials/{id}/trial`; startup and maintenance never claim. Safe results, successful claims and reservations persist in `control.sqlite3`; failed attempts retain the 24-hour backoff. Response bodies are capped at 64 KiB and never returned to the browser. Preserve the database when upgrading. `CODEBUDDY2API_AUTO_TRIAL` and `--auto-trial` are retired: old startup options warn and do nothing; saved Boolean `auto_trial` settings are ignored on load. Remove them from deployment configuration. Before reverting to older code, check these old settings to avoid re-enabling automatic claims. @@ -147,7 +147,7 @@ The WebUI supports direct uploads; these rules concern path imports through `POS ## Models and scheduling -Select client models from the WebUI or `GET /v1/models`. Raw catalogs remain cached by account/tenant, region, product and client version in `auth/model-catalog.json`, with a default 6-hour TTL. New credentials trigger synchronization; failed refreshes retain that account's trusted cache. Legacy unscoped caches do not become international sharing sources. +Select client models from the WebUI or `GET /v1/models`. Raw catalogs remain cached by account/tenant, region, product and client version in `auth/control.sqlite3`, with a default 6-hour TTL. New credentials trigger synchronization; failed refreshes retain that account's trusted cache. Legacy unscoped caches do not become international sharing sources. International CLI and WorkBuddy use a deduplicated shared view from enabled, catalog-ready international accounts. A target account must have its own synchronized catalog; its existing model declarations win unchanged. Missing IDs inherit shared declarations, retaining `catalog_source` and safe `source_variants`. Conflicting inherited rates use the higher known rate, limits the smaller known value, reasoning options their intersection and differing descriptive fields are omitted; unknown prices never mean free. Domestic catalogs, credentials, balances, bindings and `auto` defaults remain independent. Shared rates are catalog references, not billing or permission guarantees. diff --git a/docs/advanced.zh-CN.md b/docs/advanced.zh-CN.md index 4c9cc6b..dd7f8fe 100644 --- a/docs/advanced.zh-CN.md +++ b/docs/advanced.zh-CN.md @@ -6,18 +6,18 @@ ## 配置与 CLI -配置优先级：**显式 CLI 参数 > 环境变量 > WebUI 持久化配置 > 默认值**。WebUI 中可热更新的设置立即生效，标记为重启生效的设置需手动重启；锁定项须在启动配置中修改，WebUI 不改写 `.env`。 +配置优先级：**显式 CLI 参数 > 进程环境变量 > `.env` > SQLite 保存值 > 默认值**。热更新设置立即生效，标记为重启的项需手动重启；锁定项请在对应来源修改，WebUI 不改写 `.env` 或返回 API key 明文。 Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的一行不一定解除锁定。修改这些值后重建容器；若要由 WebUI 接管，还需取消 Compose 中对应的显式设置。 | 参数 | 默认值 | 说明 | |------|--------|------| | `--host` / `--port` | `127.0.0.1` / `8787` | 本地监听地址与端口 | -| `--api-key` | 无 | 管理与推理共用密钥；未设置时管理锁定 | +| `--api-key` | 已保存的默认 key | 管理与推理共用；缺失时首次本地交互启动生成并保存；显式空值锁定管理 | | `--admin-csrf [true/false]` | `true` | 管理 Origin/CSRF 校验；仅启动配置可关闭，API key 和会话鉴权不变 | | `--admin-allowed-origins` | 无 | 额外信任的管理页来源（逗号分隔，裸域名按 HTTPS），用于反代登录；热生效，可在 WebUI 配置 | | `--auth-file` | 扫描 `auth/` | 指定凭据文件，可重复传入；不再扫描其他文件 | -| `--log` | 无 | 额外文本日志，50 MiB 轮转、保留 2 份；不影响默认 SQLite 审计 | +| `--log` | 已停用 | 提示弃用且不写文件；日志统一在 WebUI 查看 SQLite 记录 | | `--desensitize` | 关 | 适配固定 CLI 模板、压缩运行时提示、零宽脱敏关键词 | | `--no-compact` | 关 | 配合脱敏保留主要行为指令，仍适配模板及裁剪运行时上下文；不关闭 Responses 投影 | | `--keep-tool-metadata [true/false]` | `false` | 保留工具描述及参数 schema 的 `description/title`，与提示词压缩独立 | @@ -40,7 +40,7 @@ Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的 | `--failover-max` | `0` | 请求在「一个字节都还没发给下游」之前失败时，最多再换几个凭证就地重放；`0` 表示如实把失败回给下游 | | `--retry-write-timeout` | `false` | 让「写请求体超时」也参与重放（换新连接与 `--failover-max` 换凭证），代价是已发出的那半截正文可能已被上游处理 | | `--max-request-bytes` | `33554432` | 处理后的上游 JSON 字节上限，须为正整数 | -| `--log-body-limit` | `65536` | 兼容文本日志正文预览字节；`0` 只记摘要，不控制 SQLite 诊断预算 | +| `--log-body-limit` | `65536` | 旧文本预览兼容项；文本输出已停用，SQLite 诊断使用独立预算 | 环境变量包括 `CODEBUDDY_AUTH_DIR`、`CODEBUDDY_IMPORT_DIR`、`CODEBUDDY2API_KEY`、`CODEBUDDY2API_ADMIN_CSRF`、`CODEBUDDY2API_ADMIN_ORIGINS`、`CODEBUDDY2API_KEEP_TOOL_METADATA`、`CODEBUDDY2API_LOG`，以及 `CODEBUDDY2API_MAX_IMAGES`、`CODEBUDDY2API_IMAGE_POLICY`、`CODEBUDDY2API_MAX_REQUEST_BYTES`、`CODEBUDDY2API_LOG_BODY_LIMIT`、`CODEBUDDY2API_FAILOVER_MAX`、`CODEBUDDY2API_RETRY_WRITE_TIMEOUT`。启动示例见[部署指南](deployment.zh-CN.md)。 @@ -82,7 +82,7 @@ scoped 模式可选传入 `X-Codebuddy-Session-ID`、`metadata.conversation_id` 旅行奖励领取与派遣共用按账号隔离的写入预留，不确定结果不超时重放；状态查询只回查并更新本地确认记录，不发上游写请求。存储失败停止领取和派遣，跨进程确认不覆盖其他请求的记录。 -体验积分仅供符合官方资格的 `intl-work` 账号手动领取：使用凭证行的领取抽屉或 `POST /admin/credentials/{id}/trial`。启动、定时维护、余额同步均不领取。结果显示安全错误类别、HTTP 状态／业务码及重试时间，响应正文限制为 64 KiB 且不返回浏览器。成功或已领取记录保存在 `auth/trial-ledger.json`，失败至少等待 24 小时才能再次手动申请；升级时保留该文件。 +体验积分仅供符合资格的 `intl-work` 账号通过凭证抽屉或 `POST /admin/credentials/{id}/trial` 手动领取，启动和维护不领取。安全结果、成功历史与请求预留保存在 `control.sqlite3`，失败保留 24 小时退避。响应正文上限 64 KiB，且不返回浏览器；升级时保留数据库。 `CODEBUDDY2API_AUTO_TRIAL` 和 `--auto-trial` 已停用：旧启动选项仅提示、不触发任务；控制库中的旧布尔 `auto_trial` 设置在加载时忽略。请从部署配置中移除；回滚旧代码前也须核对这些旧设置，避免重新启用自动领取。 @@ -147,7 +147,7 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` ## 模型与调度 -以 WebUI 和 `GET /v1/models` 为客户端选择依据。原始目录仍按账号/租户、地域、产品与客户端版本缓存到 `auth/model-catalog.json`，默认有效期 6 小时；新凭据触发同步，刷新失败保留该账号的可信旧缓存。旧未隔离缓存不作为国际共享来源。 +以 WebUI 和 `GET /v1/models` 为客户端选择依据。原始目录按账号/租户、地域、产品与客户端版本缓存到 `auth/control.sqlite3`，默认有效期 6 小时；新凭据触发同步，刷新失败保留该账号的可信旧缓存。旧未隔离缓存不作为国际共享来源。 国际 CLI／WorkBuddy 使用已启用、目录已就绪的国际账号生成去重共享视图。目标账号须完成自身目录同步；已有型号保留自己的完整声明，缺失型号才继承，并通过 `catalog_source`、安全 `source_variants` 标明来源。继承声明冲突时倍率取较高值、上限取较小值、思考选项取交集、描述类字段不一致即省略；未知倍率不当零。国内目录、凭据、余额、绑定及 `auto` 默认模型保持独立；共享倍率只是目录参考，不保证权限或实际扣分。 diff --git a/docs/deployment.md b/docs/deployment.md index 95b5ac6..fe7b051 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -63,14 +63,22 @@ Requires Python 3.12+, uv, and Node.js with the vp CLI to build the interface: ```bash uv sync --locked --no-build --python 3.12 (cd web && vp install --frozen-lockfile && vp build) -uv run --locked --no-build --env-file .env converter.py --desensitize +uv run converter.py ``` -Configure `.env` as above before starting, then open `/dashboard` to add accounts. Rebuild the WebUI after changing frontend source. +Local startup does not require `.env`. Without an explicit key, the first loopback startup generates a `cb-…` default key in `auth/control.sqlite3` and displays it once in the interactive terminal after listening succeeds. Restarts reuse it without printing it again. Keep it safe: management and API requests share this key. Configure a key explicitly for first-time headless or non-loopback deployment. -Without uv, run `python3 -m venv .venv`, activate it, install dependencies with `pip install --require-hashes --only-binary=:all: -r requirements.txt`, and start with `python3 converter.py --desensitize`. **Plain Python does not load `.env`**; export environment variables or pass CLI flags explicitly. +Without uv, run `python3 -m venv .venv`, activate it, install dependencies with `pip install --require-hashes --only-binary=:all: -r requirements.txt`, then run `python3 converter.py`. Distribution archives include the WebUI; source installs still need the frontend build. -Native binding follows explicit `--host/--port` > `CODEBUDDY2API_BIND/PORT` > saved WebUI values > defaults. Remove explicit flags if `.env` should control the listener; changes require restart. `CODEBUDDY2API_IMAGE/AUTH_PATH` remain Compose-only; use `CODEBUDDY_AUTH_DIR` for native data. +Both commands optionally read `.env` in the current working directory, never parent directories. Precedence: explicit CLI > process environment > `.env` > saved SQLite values > defaults. Overrides do not replace the saved default key; an explicitly empty key still locks management. Listener changes require restart; `CODEBUDDY2API_IMAGE/AUTH_PATH` are Compose-only. + +### Upgrade and state migration + +Stop the gateway and back up the entire data directory first. Startup imports legacy JSON sessions, cooldowns, credit/trial history, catalogs and usage into `control.sqlite3` once. Old files remain as backups and are no longer read or updated. Invalid critical state or a failed migration stops startup; repair it or restore a backup rather than deleting ledgers to bypass validation. + +All runtime logs use `logs.sqlite3`; legacy `--log` / `CODEBUDDY2API_LOG` settings warn and no longer write text files. Generated keys enter private configuration, never logs. Restrict access to the data directory; use a private user directory on Windows. + +Older versions cannot read the upgraded control database. Downgrades require a stopped gateway and matching state migration, not merely old code reading stale JSON. Never overwrite new claims, dispatches or session revocations with an outdated backup. ## Dependency locks diff --git a/docs/deployment.zh-CN.md b/docs/deployment.zh-CN.md index 6c96056..747b13c 100644 --- a/docs/deployment.zh-CN.md +++ b/docs/deployment.zh-CN.md @@ -63,14 +63,22 @@ docker compose up -d ```bash uv sync --locked --no-build --python 3.12 (cd web && vp install --frozen-lockfile && vp build) -uv run --locked --no-build --env-file .env converter.py --desensitize +uv run converter.py ``` -按上文配置 `.env` 后启动，打开 `/dashboard` 添加账号。修改前端源码后需重新构建 WebUI。 +本地启动无需 `.env`。没有显式 key 时，首次回环启动会生成 `cb-…` 默认密钥，保存到 `auth/control.sqlite3`，监听成功后仅在交互终端显示一次；以后重启复用且不再打印。请妥善保存，管理登录与 API 请求共用。首次后台或非回环部署请显式配置 key。 -没有 uv 时：运行 `python3 -m venv .venv` 并激活，用 `pip install --require-hashes --only-binary=:all: -r requirements.txt` 安装依赖，以 `python3 converter.py --desensitize` 启动。**纯 Python 不会加载 `.env`**；请显式导出环境变量或传递 CLI 参数。 +没有 uv 时：运行 `python3 -m venv .venv` 并激活，用 `pip install --require-hashes --only-binary=:all: -r requirements.txt` 安装依赖，再执行 `python3 converter.py`。发行包已包含 WebUI；源码安装仍需构建界面。 -本地监听地址优先级为：显式 `--host/--port` > `CODEBUDDY2API_BIND/PORT` > WebUI 已保存值 > 默认值。希望由 `.env` 控制监听时去掉显式参数；修改需重启。`CODEBUDDY2API_IMAGE/AUTH_PATH` 仅用于 Compose；本地数据目录用 `CODEBUDDY_AUTH_DIR`。 +两种启动方式均可选读取当前工作目录的 `.env`，不搜索父目录。优先级：显式 CLI > 进程环境变量 > `.env` > SQLite 保存值 > 默认值。覆盖不改写已保存的默认 key；显式空 key 仍锁定管理。修改监听需重启，`CODEBUDDY2API_IMAGE/AUTH_PATH` 仅用于 Compose。 + +### 升级与数据迁移 + +升级前停止网关并备份整个数据目录。首次启动将旧 JSON 会话、冷却、积分/领取、目录和用量状态一次性导入 `control.sqlite3`；原文件保留作备份，不再参与读写。关键数据损坏或迁移失败会阻止启动，请修复或恢复备份，不要删除账本绕过检查。 + +所有运行日志使用 `logs.sqlite3`；旧 `--log` / `CODEBUDDY2API_LOG` 仅提示弃用，不再输出文本文件。自动生成的 key 只进私有配置，不进任何日志。限制数据目录访问权限；Windows 应使用当前用户的私有目录。 + +旧版本不能读取升级后的控制库。回滚须停机并迁回匹配的状态，不能只切代码后复用旧 JSON；升级后发生的领取、派遣或会话撤销不能用过期备份覆盖。 ## 依赖锁定 diff --git a/docs/webui.md b/docs/webui.md index 27e4978..70af9e3 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -6,7 +6,7 @@ Start the gateway using the [deployment guide](deployment.md), then open `http://127.0.0.1:8787/dashboard` and sign in with the current API key. Source installs need a frontend build; Docker builds include it. Use HTTPS unless connecting locally. -Management is locked without a key. After changing it, sign in again and restart any unfinished OAuth login. +The first unconfigured local startup generates and saves a default key, shown once in the terminal and reused afterward. An explicitly empty key locks management. After changing the effective key, sign in again and restart unfinished OAuth logins. ## Overview @@ -74,15 +74,15 @@ Data defaults to `auth/`, or `/data/auth` inside Docker. Local installs can set | File | Contents | |------|----------| | `*.info` | Official plaintext credentials; never migrated into SQLite | -| `control.sqlite3` | Gateway settings, model rules, credential metadata and first-Buddy reservations | +| `control.sqlite3` | Settings, private default key, model rules, sessions, cooldowns, credit/reward state, usage and catalogs | | `logs.sqlite3` | Request details and independent aggregate statistics | Auditing defaults to 30-day detail retention and a 256 MiB logical detail budget, **not a hard limit on database or directory disk usage**. Detail cleanup and eviction preserve aggregates. SQLite failure diagnostics have a separate budget, defaulting to 8192 bytes. Existing text logs are retained, not backfilled as precise statistics. Mount the whole data directory on writable local storage, not just a single database file, and do not share it between gateway instances. -Stop the gateway before copying the entire directory, including databases, any WAL/SHM files, credentials and catalog/credit state files; do not back up only `.info` files. Keep this private data secure. +Stop the gateway before copying the entire directory, including databases, WAL/SHM files, credentials and migration backups; do not back up only `.info`. Protect the control database: it contains the default key and management sessions. -Back up control metadata before using new model rules or automation preferences. Reverting to older code requires the matching control-database snapshot, including its WAL/SHM state without mixing files; older readers reject the new fields. Rollback cannot undo completed upstream check-ins, claims or travel dispatches. +Legacy JSON is imported once; SQLite is authoritative afterward. Downgrades require a stopped gateway and matching state migration, never stale JSON overwriting new claims or revoked sessions. See [upgrade and state migration](deployment.md#upgrade-and-state-migration). See [client configuration](clients.md) for API keys and URLs. diff --git a/docs/webui.zh-CN.md b/docs/webui.zh-CN.md index 0a6f2db..4a5e471 100644 --- a/docs/webui.zh-CN.md +++ b/docs/webui.zh-CN.md @@ -6,7 +6,7 @@ 按[部署指南](deployment.zh-CN.md)启动网关后，打开 `http://127.0.0.1:8787/dashboard`，使用当前 API key 登录。源码安装需要先构建前端；Docker 镜像已包含。非本机访问请使用 HTTPS。 -未设置 key 时管理功能锁定。更换 key 后需重新登录，并重新开始未完成的 OAuth 登录。 +本地零配置首次启动会生成并保存默认 key，仅在终端显示一次；以后复用。显式空 key 锁定管理。更换生效 key 后需重新登录，并重新开始未完成的 OAuth 登录。 ## 概览 @@ -74,15 +74,15 @@ | 文件 | 内容 | |------|------| | `*.info` | 官方明文凭证；绝不迁移进 SQLite | -| `control.sqlite3` | 网关设置、模型规则、凭证元数据与首次领养预留 | +| `control.sqlite3` | 网关设置、私有默认 key、模型规则、会话、冷却、积分/领取、用量与目录缓存 | | `logs.sqlite3` | 请求明细与独立的聚合统计 | 审计默认保留 30 天明细与 256 MiB 逻辑明细预算，**不是数据库或目录磁盘占用的硬上限**。明细清理与淘汰都保留聚合。SQLite 失败诊断有独立预算，默认 8192 字节。已有文本日志保留原样，不回填为精确统计。 将整个数据目录挂载到可写本地存储，而不是只挂载单个数据库文件；不要在多个网关实例间共享。 -复制前停止网关，并复制整个目录——包括数据库、WAL/SHM 文件、凭证以及目录/积分状态文件；不要只备份 `.info`。这些数据属于隐私，妥善保管。 +复制前停止网关，并复制整个目录，包括数据库、WAL/SHM、凭证及迁移备份；不要只备份 `.info`。控制库包含默认密钥与登录会话，必须妥善保管。 -使用新的模型规则或自动化开关前备份控制元数据。回退到旧代码需要匹配的控制数据库快照（含 WAL/SHM，不混用文件）；旧版本会拒绝新字段。回滚无法撤销已完成的上游签到、领取或旅行派出。 +旧 JSON 只在首次升级时导入；迁移后的运行状态以 SQLite 为准。回滚须停机并迁回匹配状态，不能用旧 JSON 覆盖新的领取或会话撤销记录，见[升级与数据迁移](deployment.zh-CN.md#升级与数据迁移)。 API key 与地址见[客户端配置](clients.zh-CN.md)。 diff --git a/pyproject.toml b/pyproject.toml index 0a7312f..2537642 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ dependencies = [ "fastapi>=0.141.1", "httpx>=0.28.1", "pytest>=9.1.1", + "python-dotenv>=1.2.3", "uvicorn[standard]>=0.52.4", ] diff --git a/requirements.in b/requirements.in index 13a8f8d..46362e5 100644 --- a/requirements.in +++ b/requirements.in @@ -2,4 +2,5 @@ fastapi>=0.141.1 httpx>=0.28.1 pytest>=9.1.1 +python-dotenv>=1.2.3 uvicorn[standard]>=0.52.4 diff --git a/requirements.txt b/requirements.txt index 64fa7c5..82e7f1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -178,7 +178,9 @@ pytest==9.1.1 \ python-dotenv==1.2.3 \ --hash=sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9 \ --hash=sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35 - # via uvicorn + # via + # codebuddy2api + # uvicorn pyyaml==6.0.3 \ --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ diff --git a/tests/test_control_store.py b/tests/test_control_store.py index b16ae07..0bb1176 100644 --- a/tests/test_control_store.py +++ b/tests/test_control_store.py @@ -23,7 +23,7 @@ def test_reopen_and_schema(self): state = self.store.update_settings({"max_images": 3}, 0) self.assertEqual(state["revision"], 1) with sqlite3.connect(self.path) as db: - self.assertEqual(db.execute("PRAGMA user_version").fetchone()[0], 1) + self.assertEqual(db.execute("PRAGMA user_version").fetchone()[0], ControlStore.SCHEMA_VERSION) other = ControlStore(self.path) self.addCleanup(other.close) self.assertEqual(other.snapshot()["settings"], {"max_images": 3}) diff --git a/tests/test_environment_config.py b/tests/test_environment_config.py index 3057c18..469e906 100644 --- a/tests/test_environment_config.py +++ b/tests/test_environment_config.py @@ -36,6 +36,7 @@ def start(self, environ=None, cli=(), saved=None): env.update(environ or {}) with patch.dict(os.environ, env, clear=True), patch.dict(converter.CONFIG, dict(converter.CONFIG), clear=True), \ patch.object(sys, 'argv', ['converter.py', '--skip-check', *cli]), \ + patch.object(converter, 'load_startup_env', return_value=set()), \ patch.object(converter, 'seed_credentials') as seed, patch.object(converter, 'CredentialPool') as pool, \ patch.object(converter, '_publish_model_cache'), patch.object(runtime_management, 'install'), \ patch.object(converter.threading, 'Thread'), patch.object(converter, '_log'), \ diff --git a/tests/test_identity_sync.py b/tests/test_identity_sync.py index 4a1da3c..eda4be0 100644 --- a/tests/test_identity_sync.py +++ b/tests/test_identity_sync.py @@ -52,6 +52,7 @@ def setUp(self): self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) self.enterContext(patch.dict(os.environ, {"CODEBUDDY_AUTH_DIR": str(self.root), "CODEBUDDY2API_LOG": "", "CODEBUDDY2API_KEY": ""})) + self.enterContext(patch.object(c, "load_startup_env", return_value=set())) self.enterContext(patch.dict(c.CONFIG, {"cred_pool": None, "cred": None, "ledger": None, "model_cache": None, "model_catalogs": {}, "account_catalogs": None, "models_remote": None, "models_intl": None, "model_guard": True, "log_path": None})) @@ -403,8 +404,12 @@ def test_main_publishes_bound_empty_or_nonempty_cache_before_any_thread_or_serve ledger = credits.CreditLedger(self.root / "credits-ledger.json") ledger.bind_identity(str(path), identity) ledger.update_credits(str(path), balance("", domain=DOMAINS["intl-cli"])) + from app.control_store import ControlStore + control = ControlStore(self.root / "control.sqlite3") + self.addCleanup(control.close) + control.state.migrate(self.root) for models in ([], [model("shared")]): - cache = credits.ModelCatalogCache(self.root / "model-catalog.json") + cache = credits.ModelCatalogCache(store=control.state) cache.put(key, models) cache.put("international", [model("legacy-forbidden")]) observed = [] diff --git a/tests/test_login.py b/tests/test_login.py index c18f7ca..b565873 100644 --- a/tests/test_login.py +++ b/tests/test_login.py @@ -26,6 +26,7 @@ def setUp(self): self.addCleanup(temporary.cleanup) self.directory = Path(temporary.name) self.enterContext(patch.dict(os.environ, {"CODEBUDDY_AUTH_DIR": temporary.name})) + self.enterContext(patch.object(converter, "load_startup_env", return_value=set())) self.enterContext(patch.dict(converter.CONFIG, {"cred_pool": None, "api_key": "", "log_path": None})) self.stdout = self.enterContext(contextlib.redirect_stdout(io.StringIO())) self.stderr = self.enterContext(contextlib.redirect_stderr(io.StringIO())) diff --git a/tests/test_runtime_endpoints.py b/tests/test_runtime_endpoints.py index 3b8dab5..201fee2 100644 --- a/tests/test_runtime_endpoints.py +++ b/tests/test_runtime_endpoints.py @@ -786,7 +786,8 @@ class StartupUsageHydrationTests(unittest.TestCase): def setUp(self): self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) - self.enterContext(patch.dict(os.environ, {"CODEBUDDY_AUTH_DIR": str(self.root)}, clear=False)) + self.enterContext(patch.dict(os.environ, {"CODEBUDDY_AUTH_DIR": str(self.root), "CODEBUDDY2API_KEY": "synthetic-key"}, clear=True)) + self.enterContext(patch.object(converter, "load_startup_env", return_value=set())) self.path = self.root / "usage-snapshots.json" def credential(self, uid="synthetic-uid"): @@ -819,9 +820,6 @@ def test_main_publishes_cached_usage_before_serving(self): stack.enter_context(patch.object(converter, "credits_mod", None)) # No network. stack.enter_context(patch.object(converter, "seed_credentials")) stack.enter_context(patch.object(converter, "preflight", return_value=True)) - # Keep this test on startup ordering: skip the SQLite-backed stores so no file - # handle is left open on Windows. - stack.enter_context(patch.object(runtime_management, "initialize")) stack.enter_context(patch.object(runtime_management, "install")) def capture(*args, **kwargs): @@ -850,9 +848,6 @@ def test_main_without_a_cache_leaves_usage_empty(self): stack.enter_context(patch.object(converter, "credits_mod", None)) stack.enter_context(patch.object(converter, "seed_credentials")) stack.enter_context(patch.object(converter, "preflight", return_value=True)) - # Keep this test on startup ordering: skip the SQLite-backed stores so no file - # handle is left open on Windows. - stack.enter_context(patch.object(runtime_management, "initialize")) stack.enter_context(patch.object(runtime_management, "install")) stack.enter_context(patch.object(converter.uvicorn, "run", side_effect=lambda *a, **k: observed.setdefault("usage", @@ -878,9 +873,6 @@ def test_main_starts_no_maintenance_thread_before_hydration(self): stack.enter_context(patch.object(converter, "credits_mod", None)) stack.enter_context(patch.object(converter, "seed_credentials")) stack.enter_context(patch.object(converter, "preflight", return_value=True)) - # Keep this test on startup ordering: skip the SQLite-backed stores so no file - # handle is left open on Windows. - stack.enter_context(patch.object(runtime_management, "initialize")) stack.enter_context(patch.object(runtime_management, "install")) real_thread = converter.threading.Thread @@ -908,7 +900,8 @@ def configure(self, env=None, flags=(), invalid=False, stored=None, expected_hos store.close() stack.enter_context(patch.object(converter, "managed_auth_dir", return_value=Path(directory))) stack.enter_context(patch.object(converter, "app", FastAPI())) - stack.enter_context(patch.dict(os.environ, env or {}, clear=True)) + stack.enter_context(patch.dict(os.environ, {"CODEBUDDY2API_KEY": "", **(env or {})}, clear=True)) + stack.enter_context(patch.object(converter, "load_startup_env", return_value=set())) stack.enter_context(patch.dict(converter.CONFIG)) stack.enter_context(patch("sys.argv", ["converter.py", "--skip-check", *flags])) stack.enter_context(contextlib.redirect_stderr(io.StringIO())) @@ -1030,41 +1023,41 @@ def test_invalid_config_fails_before_side_effects(self): class LogIntegrationTests(unittest.TestCase): - def test_log_rotation_is_bounded_and_thread_safe(self): + def test_runtime_events_use_sqlite_audit_not_text_files(self): + from app.audit_store import AuditStore with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "test.log" - with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 256}), \ - patch.object(converter, "LOG_MAX_BYTES", 2048): - with ThreadPoolExecutor(max_workers=8) as executor: - list(executor.map(converter._log, [f"event={i} " + "汉字" * 1000 for i in range(30)])) - files = list(Path(directory).glob("test.log*")) - self.assertLessEqual(len(files), 3) - self.assertTrue(all(file.stat().st_size <= 2048 for file in files)) - for file in files: - file.read_text(encoding="utf-8", errors="strict") - - def test_body_previews_do_not_log_image_data_or_tokens(self): + with contextlib.closing(AuditStore(Path(directory) / "logs.sqlite3")) as audit: + with patch.dict(converter.CONFIG, {"log_path": str(path), "audit_store": audit}): + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(converter._log, ["[cred] updated" for _ in range(30)])) + self.assertEqual(len(audit.list_records("runtime", limit=100)["items"]), 30) + self.assertFalse(path.exists()) + + def test_body_previews_and_secret_hints_never_reach_persistent_logs(self): + from app.audit_store import AuditStore with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "test.log" - with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 1024}): - converter._log_json("body", {"accessToken": "synthetic-private", "image": "data:image/png;base64," + "Z" * 10000}) - converter._log("ReadTimeout: Authorization: Bearer synthetic-secret") - text = path.read_text() - self.assertNotIn("synthetic-private", text) - self.assertNotIn("synthetic-secret", text) - self.assertNotIn("Z" * 20, text) - self.assertIn("ReadTimeout", text) - self.assertLess(path.stat().st_size, 1500) - - def test_zero_body_budget_keeps_summary_only(self): + database = Path(directory) / "logs.sqlite3" + key = "cb-" + "a" * 32 + with contextlib.closing(AuditStore(database)) as audit: + with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 1024, "audit_store": audit}): + converter._log_json("body", {"accessToken": "synthetic-private", "image": "data:image/png;base64," + "Z" * 10000}) + converter._log("[cred] failure " + key) + converter._log("ReadTimeout: Authorization: Bearer synthetic-secret") + self.assertFalse(path.exists()) + raw = b"".join(file.read_bytes() for file in Path(directory).glob("logs.sqlite3*")) + for secret in (key.encode(), b"synthetic-private", b"synthetic-secret", b"Z" * 20): + self.assertNotIn(secret, raw) + + def test_retired_text_setting_cannot_reenable_file_output(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "test.log" - with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 0}): + with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 0, "audit_store": None}): converter._log_json("body", {"text": "must-not-be-logged"}) converter._log_text_body("raw", "must-not-be-logged") converter._log("request summary") - self.assertIn("request summary", path.read_text()) - self.assertNotIn("must-not-be-logged", path.read_text()) + self.assertFalse(path.exists()) if __name__ == "__main__": diff --git a/tests/test_sqlite_state.py b/tests/test_sqlite_state.py new file mode 100644 index 0000000..3173a6f --- /dev/null +++ b/tests/test_sqlite_state.py @@ -0,0 +1,341 @@ +"""Exercise SQLite migration, persisted defaults and authentication using synthetic state.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import concurrent.futures +import contextlib +import io +import json +import os +import sqlite3 +import tempfile +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from app.admin_auth import AdminAuth, SessionStoreError +from app.control_store import ControlStore +from app.credential_cooldowns import CredentialCooldowns +from app.credits import CreditLedger, ModelCatalogCache +from app.model_blocks import ModelBlocks +from app.safe_logging import sanitize_log_text +from app.settings import resolve_settings +from app.startup import announce_default_key, load_startup_env, resolve_startup_key +from app.state_store import LEGACY_FILES +from app.trial_rewards import TrialLedger, attempt_trial +from app.usage_snapshots import UsageSnapshots + +IDENTITY = "a" * 64 +FIXED_KEY = "synthetic-fixed-key" + + +class Terminal(io.StringIO): + def isatty(self): + return True + + +class SQLiteStateTests(unittest.TestCase): + def setUp(self): + self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.control = self.open_control() + self.state = self.control.state + self.now = time.time() + + def open_control(self): + control = ControlStore(self.root / "control.sqlite3") + self.addCleanup(control.close) + return control + + def create_legacy(self): + ledger = CreditLedger(self.root / LEGACY_FILES["credits"]) + ledger.bind_identity("fixture.info", IDENTITY) + ledger.mark_checkin("fixture.info", "2026-09-22", True, 0, "ok") + ledger.update_credits("fixture.info", {"credits": 12, "segments": [], "partial": True}) + cache = ModelCatalogCache(self.root / LEGACY_FILES["catalog"]) + cache.put("account:version", [{"id": "model"}]) + blocks = ModelBlocks(self.root / LEGACY_FILES["model_blocks"]) + blocks.note("https://example.invalid/chat", "model", now=self.now) + cooldowns = CredentialCooldowns(self.root / LEGACY_FILES["cooldowns"]) + cooldowns.note_credential(IDENTITY, "cn-work", self.now + 300) + cooldowns.note_model(IDENTITY, "cn-work", "model", self.now + 600) + usage = UsageSnapshots(self.root / LEGACY_FILES["usage"]) + usage.store("fixture.info", IDENTITY, "domestic", {"total_credits": 12, "requests": 2, + "by_day": {"2026-09-22": {"model": 12}}}, partial=True) + trial = TrialLedger(self.root / LEGACY_FILES["trial"]) + self.assertTrue(trial.begin(IDENTITY)) + trial.finish(IDENTITY, {"ok": True, "already": False, "code": 0, "status": 200}) + auth = AdminAuth({"api_key": FIXED_KEY, "session_path": self.root / LEGACY_FILES["sessions"]}) + result, status = auth.login(SimpleNamespace(client=SimpleNamespace(host="fixture"), cookies={}), FIXED_KEY) + self.assertEqual(status, 200) + return result[0] + + def test_all_legacy_sources_migrate_once_and_runtime_never_updates_json(self): + sid = self.create_legacy() + before = {name: (self.root / file).read_bytes() for name, file in LEGACY_FILES.items()} + self.state.migrate(self.root) + ledger = CreditLedger(store=self.state) + self.assertTrue(ledger.checkin_done("fixture.info", "2026-09-22")) + self.assertTrue(ledger.entry("fixture.info")["credits"]["partial"]) + self.assertTrue(ModelCatalogCache(store=self.state).fresh("account:version")) + self.assertTrue(ModelBlocks(store=self.state).blocked("https://example.invalid/chat", "model")) + cooldowns = CredentialCooldowns(store=self.state) + self.assertGreater(cooldowns.credential_until(IDENTITY, "cn-work"), self.now) + self.assertGreater(cooldowns.model_until(IDENTITY, "cn-work", "model"), self.now) + self.assertTrue(UsageSnapshots(store=self.state).accounts()["fixture.info"]["partial"]) + trial = TrialLedger(store=self.state) + self.assertFalse(trial.begin(IDENTITY, self.now + 3 * 86400)) + auth = AdminAuth({"api_key": FIXED_KEY, "state_store": self.state}) + auth.reconcile() + self.assertIn(sid, auth.sessions) + ledger.note_error("fixture.info", "fixture error") + cooldowns.clear_credential(IDENTITY, "cn-work") + UsageSnapshots(store=self.state).forget("fixture.info") + auth.config["api_key"] = "rotated-fixture" + auth.reconcile() + self.state.migrate(self.root) + self.assertFalse(CredentialCooldowns(store=self.state).credential_until(IDENTITY, "cn-work")) + self.assertFalse(UsageSnapshots(store=self.state).accounts()) + auth.config["api_key"] = FIXED_KEY + auth.reconcile() + self.assertNotIn(sid, auth.sessions) + self.assertEqual(before, {name: (self.root / file).read_bytes() for name, file in LEGACY_FILES.items()}) + self.assertEqual(self.control._db.execute("SELECT count(*) FROM state_imports").fetchone()[0], 7) + + def test_absent_legacy_file_cannot_resurrect_later(self): + self.state.migrate(self.root) + (self.root / LEGACY_FILES["credits"]).write_text('{"version":1,"creds":{"old":{}}}') + self.state.migrate(self.root) + self.assertIsNone(self.state.get("credits")) + + def test_invalid_migration_rolls_back_every_document_and_marker(self): + self.create_legacy() + (self.root / LEGACY_FILES["trial"]).write_text('{"version":1,"accounts":{"bad":{}}}') + with self.assertRaisesRegex(ValueError, "trial-ledger"): + self.state.migrate(self.root) + for table in ("state_imports", "runtime_state"): + self.assertEqual(self.control._db.execute(f"SELECT count(*) FROM {table}").fetchone()[0], 0) + + def test_duplicate_fields_and_upstream_tokens_are_rejected(self): + file = self.root / LEGACY_FILES["credits"] + for payload in ('{"version":1,"version":1,"creds":{}}', + '{"version":1,"creds":{"f":{"accessToken":"fixture"}}}'): + file.write_text(payload) + with self.assertRaises(ValueError): + self.state.migrate(self.root) + + def test_symlink_legacy_source_is_rejected(self): + target = self.root / "target" + target.write_text('{"version":1,"creds":{}}') + (self.root / LEGACY_FILES["credits"]).symlink_to(target) + with self.assertRaises(ValueError): + self.state.migrate(self.root) + + def test_upgrade_v1_preserves_settings_and_rejects_old_readers(self): + self.control.update_settings({"max_images": 3}, 0) + self.control._db.execute("PRAGMA user_version=1") + other = self.open_control() + self.assertEqual(other.snapshot()["settings"]["max_images"], 3) + self.assertEqual(other._db.execute("PRAGMA user_version").fetchone()[0], 2) + + def test_trial_reservation_is_atomic_across_connections_and_failures_are_closed(self): + other = self.open_control() + ledgers = [TrialLedger(store=self.state), TrialLedger(store=other.state)] + with concurrent.futures.ThreadPoolExecutor(2) as executor: + results = list(executor.map(lambda ledger: ledger.begin(IDENTITY), ledgers)) + self.assertEqual(sorted(results), [False, True]) + self.assertFalse(ledgers[0].begin(IDENTITY)) + with patch.object(self.state, "_put", side_effect=sqlite3.OperationalError("fixture")), \ + patch("app.trial_rewards.claim_trial") as claim: + with self.assertRaises(sqlite3.OperationalError): + attempt_trial(ledgers[0], "b" * 64, {"X-Domain": "www.workbuddy.ai"}) + claim.assert_not_called() + + def test_cooldown_write_failure_is_visible_and_retried(self): + cooldowns = CredentialCooldowns(store=self.state) + with patch.object(self.state, "put", side_effect=sqlite3.OperationalError("fixture")): + self.assertFalse(cooldowns.note_credential(IDENTITY, "cn-work", self.now + 300)) + self.assertEqual(cooldowns.last_error, "OperationalError") + self.assertTrue(cooldowns.note_credential(IDENTITY, "cn-work", self.now + 300)) + self.assertGreater(CredentialCooldowns(store=self.state).credential_until(IDENTITY, "cn-work"), self.now) + + def test_rebuildable_cache_write_failure_keeps_in_memory_routing(self): + blocks = ModelBlocks(store=self.state) + catalogs = ModelCatalogCache(store=self.state) + with patch.object(self.state, "put", side_effect=sqlite3.OperationalError("fixture")): + with self.assertWarns(RuntimeWarning): + blocks.note("https://example.invalid/chat", "model", now=self.now) + with self.assertWarns(RuntimeWarning): + catalogs.put("account:version", [{"id": "model"}]) + self.assertTrue(blocks.blocked("https://example.invalid/chat", "model")) + self.assertTrue(catalogs.fresh("account:version")) + self.assertEqual(catalogs.models("account:version"), [{"id": "model"}]) + + + def test_session_revocation_failure_does_not_activate_new_key(self): + auth = AdminAuth({"api_key": FIXED_KEY, "state_store": self.state}) + auth.reconcile() + auth.config["api_key"] = "rotated-fixture" + with patch.object(self.state, "put", side_effect=sqlite3.OperationalError("fixture")), \ + patch.object(self.state, "delete", side_effect=sqlite3.OperationalError("fixture")): + with self.assertRaises(SessionStoreError): + auth.reconcile() + self.assertEqual(auth._configured_key, FIXED_KEY) + + def test_default_key_is_atomic_private_and_survives_restart(self): + other = self.open_control() + with concurrent.futures.ThreadPoolExecutor(2) as executor: + results = list(executor.map(lambda state: state.default_key(create=True), [self.state, other.state])) + self.assertEqual(results[0], results[1]) + key, pending = results[0] + self.assertRegex(key, r"^cb-[0-9a-f]{32}$") + self.assertTrue(pending) + self.assertNotIn(key, json.dumps(self.control.snapshot())) + self.assertEqual(self.open_control().state.default_key(), (key, True)) + self.assertTrue(self.state.claim_announcement(key)) + self.assertFalse(other.state.claim_announcement(key)) + self.assertEqual(other.state.default_key(), (key, False)) + + def test_broken_saved_key_is_not_regenerated(self): + self.state.default_key(create=True) + self.control._db.execute("UPDATE gateway_secrets SET value='broken'") + with self.assertRaises(ValueError): + self.state.default_key(create=True) + self.assertEqual(self.control._db.execute("SELECT value FROM gateway_secrets").fetchone()[0], "broken") + + def config(self, source="default", key=""): + return {"state_store": self.state, "control_store": self.control, "api_key": key, + "host": "127.0.0.1", "settings_sources": {"api_key": source}} + + def test_key_is_announced_once_only_after_successful_commit(self): + config = self.config() + terminal = Terminal() + with patch("sys.stderr", terminal): + resolve_startup_key(config, SimpleNamespace(api_key="")) + key = config["api_key"] + self.assertEqual(terminal.getvalue(), "") + announce_default_key(config) + announce_default_key(config) + again = self.config() + resolve_startup_key(again, SimpleNamespace(api_key="")) + announce_default_key(again) + self.assertEqual(terminal.getvalue().count(key), 1) + self.assertEqual(again["api_key"], key) + self.assertIsNone(next(row for row in resolve_settings(config) if row["key"] == "api_key")["value"]) + self.assertNotIn(key, sanitize_log_text("key leaked " + key)) + with patch("sys.stderr", io.StringIO()): + resolve_startup_key(self.config(), SimpleNamespace(api_key="")) + + def test_explicit_sources_do_not_overwrite_saved_default_or_print(self): + key, _ = self.state.default_key(create=True) + for source in ("cli", "environment", "dotenv"): + for supplied in ("", "override-fixture"): + config = self.config(source, supplied) + with patch("sys.stderr", io.StringIO()) as output: + resolve_startup_key(config, SimpleNamespace(api_key=supplied)) + announce_default_key(config) + self.assertEqual(config["api_key"], supplied) + self.assertEqual(output.getvalue(), "") + self.assertEqual(self.state.default_key()[0], key) + + def test_nonterminal_and_public_first_start_do_not_generate_hidden_key(self): + with patch("sys.stderr", io.StringIO()), self.assertRaises(ValueError): + resolve_startup_key(self.config(), SimpleNamespace(api_key="")) + config = self.config() + config["host"] = "0.0.0.0" + with patch("sys.stderr", Terminal()), self.assertRaises(ValueError): + resolve_startup_key(config, SimpleNamespace(api_key="")) + self.assertEqual(self.state.default_key(), (None, False)) + + def test_dotenv_is_optional_and_process_environment_wins(self): + path = self.root / ".env" + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(load_startup_env(path), set()) + path.write_text("CODEBUDDY2API_PORT=9090\nCODEBUDDY2API_KEY=file-fixture\n") + os.environ["CODEBUDDY2API_PORT"] = "9091" + changed = load_startup_env(path) + self.assertEqual(os.environ["CODEBUDDY2API_PORT"], "9091") + self.assertEqual(os.environ["CODEBUDDY2API_KEY"], "file-fixture") + self.assertEqual(changed, {"CODEBUDDY2API_KEY"}) + config = self.config("environment", "file-fixture") + resolve_startup_key(config, SimpleNamespace(api_key="file-fixture"), changed) + self.assertEqual(config["settings_sources"]["api_key"], "dotenv") + + def test_invalid_dotenv_fails_without_loading_partial_values(self): + path = self.root / ".env" + path.write_text('CODEBUDDY2API_PORT=9090\nCODEBUDDY2API_KEY="unterminated\n') + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(ValueError, "第 2 行") as caught: + load_startup_env(path) + self.assertNotIn("unterminated", str(caught.exception)) + self.assertNotIn("CODEBUDDY2API_PORT", os.environ) + + + def test_full_startup_precedence_and_default_restore(self): + import converter + from app import runtime_management + from fastapi import FastAPI + + def boot(env=None, flags=()): + observed = {} + with contextlib.chdir(self.root), contextlib.ExitStack() as stack: + stack.enter_context(patch.dict(os.environ, {"HOME": str(self.root), "CODEBUDDY_AUTH_DIR": str(self.root), **(env or {})}, clear=True)) + stack.enter_context(patch.dict(converter.CONFIG)) + stack.enter_context(patch.object(converter, "app", FastAPI())) + stack.enter_context(patch.object(sys, "argv", ["converter.py", "--skip-check", *flags])) + stack.enter_context(patch.object(converter, "seed_credentials")) + stack.enter_context(patch.object(converter, "CredentialPool")) + stack.enter_context(patch.object(converter, "_publish_model_cache")) + stack.enter_context(patch.object(converter.threading, "Thread")) + stack.enter_context(patch.object(runtime_management, "install")) + output = stack.enter_context(patch("sys.stderr", Terminal())) + def serve(app, config, **kwargs): + observed.update(kwargs, key=config["api_key"], sources=dict(config["settings_sources"])) + announce_default_key(config) + stack.enter_context(patch.object(converter, "run_server", side_effect=serve)) + converter.main() + observed["output"] = output.getvalue() + return observed + + first = boot() + key = first["key"] + self.assertEqual(first["output"].count(key), 1) + self.assertEqual(first["sources"]["api_key"], "generated") + envfile = self.root / ".env" + envfile.write_text("CODEBUDDY2API_PORT=9092\nCODEBUDDY2API_KEY=file-fixture\n") + file_run = boot() + self.assertEqual((file_run["port"], file_run["key"], file_run["sources"]["api_key"]), (9092, "file-fixture", "dotenv")) + environment = boot({"CODEBUDDY2API_PORT": "9093", "CODEBUDDY2API_KEY": "env-fixture"}) + self.assertEqual((environment["port"], environment["key"]), (9093, "env-fixture")) + cli = boot({"CODEBUDDY2API_PORT": "9093", "CODEBUDDY2API_KEY": "env-fixture"}, + ("--port", "9094", "--api-key", "cli-fixture")) + self.assertEqual((cli["port"], cli["key"]), (9094, "cli-fixture")) + envfile.unlink() + resumed = boot() + self.assertEqual(resumed["key"], key) + self.assertNotIn(key, resumed["output"]) + self.assertEqual(self.state.default_key(), (key, False)) + self.assertFalse(list(self.root.glob("*.json"))) + + def test_listener_failure_does_not_consume_announcement(self): + import asyncio + import uvicorn + from fastapi import FastAPI + from unittest.mock import AsyncMock + from app.startup import run_server + config = self.config() + with patch("sys.stderr", Terminal()) as terminal: + resolve_startup_key(config, SimpleNamespace(api_key="")) + key = config["api_key"] + with patch.object(uvicorn.Server, "startup", new=AsyncMock(side_effect=SystemExit(3))), \ + patch.object(uvicorn.Server, "run", lambda server: asyncio.run(server.startup())): + with self.assertRaises(SystemExit): + run_server(FastAPI(), config, host="127.0.0.1", port=8787) + self.assertNotIn(key, terminal.getvalue()) + self.assertEqual(self.state.default_key(), (key, True)) + + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/webui_fixture.py b/tests/webui_fixture.py index 236d292..dcc0740 100644 --- a/tests/webui_fixture.py +++ b/tests/webui_fixture.py @@ -48,17 +48,19 @@ def main(): root = Path(directory) os.environ["CODEBUDDY_AUTH_DIR"] = directory gateway.CONFIG.update(api_key="synthetic-e2e-key", log_path=None, - trial_ledger=gateway.trial_rewards.TrialLedger(root / "trial-ledger.json"), control_store=ControlStore(root / "control.sqlite3"), audit_store=AuditStore(root / "logs.sqlite3")) + state = gateway.CONFIG["control_store"].state + state.migrate(root) + gateway.CONFIG.update(state_store=state, trial_ledger=gateway.trial_rewards.TrialLedger(store=state)) apply_persisted_settings(gateway.CONFIG, explicit=("api_key",), environ={}) credential = {"account": {"uid": "fixture-account", "nickname": "集成测试凭证"}, "auth": {"domain": "www.workbuddy.cn", "accessToken": "synthetic-browser-access", "refreshToken": "synthetic-browser-refresh", "expiresAt": (time.time() + 86400) * 1000}} path = gateway.atomic_write_credential(root, "fixture.info", json.dumps(credential).encode()) - pool = gateway.CredentialPool([path]) + pool = gateway.CredentialPool([path], state_store=state) gateway.CONFIG["cred_pool"], gateway.CONFIG["cred"] = pool, pool.first() - ledger = gateway.credits_mod.CreditLedger(root / "credits-ledger.json") + ledger = gateway.credits_mod.CreditLedger(store=state) pool.set_ledger(ledger) ledger.update_credits(str(path), {"credits": 125, "intl": False, "segments": []}) identity = pool.entries()[0]["account_key"] diff --git a/uv.lock b/uv.lock index 008e420..bfde9bf 100644 --- a/uv.lock +++ b/uv.lock @@ -59,6 +59,7 @@ dependencies = [ { name = "fastapi" }, { name = "httpx" }, { name = "pytest" }, + { name = "python-dotenv" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -67,6 +68,7 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.141.1" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "pytest", specifier = ">=9.1.1" }, + { name = "python-dotenv", specifier = ">=1.2.3" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.52.4" }, ] diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index 6c1b4b2..26595f3 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -28,6 +28,8 @@ const sourceLabels: Record = { cli: "命令行", environment: "环境变量", env: "环境变量", + dotenv: ".env 文件", + generated: "本地默认密钥", management: "管理界面", default: "默认值", internal: "内置", From e61ae221f9dde89893e89487bf83459a73ce987e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?= <277378677+maiphucgiang@users.noreply.github.com> Date: Tue, 22 Sep 2026 07:51:08 +0800 Subject: [PATCH 2/4] Keep default-key disclosure on the controlling terminal Preserve authoritative SQLite state on read failures and leave failed session epochs inactive for retry. Publish CodeQL results against the checked-out pull request head and use a literal schema-version statement. --- .github/workflows/codeql.yml | 4 ++ app/admin_auth.py | 23 ++++--- app/control_store.py | 2 +- app/credits.py | 4 +- app/model_blocks.py | 12 ++-- app/startup.py | 40 ++++++++---- docs/deployment.md | 4 ++ docs/deployment.zh-CN.md | 4 ++ tests/test_sqlite_state.py | 85 +++++++++++++++++++++++-- tests/test_terminal_output.py | 114 ++++++++++++++++++++++++++++++++++ 10 files changed, 257 insertions(+), 35 deletions(-) create mode 100644 tests/test_terminal_output.py diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1aa9ee6..67a119a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -60,6 +60,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` @@ -101,3 +103,5 @@ jobs: uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" + ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/head', github.event.pull_request.number) || github.ref }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/app/admin_auth.py b/app/admin_auth.py index b09d05d..58f9e45 100644 --- a/app/admin_auth.py +++ b/app/admin_auth.py @@ -25,11 +25,7 @@ class SessionStoreError(RuntimeError): - """A superseded session snapshot could not be durably revoked. - - Startup must abort on this: the surviving snapshot could otherwise be adopted by a - later start under the superseded key, resurrecting sessions that were meant to die. - """ + """Authoritative session state could not be read or durably revoked.""" # Optional persisted session table so a restart does not force another login. @@ -141,7 +137,7 @@ def __init__(self, config, *, clock=time.monotonic, wall_clock=time.time): def _fingerprint(key): """Keyed fingerprint naming the key epoch a snapshot belongs to. - It identifies the epoch; revocation itself is performed by clearing the file. + It identifies the epoch; revocation clears its persisted snapshot. """ return hmac.new(key.encode(), _SESSION_KEY_LABEL, hashlib.sha256).hexdigest() @@ -222,9 +218,11 @@ def _restore(self, key): if self._store is not None: try: document = self._store.get("sessions") + self._storage_ok() return True if document is None else self._adopt(document, key) - except (OSError, ValueError, sqlite3.Error): - return False + except (OSError, ValueError, sqlite3.Error) as error: + self._storage_failed(error) + raise SessionStoreError("无法读取 SQLite 会话状态；保留原记录，请修复控制库后重启") from None if self._path is None: return True try: @@ -349,7 +347,12 @@ def _key(self): if restoring: # Adopt only a snapshot that matches the current epoch; anything else is # revoked, so switching back to an old key cannot resurrect its sessions. - durable = self._restore(key) or self._revoke() + try: + durable = self._restore(key) or self._revoke() + except SessionStoreError: + self._configured_key = previous + self.sessions.clear() + raise else: # A rotated or cleared key revokes every session, on disk as well. durable = self._persist() @@ -479,7 +482,7 @@ async def no_cache(message): # The key epoch changed but its superseded snapshot survives. Deny rather than # continue: startup refuses this case, so it is only reachable mid-process. return await error_response( - 503, "会话快照无法持久撤销，管理接口已锁定；请检查管理目录权限后重启")(scope, receive, no_cache) + 503, "会话快照无法安全读取或持久化，管理接口已锁定；请检查控制库后重启")(scope, receive, no_cache) if not enabled: return await error_response(503, "未配置 API key，管理接口已锁定")(scope, receive, no_cache) path, method = scope["path"], scope["method"] diff --git a/app/control_store.py b/app/control_store.py index 2da74f8..6d3351d 100644 --- a/app/control_store.py +++ b/app/control_store.py @@ -110,7 +110,7 @@ def __init__(self, path): self._db.execute("CREATE TABLE IF NOT EXISTS runtime_state (name TEXT PRIMARY KEY, payload TEXT NOT NULL)") self._db.execute("CREATE TABLE IF NOT EXISTS state_imports (name TEXT PRIMARY KEY, imported INTEGER NOT NULL, migrated_at REAL NOT NULL)") self._db.execute("CREATE TABLE IF NOT EXISTS gateway_secrets (name TEXT PRIMARY KEY, value TEXT NOT NULL, announced INTEGER NOT NULL DEFAULT 0 CHECK(announced IN (0,1)))") - self._db.execute(f"PRAGMA user_version={self.SCHEMA_VERSION}") + self._db.execute("PRAGMA user_version=2") self._db.execute("COMMIT") except Exception: if self._db.in_transaction: diff --git a/app/credits.py b/app/credits.py index 0e84217..20e33d1 100644 --- a/app/credits.py +++ b/app/credits.py @@ -754,8 +754,10 @@ def __init__(self, path: Path | None = None, ttl: float = 6 * 3600, *, store=Non def _load(self): with self._lock: + d = (self._store.get("catalog") or self._data) if self._store is not None else None try: - d = (self._store.get("catalog") or self._data) if self._store is not None else json.loads(self.path.read_text(encoding="utf-8")) + if self._store is None: + d = json.loads(self.path.read_text(encoding="utf-8")) if (not isinstance(d, dict) or d.get("version") not in (1, self.SCHEMA_VERSION) or not isinstance(d.get("groups"), dict)): return diff --git a/app/model_blocks.py b/app/model_blocks.py index fd360df..b7fb5fd 100644 --- a/app/model_blocks.py +++ b/app/model_blocks.py @@ -29,14 +29,14 @@ def __init__(self, path=None, ttl_s: float = DEFAULT_TTL_S, max_ttl_s: float = M # Persistence def _load(self): - try: - if self._store is not None: - data = self._store.get("model_blocks") - else: + if self._store is not None: + data = self._store.get("model_blocks") + else: + try: with open(self.path, "r", encoding="utf-8") as f: data = json.load(f) - except (OSError, ValueError): - return + except (OSError, ValueError): + return if not isinstance(data, dict): return out: dict[str, dict] = {} diff --git a/app/startup.py b/app/startup.py index df0b56f..d5a8834 100644 --- a/app/startup.py +++ b/app/startup.py @@ -5,7 +5,6 @@ import os from pathlib import Path import stat -import sys from dotenv import load_dotenv from dotenv.parser import parse_stream @@ -41,6 +40,19 @@ def load_startup_env(path=None): return set(os.environ) - previous +def terminal_stream(): + """Open the controlling terminal, never a redirected standard stream or regular file.""" + device = "CONOUT$" if os.name == "nt" else "/dev/tty" + fd = os.open(device, os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NOCTTY", 0)) + try: + if not stat.S_ISCHR(os.fstat(fd).st_mode) or not os.isatty(fd): + raise OSError("No interactive console") + return os.fdopen(fd, "w", encoding="utf-8", buffering=1) + except BaseException: + os.close(fd) + raise + + def resolve_startup_key(config, args, dotenv_keys=()): sources = config["settings_sources"] for name, spec in SCHEMA.items(): @@ -51,27 +63,29 @@ def resolve_startup_key(config, args, dotenv_keys=()): return store = config["state_store"] key, pending = store.default_key() + if key is None and config["host"] not in ("127.0.0.1", "::1", "localhost"): + raise ValueError("非回环监听请显式设置 API key；默认密钥仅在本地首次启动时生成") + if key is None or pending: + try: + with terminal_stream(): + pass + except OSError: + raise ValueError("首次显示默认 API key 需要交互终端；后台运行请显式配置 CODEBUDDY2API_KEY") from None if key is None: - if config["host"] not in ("127.0.0.1", "::1", "localhost"): - raise ValueError("非回环监听请显式设置 API key；默认密钥仅在本地首次启动时生成") - if not sys.stderr.isatty(): - raise ValueError("首次生成 API key 需要交互终端；后台运行请显式配置 CODEBUDDY2API_KEY") key, pending = store.default_key(create=True) - if pending and not sys.stderr.isatty(): - raise ValueError("默认 API key 尚未显示；请先在交互终端启动，或显式配置 CODEBUDDY2API_KEY") config["api_key"] = args.api_key = key sources["api_key"] = "generated" config["announce_default_key"] = pending def announce_default_key(config): - if not config.get("announce_default_key") or not sys.stderr.isatty(): + if not config.get("announce_default_key"): return - key = config["api_key"] - if config["state_store"].claim_announcement(key): - # This is deliberately outside every logging/audit path. - sys.stderr.write(f"\n默认 API key：{key}\n已保存到 control.sqlite3，仅显示这一次；管理登录与 API 请求共用。\n") - sys.stderr.flush() + with terminal_stream() as terminal: + key = config["api_key"] + if config["state_store"].claim_announcement(key): + terminal.write(f"\n默认 API key：{key}\n已保存到 control.sqlite3，仅显示这一次；管理登录与 API 请求共用。\n") + terminal.flush() config["announce_default_key"] = False diff --git a/docs/deployment.md b/docs/deployment.md index fe7b051..3630067 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -68,6 +68,8 @@ uv run converter.py Local startup does not require `.env`. Without an explicit key, the first loopback startup generates a `cb-…` default key in `auth/control.sqlite3` and displays it once in the interactive terminal after listening succeeds. Restarts reuse it without printing it again. Keep it safe: management and API requests share this key. Configure a key explicitly for first-time headless or non-loopback deployment. +One-time disclosure goes directly to the controlling terminal (`/dev/tty` or Windows `CONOUT$`), not stdout/stderr; redirecting standard output does not capture the key. Terminal recording remains outside the gateway's control. + Without uv, run `python3 -m venv .venv`, activate it, install dependencies with `pip install --require-hashes --only-binary=:all: -r requirements.txt`, then run `python3 converter.py`. Distribution archives include the WebUI; source installs still need the frontend build. Both commands optionally read `.env` in the current working directory, never parent directories. Precedence: explicit CLI > process environment > `.env` > saved SQLite values > defaults. Overrides do not replace the saved default key; an explicitly empty key still locks management. Listener changes require restart; `CODEBUDDY2API_IMAGE/AUTH_PATH` are Compose-only. @@ -76,6 +78,8 @@ Both commands optionally read `.env` in the current working directory, never par Stop the gateway and back up the entire data directory first. Startup imports legacy JSON sessions, cooldowns, credit/trial history, catalogs and usage into `control.sqlite3` once. Old files remain as backups and are no longer read or updated. Invalid critical state or a failed migration stops startup; repair it or restore a backup rather than deleting ledgers to bypass validation. +SQLite state read/validation failures stop startup without deleting sessions or replacing state with empty defaults. Repair the control database before retrying; only rebuildable cache write failures may retain the current in-memory data with a warning. + All runtime logs use `logs.sqlite3`; legacy `--log` / `CODEBUDDY2API_LOG` settings warn and no longer write text files. Generated keys enter private configuration, never logs. Restrict access to the data directory; use a private user directory on Windows. Older versions cannot read the upgraded control database. Downgrades require a stopped gateway and matching state migration, not merely old code reading stale JSON. Never overwrite new claims, dispatches or session revocations with an outdated backup. diff --git a/docs/deployment.zh-CN.md b/docs/deployment.zh-CN.md index 747b13c..c00ed9d 100644 --- a/docs/deployment.zh-CN.md +++ b/docs/deployment.zh-CN.md @@ -68,6 +68,8 @@ uv run converter.py 本地启动无需 `.env`。没有显式 key 时，首次回环启动会生成 `cb-…` 默认密钥，保存到 `auth/control.sqlite3`，监听成功后仅在交互终端显示一次；以后重启复用且不再打印。请妥善保存，管理登录与 API 请求共用。首次后台或非回环部署请显式配置 key。 +密码提示直接写入控制终端（`/dev/tty` 或 Windows `CONOUT$`），不经过 stdout/stderr，重定向标准输出不会收集密码；外部终端录制不在程序控制范围内。 + 没有 uv 时：运行 `python3 -m venv .venv` 并激活，用 `pip install --require-hashes --only-binary=:all: -r requirements.txt` 安装依赖，再执行 `python3 converter.py`。发行包已包含 WebUI；源码安装仍需构建界面。 两种启动方式均可选读取当前工作目录的 `.env`，不搜索父目录。优先级：显式 CLI > 进程环境变量 > `.env` > SQLite 保存值 > 默认值。覆盖不改写已保存的默认 key；显式空 key 仍锁定管理。修改监听需重启，`CODEBUDDY2API_IMAGE/AUTH_PATH` 仅用于 Compose。 @@ -76,6 +78,8 @@ uv run converter.py 升级前停止网关并备份整个数据目录。首次启动将旧 JSON 会话、冷却、积分/领取、目录和用量状态一次性导入 `control.sqlite3`；原文件保留作备份，不再参与读写。关键数据损坏或迁移失败会阻止启动，请修复或恢复备份，不要删除账本绕过检查。 +SQLite 状态读取或校验失败会阻止启动，不删除会话、不回退为空状态；修复控制库后再启动。仅可重建缓存的写入失败允许保留当前内存数据并告警。 + 所有运行日志使用 `logs.sqlite3`；旧 `--log` / `CODEBUDDY2API_LOG` 仅提示弃用，不再输出文本文件。自动生成的 key 只进私有配置，不进任何日志。限制数据目录访问权限；Windows 应使用当前用户的私有目录。 旧版本不能读取升级后的控制库。回滚须停机并迁回匹配的状态，不能只切代码后复用旧 JSON；升级后发生的领取、派遣或会话撤销不能用过期备份覆盖。 diff --git a/tests/test_sqlite_state.py b/tests/test_sqlite_state.py index 3173a6f..3f88296 100644 --- a/tests/test_sqlite_state.py +++ b/tests/test_sqlite_state.py @@ -42,6 +42,12 @@ def setUp(self): self.control = self.open_control() self.state = self.control.state self.now = time.time() + self.console_open = self.enterContext(patch("app.startup.terminal_stream", side_effect=OSError("no fixture terminal"))) + + def console(self): + terminal = Terminal() + self.console_open.side_effect = lambda: contextlib.nullcontext(terminal) + return terminal def open_control(self): control = ControlStore(self.root / "control.sqlite3") @@ -173,6 +179,50 @@ def test_rebuildable_cache_write_failure_keeps_in_memory_routing(self): self.assertEqual(catalogs.models("account:version"), [{"id": "model"}]) + def test_sqlite_session_read_failure_preserves_record_and_retries_epoch(self): + self.create_legacy() + self.state.migrate(self.root) + before = self.state.get("sessions") + for error in (sqlite3.OperationalError("locked"), sqlite3.DatabaseError("malformed"), + OSError("unreadable"), ValueError("invalid state")): + auth = AdminAuth({"api_key": FIXED_KEY, "state_store": self.state}) + with self.subTest(error=type(error).__name__), \ + patch.object(self.state, "get", side_effect=error), \ + patch.object(self.state, "delete") as delete: + for attempt in (auth.reconcile, auth.enabled): + with self.assertRaises(SessionStoreError): + attempt() + self.assertIsNone(auth._configured_key) + self.assertFalse(auth.sessions) + delete.assert_not_called() + self.assertEqual(auth.storage()["last_error"], type(error).__name__) + self.assertEqual(self.state.get("sessions"), before) + auth.reconcile() + self.assertEqual(auth._configured_key, FIXED_KEY) + self.assertEqual(set(auth.sessions), set(before["sessions"])) + self.assertFalse(auth.storage()["degraded"]) + + def test_invalid_sqlite_sessions_stop_startup_without_deleting_evidence(self): + self.control._db.execute("INSERT INTO runtime_state VALUES('sessions', 'invalid-json')") + auth = AdminAuth({"api_key": FIXED_KEY, "state_store": self.state}) + with self.assertRaises(SessionStoreError): + auth.reconcile() + self.assertIsNone(auth._configured_key) + self.assertEqual(self.control._db.execute("SELECT payload FROM runtime_state WHERE name='sessions'").fetchone()[0], + "invalid-json") + + def test_all_sqlite_loaders_propagate_read_failures_without_empty_fallbacks(self): + loaders = (lambda: CreditLedger(store=self.state), lambda: ModelCatalogCache(store=self.state), + lambda: ModelBlocks(store=self.state), lambda: CredentialCooldowns(store=self.state), + lambda: UsageSnapshots(store=self.state), lambda: TrialLedger(store=self.state).snapshot()) + for error in (sqlite3.OperationalError("locked"), ValueError("invalid state"), OSError("unreadable")): + for loader in loaders: + with self.subTest(error=type(error).__name__, loader=loader), \ + patch.object(self.state, "get", side_effect=error): + with self.assertRaises(type(error)): + loader() + + def test_session_revocation_failure_does_not_activate_new_key(self): auth = AdminAuth({"api_key": FIXED_KEY, "state_store": self.state}) auth.reconcile() @@ -210,8 +260,8 @@ def config(self, source="default", key=""): def test_key_is_announced_once_only_after_successful_commit(self): config = self.config() - terminal = Terminal() - with patch("sys.stderr", terminal): + terminal = self.console() + with patch("sys.stderr", io.StringIO()) as output: resolve_startup_key(config, SimpleNamespace(api_key="")) key = config["api_key"] self.assertEqual(terminal.getvalue(), "") @@ -221,12 +271,34 @@ def test_key_is_announced_once_only_after_successful_commit(self): resolve_startup_key(again, SimpleNamespace(api_key="")) announce_default_key(again) self.assertEqual(terminal.getvalue().count(key), 1) + self.assertNotIn(key, output.getvalue()) self.assertEqual(again["api_key"], key) self.assertIsNone(next(row for row in resolve_settings(config) if row["key"] == "api_key")["value"]) self.assertNotIn(key, sanitize_log_text("key leaked " + key)) with patch("sys.stderr", io.StringIO()): resolve_startup_key(self.config(), SimpleNamespace(api_key="")) + def test_missing_terminal_at_disclosure_does_not_consume_announcement(self): + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + self.console_open.side_effect = OSError("terminal gone") + with self.assertRaises(OSError): + announce_default_key(config) + self.assertEqual(self.state.default_key(), (config["api_key"], True)) + self.assertEqual(terminal.getvalue(), "") + + def test_announcement_commit_failure_never_writes_terminal(self): + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + with patch.object(self.state, "claim_announcement", side_effect=sqlite3.OperationalError("locked")), \ + self.assertRaises(sqlite3.OperationalError): + announce_default_key(config) + self.assertEqual(terminal.getvalue(), "") + self.assertEqual(self.state.default_key(), (config["api_key"], True)) + + def test_explicit_sources_do_not_overwrite_saved_default_or_print(self): key, _ = self.state.default_key(create=True) for source in ("cli", "environment", "dotenv"): @@ -279,6 +351,7 @@ def test_full_startup_precedence_and_default_restore(self): def boot(env=None, flags=()): observed = {} + terminal = self.console() with contextlib.chdir(self.root), contextlib.ExitStack() as stack: stack.enter_context(patch.dict(os.environ, {"HOME": str(self.root), "CODEBUDDY_AUTH_DIR": str(self.root), **(env or {})}, clear=True)) stack.enter_context(patch.dict(converter.CONFIG)) @@ -296,11 +369,13 @@ def serve(app, config, **kwargs): stack.enter_context(patch.object(converter, "run_server", side_effect=serve)) converter.main() observed["output"] = output.getvalue() + observed["terminal"] = terminal.getvalue() return observed first = boot() key = first["key"] - self.assertEqual(first["output"].count(key), 1) + self.assertEqual(first["terminal"].count(key), 1) + self.assertNotIn(key, first["output"]) self.assertEqual(first["sources"]["api_key"], "generated") envfile = self.root / ".env" envfile.write_text("CODEBUDDY2API_PORT=9092\nCODEBUDDY2API_KEY=file-fixture\n") @@ -315,6 +390,7 @@ def serve(app, config, **kwargs): resumed = boot() self.assertEqual(resumed["key"], key) self.assertNotIn(key, resumed["output"]) + self.assertEqual(resumed["terminal"], "") self.assertEqual(self.state.default_key(), (key, False)) self.assertFalse(list(self.root.glob("*.json"))) @@ -325,7 +401,8 @@ def test_listener_failure_does_not_consume_announcement(self): from unittest.mock import AsyncMock from app.startup import run_server config = self.config() - with patch("sys.stderr", Terminal()) as terminal: + terminal = self.console() + with patch("sys.stderr", io.StringIO()): resolve_startup_key(config, SimpleNamespace(api_key="")) key = config["api_key"] with patch.object(uvicorn.Server, "startup", new=AsyncMock(side_effect=SystemExit(3))), \ diff --git a/tests/test_terminal_output.py b/tests/test_terminal_output.py new file mode 100644 index 0000000..d9988e5 --- /dev/null +++ b/tests/test_terminal_output.py @@ -0,0 +1,114 @@ +"""Verify secret disclosure uses an actual terminal instead of stdout/stderr.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import contextlib +import io +import os +import sqlite3 +import stat +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from app.startup import terminal_stream + + +class TerminalOutputTests(unittest.TestCase): + def test_regular_file_and_non_tty_device_are_rejected_and_closed(self): + for mode, tty in ((stat.S_IFREG, True), (stat.S_IFCHR, False)): + with self.subTest(mode=mode, tty=tty), \ + patch('app.startup.os.open', return_value=123), \ + patch('app.startup.os.fstat', return_value=type('Stat', (), {'st_mode': mode})()), \ + patch('app.startup.os.isatty', return_value=tty), \ + patch('app.startup.os.close') as close, \ + patch('app.startup.os.fdopen') as fdopen: + with self.assertRaises(OSError): + terminal_stream() + close.assert_called_once_with(123) + fdopen.assert_not_called() + + def test_platform_console_path_is_fixed(self): + for platform, device in (('nt', 'CONOUT$'), ('posix', '/dev/tty')): + with self.subTest(platform=platform), \ + patch('app.startup.os.name', platform), \ + patch('app.startup.os.open', return_value=123) as opened, \ + patch('app.startup.os.fstat', return_value=type('Stat', (), {'st_mode': stat.S_IFCHR})()), \ + patch('app.startup.os.isatty', return_value=True), \ + patch('app.startup.os.fdopen', return_value=io.StringIO()): + with terminal_stream(): + pass + self.assertEqual(opened.call_args.args[0], device) + self.assertFalse(opened.call_args.args[1] & os.O_CREAT) + + @unittest.skipUnless(os.name == 'posix', 'Real controlling PTY requires POSIX') + def test_real_pty_with_redirected_standard_streams_discloses_once(self): + import pty + with tempfile.TemporaryDirectory() as directory: + database = Path(directory) / 'control.sqlite3' + for attempt in range(2): + master, slave = pty.openpty() + try: + script = ''' +import fcntl, os, sys, termios +from types import SimpleNamespace +from app.control_store import ControlStore +from app.startup import resolve_startup_key, announce_default_key +os.setsid() +fcntl.ioctl(int(sys.argv[1]), termios.TIOCSCTTY, 0) +control = ControlStore(sys.argv[2]) +try: + config = {"settings_sources": {}, "state_store": control.state, "host": "127.0.0.1"} + resolve_startup_key(config, SimpleNamespace(api_key="")) + announce_default_key(config) + announce_default_key(config) +finally: + control.close() +''' + result = subprocess.run([sys.executable, '-B', '-c', script, str(slave), str(database)], + pass_fds=(slave,), stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=20, check=True) + with contextlib.closing(sqlite3.connect(database)) as db: + key, announced = db.execute('SELECT value,announced FROM gateway_secrets').fetchone() + self.assertEqual(announced, 1) + os.set_blocking(master, False) + try: + output = os.read(master, 16384) + except BlockingIOError: + output = b'' + self.assertEqual(output.count(key.encode()), 1 if attempt == 0 else 0) + self.assertEqual((result.stdout, result.stderr), (b'', b'')) + finally: + os.close(slave) + os.close(master) + + @unittest.skipUnless(os.name == 'posix', 'Detached process requires POSIX') + def test_no_controlling_terminal_cannot_create_hidden_key(self): + with tempfile.TemporaryDirectory() as directory: + database = Path(directory) / 'control.sqlite3' + script = ''' +import sys +from types import SimpleNamespace +from app.control_store import ControlStore +from app.startup import resolve_startup_key +control = ControlStore(sys.argv[1]) +try: + try: + resolve_startup_key({"settings_sources": {}, "state_store": control.state, "host": "127.0.0.1"}, + SimpleNamespace(api_key="")) + except ValueError: + assert control.state.default_key() == (None, False) + else: + raise AssertionError("Headless startup unexpectedly generated a key") +finally: + control.close() +''' + result = subprocess.run([sys.executable, '-B', '-c', script, str(database)], + start_new_session=True, capture_output=True, timeout=20, check=True) + self.assertEqual((result.stdout, result.stderr), (b'', b'')) + + +if __name__ == '__main__': + unittest.main() From 22778a6c28710fcafeb07028b5551e59deee96c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?= <277378677+maiphucgiang@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:05:24 +0800 Subject: [PATCH 3/4] Honor explicit anonymous startup before generating keys Use the existing unsafe opt-in for unconfigured headless launches while keeping saved and externally supplied keys authoritative. --- app/startup.py | 7 +++++ converter.py | 4 +-- docs/advanced.md | 4 +-- docs/advanced.zh-CN.md | 4 +-- tests/test_native_noauth.py | 57 +++++++++++++++++++++++++++++++++ tests/test_sqlite_state.py | 63 +++++++++++++++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 tests/test_native_noauth.py diff --git a/app/startup.py b/app/startup.py index d5a8834..3193a50 100644 --- a/app/startup.py +++ b/app/startup.py @@ -53,6 +53,11 @@ def terminal_stream(): raise +def allows_open_noauth(): + """Honor the existing explicit unsafe opt-in without overriding a configured key.""" + return os.environ.get("CODEBUDDY2API_ALLOW_OPEN_NOAUTH", "").lower() in ("1", "true", "yes") + + def resolve_startup_key(config, args, dotenv_keys=()): sources = config["settings_sources"] for name, spec in SCHEMA.items(): @@ -63,6 +68,8 @@ def resolve_startup_key(config, args, dotenv_keys=()): return store = config["state_store"] key, pending = store.default_key() + if key is None and allows_open_noauth(): + return if key is None and config["host"] not in ("127.0.0.1", "::1", "localhost"): raise ValueError("非回环监听请显式设置 API key；默认密钥仅在本地首次启动时生成") if key is None or pending: diff --git a/converter.py b/converter.py index e81ab88..fab676b 100644 --- a/converter.py +++ b/converter.py @@ -72,7 +72,7 @@ def desensitize_body(body, roles=("system",), desensitize_harness_user=False, from app.content_filter import ContentFilterDetector, is_filter_error from app.request_limits import ImageLimitError, apply_image_policy from app.safe_logging import format_log_body, sanitize_log_text -from app.startup import load_startup_env, run_server +from app.startup import allows_open_noauth, load_startup_env, run_server from app.site_routing import (DOMESTIC, INTERNATIONAL, PROFILE_ENDPOINTS, site_for_auth, site_for_headers, profile_for_auth, profile_for_headers, profile_region, profile_product, profile_site, chat_url_for_headers, refresh_url_for_auth) @@ -3867,7 +3867,7 @@ def main(): runtime_management.initialize(sys.modules[__name__], args, parser=ap, dotenv_keys=dotenv_keys) # Validate effective binding and authentication before credential scans or background work. if (args.host not in ("127.0.0.1", "::1", "localhost") and not CONFIG.get("api_key") - and os.environ.get("CODEBUDDY2API_ALLOW_OPEN_NOAUTH", "").lower() not in ("1", "true", "yes")): + and not allows_open_noauth()): runtime_management.close(CONFIG) ap.error("非回环绑定且未设置 API key 会匿名开放推理额度；" "请设置 CODEBUDDY2API_KEY，或确知风险后以 CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true 显式放行") diff --git a/docs/advanced.md b/docs/advanced.md index 30bb987..604ac66 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -191,11 +191,11 @@ Both international profiles merge image-bearing consecutive `user` runs only aft - Valid upstream `Retry-After` values (0–86400 seconds or equivalent HTTP dates) are returned as seconds before streaming starts; 429 only cools the selected account/model. Invalid or expired values fall back to the body's reset time or 600 seconds. Pool-generated 429 responses include the remaining wait. - Chat and Responses preserve an explicit client `prompt_cache_key` without generating one; cache hits and savings depend on the upstream. - Unsupported capabilities are rejected rather than silently degraded: chat `n` other than 1 and the Responses state fields `previous_response_id`/`conversation` (this gateway keeps no server-side response state) return 400; length-truncated or content-filtered Responses are reported as `incomplete`, never disguised as `completed`. -- Text logs and SQLite auditing have separate budgets. Logs contain bounded, redacted previews, not complete original requests. Treat logs, credential exports and backups as private data. +- SQLite auditing retains only bounded, redacted diagnostics, not complete original requests. Treat logs, credential exports and backups as private data. ## Deployment exposure and credential intake -- The compose port mapping binds loopback by default (`CODEBUDDY2API_BIND` defaults to 127.0.0.1); after resolving CLI, environment and saved settings, a native non-loopback bind with an empty effective API key refuses to start unless `CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true` is set explicitly. +- Compose maps loopback by default. Without a configured or saved key, `CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true` explicitly permits headless/non-loopback inference without generating a key; management stays locked. The shipped image sets this compatibility opt-in, so configure `CODEBUDDY2API_KEY` before exposing it. The opt-in never disables an existing key. - When a key is configured, generation and token-count POSTs verify request headers before buffering bodies or reserving inference capacity; invalid keys return 401 even while generation slots are full. Other routes retain their existing authentication and routing behavior. - Credential imports/uploads persist the normalized form (token aliases folded into the canonical fields); strict JSON parsing rejects NaN/Infinity, and `expiresAt`/`lastRefreshTime` must be plausible finite millisecond timestamps. diff --git a/docs/advanced.zh-CN.md b/docs/advanced.zh-CN.md index dd7f8fe..2a09e06 100644 --- a/docs/advanced.zh-CN.md +++ b/docs/advanced.zh-CN.md @@ -191,11 +191,11 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` - 上游有效 `Retry-After`（0–86400 秒或对应 HTTP 日期）规范化为秒并在开流前返回；429 仅冷却对应账号/模型。无效或过期值回落正文重置时间或默认 600 秒；本地全凭据冷却的 429 返回剩余等待秒数。 - Chat 与 Responses 保留客户端显式 `prompt_cache_key`，不自动生成；缓存命中和节费取决于上游。 - 不支持的能力显式拒绝而非静默降级：Chat 的 `n≠1`、Responses 的 `previous_response_id`/`conversation`（本网关不保存服务端响应状态）返回 400；长度截断或审核过滤的 Responses 标记为 `incomplete`，不伪装为 `completed`。 -- 兼容文本日志和 SQLite 审计使用独立预算；日志仅记录有界、脱敏预览，不是完整原始请求。日志、凭证导出和备份仍须按私有数据保管。 +- SQLite 审计只保留有界、脱敏诊断，不是完整原始请求。日志、凭证导出和备份仍须按私有数据保管。 ## 部署暴露与凭据导入 -- Compose 端口映射默认只绑回环（`CODEBUDDY2API_BIND` 默认 127.0.0.1）；原生运行在合并 CLI、环境和持久化设置后，若实际地址非回环且生效 key 为空则拒绝启动，须显式设 `CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true` 放行。 +- Compose 默认映射回环地址。没有配置或已保存的 key 时，`CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true` 显式允许后台／非回环匿名推理，不生成密钥，管理仍锁定。镜像保留此兼容开关，对外暴露前请配置 `CODEBUDDY2API_KEY`；该开关不会停用已有 key。 - 配置 key 时，生成及 token 估算 POST 在缓冲请求体、预留推理名额之前校验请求头；即使名额已满，无效 key 仍返回 401。其他路由保留原有鉴权和路由行为。 - 凭据导入/上传在落盘前把 token 别名归一化为官方字段名；严格 JSON 解析拒绝 NaN/Infinity，`expiresAt`/`lastRefreshTime` 必须是合理的有限毫秒时间戳。 diff --git a/tests/test_native_noauth.py b/tests/test_native_noauth.py new file mode 100644 index 0000000..6083ad2 --- /dev/null +++ b/tests/test_native_noauth.py @@ -0,0 +1,57 @@ +"""Exercise the shipped image's no-TTY startup contract with disposable state.""" +import contextlib +import json +import os +from pathlib import Path +import signal +import socket +import sqlite3 +import subprocess +import sys +import tempfile +import unittest + +import httpx + +ROOT = Path(__file__).resolve().parents[1] + + +class NativeNoAuthTests(unittest.TestCase): + def test_image_default_starts_without_key_but_keeps_management_locked(self): + dockerfile = (ROOT / 'Dockerfile').read_text() + command = json.loads(next(line[4:] for line in dockerfile.splitlines() if line.startswith('CMD '))) + self.assertIn('ENV CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true', dockerfile) + with tempfile.TemporaryDirectory() as directory, socket.socket() as probe: + root = Path(directory) + probe.bind(('127.0.0.1', 0)) + port = probe.getsockname()[1] + probe.close() + env = {name: os.environ[name] for name in ('PATH', 'SYSTEMROOT') if name in os.environ} + env.update(HOME=directory, USERPROFILE=directory, TMPDIR=directory, TEMP=directory, + CODEBUDDY_AUTH_DIR=str(root / 'auth'), CODEBUDDY2API_ALLOW_OPEN_NOAUTH='true', + PYTHONDONTWRITEBYTECODE='1') + process = subprocess.Popen([sys.executable, '-B', str(ROOT / command[1]), *command[2:], '--port', str(port)], + cwd=root, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) + try: + with httpx.Client(base_url=f'http://127.0.0.1:{port}', transport=httpx.HTTPTransport(retries=5), + timeout=5, trust_env=False) as client: + self.assertEqual(client.get('/health').json(), {'status': 'ok'}) + self.assertEqual(client.get('/admin/session').status_code, 503) + self.assertEqual(client.get('/v1/models').status_code, 200) + with contextlib.closing(sqlite3.connect(root / 'auth/control.sqlite3')) as db: + self.assertEqual(db.execute('SELECT count(*) FROM gateway_secrets').fetchone()[0], 0) + finally: + if process.poll() is None: + process.send_signal(signal.SIGINT) if os.name == 'posix' else process.terminate() + try: + output, error = process.communicate(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5) + self.fail('Owned no-auth fixture did not shut down') + self.assertNotIn(b'cb-', output + error) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_sqlite_state.py b/tests/test_sqlite_state.py index 3f88296..12978b2 100644 --- a/tests/test_sqlite_state.py +++ b/tests/test_sqlite_state.py @@ -320,6 +320,69 @@ def test_nonterminal_and_public_first_start_do_not_generate_hidden_key(self): resolve_startup_key(config, SimpleNamespace(api_key="")) self.assertEqual(self.state.default_key(), (None, False)) + def test_explicit_noauth_opt_in_skips_first_key_generation_without_a_terminal(self): + for value in ("1", "true", "yes", "TRUE"): + for host in ("127.0.0.1", "0.0.0.0"): + config = {**self.config(), "host": host} + with self.subTest(value=value, host=host), \ + patch.dict(os.environ, {"CODEBUDDY2API_ALLOW_OPEN_NOAUTH": value}, clear=True): + resolve_startup_key(config, SimpleNamespace(api_key="")) + self.assertFalse(config["api_key"]) + self.assertFalse(config["announce_default_key"]) + self.assertEqual(self.state.default_key(), (None, False)) + self.console_open.assert_not_called() + + def test_disabled_noauth_opt_in_keeps_first_start_protection(self): + for value in ("", "0", "false", "no", "invalid"): + with self.subTest(value=value), \ + patch.dict(os.environ, {"CODEBUDDY2API_ALLOW_OPEN_NOAUTH": value}, clear=True), \ + self.assertRaises(ValueError): + resolve_startup_key({**self.config(), "host": "0.0.0.0"}, SimpleNamespace(api_key="")) + self.assertEqual(self.state.default_key(), (None, False)) + + def test_noauth_opt_in_never_downgrades_an_existing_default_or_override(self): + key, _ = self.state.default_key(create=True) + with patch.dict(os.environ, {"CODEBUDDY2API_ALLOW_OPEN_NOAUTH": "true"}, clear=True): + with self.assertRaises(ValueError): + resolve_startup_key(self.config(), SimpleNamespace(api_key="")) + self.state.claim_announcement(key) + config = {**self.config(), "host": "0.0.0.0"} + resolve_startup_key(config, SimpleNamespace(api_key="")) + self.assertEqual(config["api_key"], key) + for source in ("cli", "environment", "dotenv"): + config = self.config(source, "configured-fixture") + resolve_startup_key(config, SimpleNamespace(api_key="configured-fixture")) + self.assertEqual(config["api_key"], "configured-fixture") + + def test_docker_default_without_key_or_terminal_reaches_server(self): + import converter + from fastapi import FastAPI + dockerfile = (Path(__file__).resolve().parents[1] / "Dockerfile").read_text() + command = json.loads(next(line[4:] for line in dockerfile.splitlines() if line.startswith("CMD "))) + self.assertIn("ENV CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true", dockerfile) + with contextlib.chdir(self.root), contextlib.ExitStack() as stack: + stack.enter_context(patch.dict(os.environ, {"HOME": str(self.root), "CODEBUDDY_AUTH_DIR": str(self.root), + "CODEBUDDY2API_ALLOW_OPEN_NOAUTH": "true"}, clear=True)) + stack.enter_context(patch.dict(converter.CONFIG)) + stack.enter_context(patch.object(converter, "app", FastAPI())) + stack.enter_context(patch.object(sys, "argv", command[1:])) + stack.enter_context(patch.object(converter, "seed_credentials")) + stack.enter_context(patch.object(converter, "CredentialPool")) + stack.enter_context(patch.object(converter, "_publish_model_cache")) + stack.enter_context(patch.object(converter.threading, "Thread")) + stack.enter_context(patch("sys.stderr", io.StringIO())) + captured = {} + def serve(app, config, **kwargs): + captured.update(kwargs, key=config["api_key"], announcement=config["announce_default_key"]) + stack.enter_context(patch.object(converter, "run_server", side_effect=serve)) + converter.main() + self.assertEqual((captured["host"], captured["port"]), ("0.0.0.0", 8787)) + self.assertFalse(captured["key"]) + self.assertFalse(captured["announcement"]) + self.assertEqual(self.state.default_key(), (None, False)) + self.console_open.assert_not_called() + + def test_dotenv_is_optional_and_process_environment_wins(self): path = self.root / ".env" with patch.dict(os.environ, {}, clear=True): From b20f2fb9884e599cf9e14fa93b95d2a7632c304f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A2=A8=E8=8F=8A?= <277378677+maiphucgiang@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:20:51 +0800 Subject: [PATCH 4/4] Keep default-key disclosure recoverable after output failures Commit the announcement marker after terminal flush under an exclusive transaction. Roll back failed writes, interrupted disclosures and failed commits so the saved key remains recoverable. Document possible redisclosure after a crash between terminal output and commit. --- app/startup.py | 7 ++- app/state_store.py | 8 ++- docs/deployment.md | 2 + docs/deployment.zh-CN.md | 2 + tests/test_sqlite_state.py | 110 +++++++++++++++++++++++++++++++++++-- 5 files changed, 118 insertions(+), 11 deletions(-) diff --git a/app/startup.py b/app/startup.py index 3193a50..d68c88f 100644 --- a/app/startup.py +++ b/app/startup.py @@ -90,9 +90,10 @@ def announce_default_key(config): return with terminal_stream() as terminal: key = config["api_key"] - if config["state_store"].claim_announcement(key): - terminal.write(f"\n默认 API key：{key}\n已保存到 control.sqlite3，仅显示这一次；管理登录与 API 请求共用。\n") - terminal.flush() + with config["state_store"].claim_announcement(key) as claimed: + if claimed: + terminal.write(f"\n默认 API key：{key}\n已保存到 control.sqlite3，仅显示这一次；管理登录与 API 请求共用。\n") + terminal.flush() config["announce_default_key"] = False diff --git a/app/state_store.py b/app/state_store.py index 256b157..ff35031 100644 --- a/app/state_store.py +++ b/app/state_store.py @@ -269,8 +269,10 @@ def default_key(self, *, create=False): self._db.execute("INSERT INTO gateway_secrets(name,value,announced) VALUES('default_api_key',?,0)", (key,)) return key, True + @contextmanager def claim_announcement(self, key): - """Commit before printing: never repeat a secret after a crash or concurrent startup.""" + """Serialize disclosure and roll back the marker if terminal output or commit fails.""" with self.transaction(): - return self._db.execute("UPDATE gateway_secrets SET announced=1 WHERE name='default_api_key' " - "AND value=? AND announced=0", (key,)).rowcount == 1 + claimed = self._db.execute("UPDATE gateway_secrets SET announced=1 WHERE name='default_api_key' " + "AND value=? AND announced=0", (key,)).rowcount == 1 + yield claimed diff --git a/docs/deployment.md b/docs/deployment.md index 3630067..40c4a32 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -70,6 +70,8 @@ Local startup does not require `.env`. Without an explicit key, the first loopba One-time disclosure goes directly to the controlling terminal (`/dev/tty` or Windows `CONOUT$`), not stdout/stderr; redirecting standard output does not capture the key. Terminal recording remains outside the gateway's control. +A failed terminal write or database commit leaves the same key pending for the next interactive start. A crash between display and commit can therefore repeat the notice; a successfully committed display is not repeated. + Without uv, run `python3 -m venv .venv`, activate it, install dependencies with `pip install --require-hashes --only-binary=:all: -r requirements.txt`, then run `python3 converter.py`. Distribution archives include the WebUI; source installs still need the frontend build. Both commands optionally read `.env` in the current working directory, never parent directories. Precedence: explicit CLI > process environment > `.env` > saved SQLite values > defaults. Overrides do not replace the saved default key; an explicitly empty key still locks management. Listener changes require restart; `CODEBUDDY2API_IMAGE/AUTH_PATH` are Compose-only. diff --git a/docs/deployment.zh-CN.md b/docs/deployment.zh-CN.md index c00ed9d..569756a 100644 --- a/docs/deployment.zh-CN.md +++ b/docs/deployment.zh-CN.md @@ -70,6 +70,8 @@ uv run converter.py 密码提示直接写入控制终端（`/dev/tty` 或 Windows `CONOUT$`），不经过 stdout/stderr，重定向标准输出不会收集密码；外部终端录制不在程序控制范围内。 +终端写入或数据库提交失败时，同一密钥保留为待显示，下次交互启动可重试。显示后、提交前崩溃可能再次提示；成功提交后不重复显示。 + 没有 uv 时：运行 `python3 -m venv .venv` 并激活，用 `pip install --require-hashes --only-binary=:all: -r requirements.txt` 安装依赖，再执行 `python3 converter.py`。发行包已包含 WebUI；源码安装仍需构建界面。 两种启动方式均可选读取当前工作目录的 `.env`，不搜索父目录。优先级：显式 CLI > 进程环境变量 > `.env` > SQLite 保存值 > 默认值。覆盖不改写已保存的默认 key；显式空 key 仍锁定管理。修改监听需重启，`CODEBUDDY2API_IMAGE/AUTH_PATH` 仅用于 Compose。 diff --git a/tests/test_sqlite_state.py b/tests/test_sqlite_state.py index 12978b2..0e6d607 100644 --- a/tests/test_sqlite_state.py +++ b/tests/test_sqlite_state.py @@ -243,8 +243,10 @@ def test_default_key_is_atomic_private_and_survives_restart(self): self.assertTrue(pending) self.assertNotIn(key, json.dumps(self.control.snapshot())) self.assertEqual(self.open_control().state.default_key(), (key, True)) - self.assertTrue(self.state.claim_announcement(key)) - self.assertFalse(other.state.claim_announcement(key)) + with self.state.claim_announcement(key) as claimed: + self.assertTrue(claimed) + with other.state.claim_announcement(key) as claimed: + self.assertFalse(claimed) self.assertEqual(other.state.default_key(), (key, False)) def test_broken_saved_key_is_not_regenerated(self): @@ -258,7 +260,7 @@ def config(self, source="default", key=""): return {"state_store": self.state, "control_store": self.control, "api_key": key, "host": "127.0.0.1", "settings_sources": {"api_key": source}} - def test_key_is_announced_once_only_after_successful_commit(self): + def test_successful_announcement_is_committed_and_not_repeated(self): config = self.config() terminal = self.console() with patch("sys.stderr", io.StringIO()) as output: @@ -288,7 +290,7 @@ def test_missing_terminal_at_disclosure_does_not_consume_announcement(self): self.assertEqual(self.state.default_key(), (config["api_key"], True)) self.assertEqual(terminal.getvalue(), "") - def test_announcement_commit_failure_never_writes_terminal(self): + def test_announcement_reservation_failure_never_writes_terminal(self): terminal = self.console() config = self.config() resolve_startup_key(config, SimpleNamespace(api_key="")) @@ -299,6 +301,103 @@ def test_announcement_commit_failure_never_writes_terminal(self): self.assertEqual(self.state.default_key(), (config["api_key"], True)) + def test_terminal_write_and_flush_failures_leave_default_key_pending(self): + for operation in ("write", "flush"): + with self.subTest(operation=operation): + self.control._db.execute("DELETE FROM gateway_secrets") + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + key = config["api_key"] + with patch.object(terminal, operation, side_effect=BrokenPipeError("fixture terminal lost")), \ + self.assertRaises(BrokenPipeError): + announce_default_key(config) + self.assertEqual(self.state.default_key(), (key, True)) + self.assertTrue(config["announce_default_key"]) + recovered = self.console() + announce_default_key(config) + self.assertEqual(recovered.getvalue().count(key), 1) + self.assertEqual(self.state.default_key(), (key, False)) + + def test_process_crash_during_disclosure_rolls_back_the_marker(self): + import subprocess + key, _ = self.state.default_key(create=True) + script = ''' +import os, sys +from app.control_store import ControlStore +control = ControlStore(sys.argv[1]) +key, _ = control.state.default_key() +with control.state.claim_announcement(key) as claimed: + assert claimed + os._exit(17) +''' + result = subprocess.run([sys.executable, "-B", "-c", script, str(self.control.path)], + capture_output=True, timeout=15) + self.assertEqual(result.returncode, 17) + self.assertEqual((result.stdout, result.stderr), (b"", b"")) + self.assertEqual(self.open_control().state.default_key(), (key, True)) + + + def test_commit_failure_after_disclosure_keeps_recovery_pending(self): + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + key = config["api_key"] + def deny_commit(action, arg1, arg2, database, trigger): + return sqlite3.SQLITE_DENY if action == sqlite3.SQLITE_TRANSACTION and arg1 == "COMMIT" else sqlite3.SQLITE_OK + self.control._db.set_authorizer(deny_commit) + try: + with self.assertRaises(sqlite3.DatabaseError): + announce_default_key(config) + finally: + self.control._db.set_authorizer(None) + self.assertFalse(self.control._db.in_transaction) + self.assertEqual(terminal.getvalue().count(key), 1) + self.assertEqual(self.state.default_key(), (key, True)) + self.assertTrue(config["announce_default_key"]) + recovered = self.console() + announce_default_key(config) + self.assertEqual(recovered.getvalue().count(key), 1) + self.assertEqual(self.state.default_key(), (key, False)) + + def test_concurrent_disclosures_print_only_after_acquiring_exclusive_claim(self): + import threading + terminal = self.console() + key, _ = self.state.default_key(create=True) + other = self.open_control() + configs = [{**self.config(), "state_store": state, "api_key": key, "announce_default_key": True} + for state in (self.state, other.state)] + barrier = threading.Barrier(2, timeout=5) + def announce(config): + barrier.wait() + announce_default_key(config) + with concurrent.futures.ThreadPoolExecutor(2) as executor: + list(executor.map(announce, configs)) + self.assertEqual(terminal.getvalue().count(key), 1) + self.assertEqual(self.state.default_key(), (key, False)) + self.assertTrue(all(not config["announce_default_key"] for config in configs)) + + def test_terminal_failure_after_binding_shuts_down_with_pending_key(self): + import asyncio + import uvicorn + from fastapi import FastAPI + from unittest.mock import AsyncMock + from app.startup import run_server + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + async def ready(server, sockets=None): + server.started = True + with patch.object(uvicorn.Server, "startup", ready), \ + patch.object(uvicorn.Server, "shutdown", new=AsyncMock()) as shutdown, \ + patch.object(uvicorn.Server, "run", lambda server: asyncio.run(server.startup())), \ + patch.object(terminal, "flush", side_effect=BrokenPipeError("fixture")), \ + self.assertRaises(BrokenPipeError): + run_server(FastAPI(), config, host="127.0.0.1", port=8787) + shutdown.assert_awaited_once() + self.assertEqual(self.state.default_key(), (config["api_key"], True)) + + def test_explicit_sources_do_not_overwrite_saved_default_or_print(self): key, _ = self.state.default_key(create=True) for source in ("cli", "environment", "dotenv"): @@ -345,7 +444,8 @@ def test_noauth_opt_in_never_downgrades_an_existing_default_or_override(self): with patch.dict(os.environ, {"CODEBUDDY2API_ALLOW_OPEN_NOAUTH": "true"}, clear=True): with self.assertRaises(ValueError): resolve_startup_key(self.config(), SimpleNamespace(api_key="")) - self.state.claim_announcement(key) + with self.state.claim_announcement(key): + pass config = {**self.config(), "host": "0.0.0.0"} resolve_startup_key(config, SimpleNamespace(api_key="")) self.assertEqual(config["api_key"], key)