diff --git a/.env.example b/.env.example index 1d84684..33534d6 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # Copy to .env for first use; preserve existing settings when updating. -# Compose loads .env automatically; native launches use uv run --env-file .env. +# Compose and native launches read .env; native zero-configuration startup can omit it. # Native listener and Compose host mapping; explicit --host/--port override native values. CODEBUDDY2API_BIND=127.0.0.1 @@ -15,6 +15,7 @@ CODEBUDDY2API_AUTH_PATH=./auth # CODEBUDDY_IMPORT_DIR=./auth/imports # Empty locks management; set a random key before exposing inference beyond loopback. +# Native: omit/comment the variable to use the saved default key (first local start displays it once). CODEBUDDY2API_KEY= # Native-only unsafe opt-in; Compose uses its host mapping to control exposure. # CODEBUDDY2API_ALLOW_OPEN_NOAUTH=false @@ -48,9 +49,9 @@ CODEBUDDY2API_MAX_CONCURRENT=64 # Disable only declared model-capability preflight; other safeguards and Intl image merging remain enabled. # CODEBUDDY2API_MODEL_CAPABILITY_GUARD=true -# SQLite audit defaults to the data directory; this optional path enables a separate text log. +# Retired: this setting only warns; all persistent logs use logs.sqlite3. CODEBUDDY2API_LOG= -# Redacted text preview bytes; zero logs metadata only. +# Legacy text-preview limit; text output is retired. SQLite diagnostics have a separate budget. CODEBUDDY2API_LOG_BODY_LIMIT=65536 # International trial credits remain manual-only in the WebUI. diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1aa9ee6..67a119a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -60,6 +60,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` @@ -101,3 +103,5 @@ jobs: uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" + ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/head', github.event.pull_request.number) || github.ref }} + sha: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/README.md b/README.md index 54ca005..fd97942 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The template binds to localhost only. Configure HTTPS and restrict network acces ## FAQ - **WebUI sign-in fails behind an HTTPS domain/reverse proxy?** Trust your public origin via `admin_allowed_origins` — see [Management Origin checks](docs/advanced.md#management-origin--csrf-switch). -- **No Docker on the server?** Run the terminal login and the gateway with Python directly — see [Local Python setup](docs/deployment.md#local-python-setup). +- **No Docker?** After installing dependencies and the WebUI, run `uv run converter.py` or `python3 converter.py` without `.env`. First local startup saves a default key and displays it once — see [Local Python setup](docs/deployment.md#local-python-setup). - **Where is my data?** Everything lives in `auth/` (or `/data/auth` in Docker): credentials, settings and log databases — see [Data and backups](docs/webui.md#data-and-backups). - **Which image tag should I use?** `latest` follows stable releases, `edge` follows main, version tags pin one release — see [Published images](docs/deployment.md#use-published-images). diff --git a/README.zh-CN.md b/README.zh-CN.md index 621a835..506bf57 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -82,7 +82,7 @@ docker compose up -d --no-build ## 常见问题 - **绑定域名后经 HTTPS 反代无法登录 WebUI？** 将对外来源加入 `admin_allowed_origins` 信任列表——见[管理 Origin 校验](docs/advanced.zh-CN.md#管理-origin--csrf-开关)。 -- **服务器没有 Docker？** 可用 Python 直接完成终端登录并运行网关——见[本地 Python 运行](docs/deployment.zh-CN.md#本地-python-运行)。 +- **没有 Docker？** 准备依赖和 WebUI 后，直接 `uv run converter.py` 或 `python3 converter.py`，无需 `.env`；首次本地启动保存默认 key 并仅在终端显示一次——见[本地 Python 运行](docs/deployment.zh-CN.md#本地-python-运行)。 - **数据在哪里？** 全部位于 `auth/`（Docker 中为 `/data/auth`）：凭证、设置与日志数据库——见[数据与备份](docs/webui.zh-CN.md#数据与备份)。 - **镜像标签怎么选？** `latest` 跟随稳定版，`edge` 跟随 main，版本标签固定某一发行版——见[使用已发布镜像](docs/deployment.zh-CN.md#使用已发布镜像)。 diff --git a/app/admin_auth.py b/app/admin_auth.py index 79d3945..58f9e45 100644 --- a/app/admin_auth.py +++ b/app/admin_auth.py @@ -10,6 +10,7 @@ from pathlib import Path import re import secrets +import sqlite3 import stat import tempfile import threading @@ -24,11 +25,7 @@ class SessionStoreError(RuntimeError): - """A superseded session snapshot could not be durably revoked. - - Startup must abort on this: the surviving snapshot could otherwise be adopted by a - later start under the superseded key, resurrecting sessions that were meant to die. - """ + """Authoritative session state could not be read or durably revoked.""" # Optional persisted session table so a restart does not force another login. @@ -132,14 +129,15 @@ def __init__(self, config, *, clock=time.monotonic, wall_clock=time.time): self.failures = OrderedDict() self._configured_key = None self._identity = None - self._path = _session_path(config.get("session_path")) + self._store = config.get("state_store") + self._path = self._store.path if self._store is not None else _session_path(config.get("session_path")) self._storage_error = None # Set when the snapshot could not be written or cleared. @staticmethod def _fingerprint(key): """Keyed fingerprint naming the key epoch a snapshot belongs to. - It identifies the epoch; revocation itself is performed by clearing the file. + It identifies the epoch; revocation clears its persisted snapshot. """ return hmac.new(key.encode(), _SESSION_KEY_LABEL, hashlib.sha256).hexdigest() @@ -179,6 +177,14 @@ def storage(self): def _revoke(self): """Revoke the persisted snapshot; returns False when it could not be cleared.""" + if self._store is not None: + try: + self._store.delete("sessions") + except (OSError, ValueError, sqlite3.Error) as error: + self._storage_failed(error) + return False + self._storage_ok() + return True if self._path is None: return True try: @@ -209,6 +215,14 @@ def _restore(self, key): Returns False when a snapshot exists that cannot be trusted or validated, so the caller revokes it instead of leaving it available to a later start. """ + if self._store is not None: + try: + document = self._store.get("sessions") + self._storage_ok() + return True if document is None else self._adopt(document, key) + except (OSError, ValueError, sqlite3.Error) as error: + self._storage_failed(error) + raise SessionStoreError("无法读取 SQLite 会话状态；保留原记录，请修复控制库后重启") from None if self._path is None: return True try: @@ -237,6 +251,9 @@ def _restore(self, key): except (ValueError, UnicodeDecodeError, RecursionError): # RecursionError: size-bounded but deeply nested JSON still exhausts the parser. return False + return self._adopt(document, key) + + def _adopt(self, document, key): if (not isinstance(document, dict) or set(document) != {"version", "fingerprint", "sessions"} or type(document["version"]) is not int or document["version"] != SESSION_FILE_VERSION): return False @@ -280,6 +297,14 @@ def _persist(self): document = {"version": SESSION_FILE_VERSION, "fingerprint": self._fingerprint(self._configured_key), "sessions": {sid: {"csrf_token": item["csrf_token"], "expires": item["expires"]} for sid, item in self.sessions.items()}} + if self._store is not None: + try: + self._store.put("sessions", document) + self._storage_ok() + return True + except (OSError, ValueError, sqlite3.Error) as error: + self._storage_failed(error) + return self._revoke() try: content = json.dumps(document, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") @@ -322,7 +347,12 @@ def _key(self): if restoring: # Adopt only a snapshot that matches the current epoch; anything else is # revoked, so switching back to an old key cannot resurrect its sessions. - durable = self._restore(key) or self._revoke() + try: + durable = self._restore(key) or self._revoke() + except SessionStoreError: + self._configured_key = previous + self.sessions.clear() + raise else: # A rotated or cleared key revokes every session, on disk as well. durable = self._persist() @@ -452,7 +482,7 @@ async def no_cache(message): # The key epoch changed but its superseded snapshot survives. Deny rather than # continue: startup refuses this case, so it is only reachable mid-process. return await error_response( - 503, "会话快照无法持久撤销，管理接口已锁定；请检查管理目录权限后重启")(scope, receive, no_cache) + 503, "会话快照无法安全读取或持久化，管理接口已锁定；请检查控制库后重启")(scope, receive, no_cache) if not enabled: return await error_response(503, "未配置 API key，管理接口已锁定")(scope, receive, no_cache) path, method = scope["path"], scope["method"] diff --git a/app/audit_store.py b/app/audit_store.py index 3f294a9..7a608a9 100644 --- a/app/audit_store.py +++ b/app/audit_store.py @@ -21,7 +21,7 @@ METRICS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_creation_tokens", "reasoning_tokens", "total_tokens", "credit") _IDENTIFIER = re.compile(r"^[A-Za-z0-9_.:/@-]{1,160}$") -_SECRET = re.compile(r"(?i)(bearer|sk-|access[_-]?token|refresh[_-]?token|api[_-]?key|eyJ|://)") +_SECRET = re.compile(r"(?i)(bearer|sk-|cb-|access[_-]?token|refresh[_-]?token|api[_-]?key|eyJ|://)") def safe_label(value: Any, limit: int = 160) -> str | None: diff --git a/app/control_store.py b/app/control_store.py index 038d899..6d3351d 100644 --- a/app/control_store.py +++ b/app/control_store.py @@ -70,12 +70,13 @@ def validate_model(source, rule, models=None, known_models=(), *, legacy_scopes= class ControlStore: - SCHEMA_VERSION = 1 + SCHEMA_VERSION = 2 def __init__(self, path): path = Path(path) existed = path.exists() path = _secure_path(path) + self.path = path self._lock = threading.RLock() self._db = sqlite3.connect(path, check_same_thread=False, isolation_level=None, timeout=1) try: @@ -88,7 +89,7 @@ def __init__(self, path): self._db.execute("CREATE TABLE control (id INTEGER PRIMARY KEY CHECK(id=1), revision INTEGER NOT NULL, payload TEXT NOT NULL)") self._db.execute("INSERT INTO control VALUES (1,0,?)", (json.dumps({"settings": {}, "models": {}, "credentials": {}}),)) self._db.execute("PRAGMA user_version=1") - elif version != self.SCHEMA_VERSION: + elif version not in (1, self.SCHEMA_VERSION): raise ValueError("管理数据库 schema 不受支持或已有库为空，未执行初始化") self._snapshot = self._load() self._db.execute("CREATE TABLE IF NOT EXISTS buddy_bootstrap (" @@ -106,12 +107,18 @@ def __init__(self, path): "accept_started INTEGER NOT NULL DEFAULT 0, chat_started INTEGER NOT NULL DEFAULT 0, " "completed INTEGER NOT NULL DEFAULT 0, model TEXT, chat_state TEXT NOT NULL DEFAULT 'pending', " "total_tokens INTEGER, updated_at REAL NOT NULL)") + self._db.execute("CREATE TABLE IF NOT EXISTS runtime_state (name TEXT PRIMARY KEY, payload TEXT NOT NULL)") + self._db.execute("CREATE TABLE IF NOT EXISTS state_imports (name TEXT PRIMARY KEY, imported INTEGER NOT NULL, migrated_at REAL NOT NULL)") + self._db.execute("CREATE TABLE IF NOT EXISTS gateway_secrets (name TEXT PRIMARY KEY, value TEXT NOT NULL, announced INTEGER NOT NULL DEFAULT 0 CHECK(announced IN (0,1)))") + self._db.execute("PRAGMA user_version=2") self._db.execute("COMMIT") except Exception: if self._db.in_transaction: self._db.execute("ROLLBACK") self._db.close() raise + from .state_store import StateStore + self.state = StateStore(self) def _load(self): row = self._db.execute("SELECT revision,payload FROM control WHERE id=1").fetchone() diff --git a/app/credential_cooldowns.py b/app/credential_cooldowns.py index bac7f40..2ba2490 100644 --- a/app/credential_cooldowns.py +++ b/app/credential_cooldowns.py @@ -17,7 +17,7 @@ import json import os import re -import stat +import sqlite3 import tempfile import threading import time @@ -95,8 +95,9 @@ class CredentialCooldowns: """Thread-safe, optionally persisted cooldown table keyed by validated account identity.""" def __init__(self, path=None, *, auth_ceiling_s: float = AUTH_CEILING_S, - model_ceiling_s: float = MODEL_CEILING_S): - self.path = str(path) if path else None + model_ceiling_s: float = MODEL_CEILING_S, store=None): + self._store = store + self.path = str(store.path) if store is not None else (str(path) if path else None) self.auth_ceiling_s = _bounded_ceiling(auth_ceiling_s, AUTH_CEILING_S) self.model_ceiling_s = _bounded_ceiling(model_ceiling_s, MODEL_CEILING_S) self._lock = threading.RLock() @@ -113,28 +114,8 @@ def __init__(self, path=None, *, auth_ceiling_s: float = AUTH_CEILING_S, def _load(self): """Adopt only a fully valid snapshot; anything else leaves the table empty.""" - try: - # Reject a symlink or FIFO *before* opening, so a device cannot block the read. - # This is best effort on Windows and is backed up by the fstat check below. - if not stat.S_ISREG(os.lstat(self.path).st_mode): - return - fd = os.open(self.path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - | getattr(os, "O_NONBLOCK", 0)) - except OSError: - return - try: - with os.fdopen(fd, "rb") as stream: - if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): - return - raw = stream.read(MAX_BYTES + 1) - except OSError: - return - if len(raw) > MAX_BYTES: - return - try: - document = _strict_json(raw) - except (ValueError, UnicodeDecodeError, RecursionError): - return + from .state_store import load_document + document = load_document(self.path, self._store, "cooldowns", MAX_BYTES) if (not isinstance(document, dict) or set(document) != {"version", "accounts"} or type(document["version"]) is not int or document["version"] != VERSION): return @@ -204,6 +185,16 @@ def _save_locked(self) -> bool: self.last_error = "serialize" self._dirty = True return False + if self._store is not None: + try: + self._store.put("cooldowns", json.loads(content)) + except (OSError, ValueError, sqlite3.Error) as error: + self.last_error = type(error).__name__ + self._dirty = True + return False + self.last_error = "capacity" if shed else None + self._dirty = shed + return not shed temporary = None try: directory = os.path.dirname(self.path) or "." diff --git a/app/credits.py b/app/credits.py index 0cc7278..20e33d1 100644 --- a/app/credits.py +++ b/app/credits.py @@ -630,13 +630,17 @@ def fetch_request_usage(access_token: str, days: int = USAGE_MAX_DAYS, class CreditLedger: """Persist per-credential check-in and balance snapshots for expiry-aware scheduling.""" - def __init__(self, path: Path): - self.path = Path(path) + def __init__(self, path: Path | None = None, *, store=None): + self._store = store + self.path = Path(store.path if store is not None else path) self._lock = threading.Lock() self._data: dict = {"version": 1, "creds": {}} self._load() def _load(self): + if self._store is not None: + self._data = self._store.get("credits") or {"version": 1, "creds": {}} + return try: self._data = json.loads(self.path.read_text(encoding="utf-8")) if "creds" not in self._data: @@ -645,6 +649,9 @@ def _load(self): self._data = {"version": 1, "creds": {}} def _save(self): + if self._store is not None: + self._store.put("credits", self._data) + return try: self.path.parent.mkdir(parents=True, exist_ok=True) tmp = self.path.with_suffix(self.path.suffix + ".tmp") @@ -737,8 +744,9 @@ class ModelCatalogCache: SCHEMA_VERSION = 2 - def __init__(self, path: Path, ttl: float = 6 * 3600): - self.path = Path(path) + def __init__(self, path: Path | None = None, ttl: float = 6 * 3600, *, store=None): + self._store = store + self.path = Path(store.path if store is not None else path) self.ttl = max(60.0, float(ttl or 0)) self._lock = threading.Lock() self._data: dict = {"version": self.SCHEMA_VERSION, "groups": {}} @@ -746,8 +754,10 @@ def __init__(self, path: Path, ttl: float = 6 * 3600): def _load(self): with self._lock: + d = (self._store.get("catalog") or self._data) if self._store is not None else None try: - d = json.loads(self.path.read_text(encoding="utf-8")) + if self._store is None: + d = json.loads(self.path.read_text(encoding="utf-8")) if (not isinstance(d, dict) or d.get("version") not in (1, self.SCHEMA_VERSION) or not isinstance(d.get("groups"), dict)): return @@ -773,6 +783,9 @@ def _load(self): pass # Keep the loaded catalog when disk reads fail. def _save(self): + if self._store is not None: + self._store.put_cache("catalog", self._data) + return try: self.path.parent.mkdir(parents=True, exist_ok=True) tmp = self.path.with_suffix(self.path.suffix + ".tmp") diff --git a/app/model_blocks.py b/app/model_blocks.py index 6b4958b..b7fb5fd 100644 --- a/app/model_blocks.py +++ b/app/model_blocks.py @@ -16,8 +16,9 @@ class ModelBlocks: """Maintain thread-safe per-backend model backoff with optional persistence.""" - def __init__(self, path=None, ttl_s: float = DEFAULT_TTL_S, max_ttl_s: float = MAX_TTL_S): - self.path = str(path) if path else None + def __init__(self, path=None, ttl_s: float = DEFAULT_TTL_S, max_ttl_s: float = MAX_TTL_S, *, store=None): + self._store = store + self.path = str(store.path) if store is not None else (str(path) if path else None) self.ttl_s = max(60.0, float(ttl_s or 0)) self.max_ttl_s = max(self.ttl_s, float(max_ttl_s or 0)) self._lock = threading.Lock() @@ -28,11 +29,14 @@ def __init__(self, path=None, ttl_s: float = DEFAULT_TTL_S, max_ttl_s: float = M # Persistence def _load(self): - try: - with open(self.path, "r", encoding="utf-8") as f: - data = json.load(f) - except (OSError, ValueError): - return + if self._store is not None: + data = self._store.get("model_blocks") + else: + try: + with open(self.path, "r", encoding="utf-8") as f: + data = json.load(f) + except (OSError, ValueError): + return if not isinstance(data, dict): return out: dict[str, dict] = {} @@ -52,6 +56,9 @@ def _load(self): self._data = out def _save_locked(self): + if self._store is not None: + self._store.put_cache("model_blocks", {"version": 1, "blocks": self._data}) + return if not self.path: return try: diff --git a/app/runtime_management.py b/app/runtime_management.py index ba0b1f3..eb98c45 100644 --- a/app/runtime_management.py +++ b/app/runtime_management.py @@ -59,16 +59,19 @@ def close(self): pass -def initialize(gateway, args, argv=None, *, parser=None): +def initialize(gateway, args, argv=None, *, parser=None, dotenv_keys=()): config = gateway.CONFIG config["auto_accept_buddy"] = buddy.auto_accept_from_env(os.environ) config["auto_accept_buddy_source"] = "environment" if "CODEBUDDY2API_AUTO_ACCEPT_BUDDY" in os.environ else "default" root = gateway.managed_auth_dir() control = ControlStore(root / "control.sqlite3") config["control_store"] = control - # Persist management sessions so a restart does not force another API-key login. - # The file stores an HMAC fingerprint of the key epoch, so key rotation still revokes them. - config["session_path"] = root / "admin-sessions.json" + config["state_store"] = control.state + try: + control.state.migrate(root) + except BaseException: + control.close() + raise config.update(vars(args)) config["model_guard"] = not args.no_model_guard aliases = {"log": "log_path", "no_model_guard": "model_guard"} @@ -82,10 +85,16 @@ def initialize(gateway, args, argv=None, *, parser=None): key = options[matches[0]].dest if len(matches) == 1 else flag[2:].replace("-", "_") explicit.add(aliases.get(key, key)) apply_persisted_settings(config, explicit=explicit) + from .startup import resolve_startup_key + try: + resolve_startup_key(config, args, dotenv_keys) + except BaseException: + control.close() + raise for key in SCHEMA: if hasattr(args, key) and key != "api_key": setattr(args, key, config[key]) - config["trial_ledger"] = gateway.trial_rewards.TrialLedger(root / "trial-ledger.json") + config["trial_ledger"] = gateway.trial_rewards.TrialLedger(store=control.state) try: config["audit_store"] = AuditStore(root / "logs.sqlite3", max_bytes=config["audit_max_bytes"], retention_days=config["audit_retention_days"], diff --git a/app/safe_logging.py b/app/safe_logging.py index b58e20e..19cfa37 100644 --- a/app/safe_logging.py +++ b/app/safe_logging.py @@ -47,7 +47,7 @@ # hex strings/UUIDs: those often carry useful request and trace identifiers. _JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]*){0,2}") _API_TOKEN = re.compile( - r"(? 256 * 1024: + raise ValueError(".env 必须是普通文件且不超过 256 KiB") + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + with os.fdopen(fd, "rb") as stream: + opened = os.fstat(stream.fileno()) + if (not stat.S_ISREG(opened.st_mode) + or (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino)): + raise ValueError(".env 文件在读取期间发生变化") + content = stream.read(256 * 1024 + 1) + if len(content) > 256 * 1024: + raise ValueError(".env 文件过大") + text = content.decode("utf-8-sig") + for binding in parse_stream(io.StringIO(text)): + if binding.error: + raise ValueError(f".env 第 {binding.original.line} 行语法错误") + previous = set(os.environ) + load_dotenv(stream=io.StringIO(text), override=False) + return set(os.environ) - previous + + +def terminal_stream(): + """Open the controlling terminal, never a redirected standard stream or regular file.""" + device = "CONOUT$" if os.name == "nt" else "/dev/tty" + fd = os.open(device, os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NOCTTY", 0)) + try: + if not stat.S_ISCHR(os.fstat(fd).st_mode) or not os.isatty(fd): + raise OSError("No interactive console") + return os.fdopen(fd, "w", encoding="utf-8", buffering=1) + except BaseException: + os.close(fd) + raise + + +def allows_open_noauth(): + """Honor the existing explicit unsafe opt-in without overriding a configured key.""" + return os.environ.get("CODEBUDDY2API_ALLOW_OPEN_NOAUTH", "").lower() in ("1", "true", "yes") + + +def resolve_startup_key(config, args, dotenv_keys=()): + sources = config["settings_sources"] + for name, spec in SCHEMA.items(): + if sources.get(name) == "environment" and spec["env"] in dotenv_keys: + sources[name] = "dotenv" + config["announce_default_key"] = False + if sources.get("api_key") in ("cli", "environment", "dotenv"): + return + store = config["state_store"] + key, pending = store.default_key() + if key is None and allows_open_noauth(): + return + if key is None and config["host"] not in ("127.0.0.1", "::1", "localhost"): + raise ValueError("非回环监听请显式设置 API key；默认密钥仅在本地首次启动时生成") + if key is None or pending: + try: + with terminal_stream(): + pass + except OSError: + raise ValueError("首次显示默认 API key 需要交互终端；后台运行请显式配置 CODEBUDDY2API_KEY") from None + if key is None: + key, pending = store.default_key(create=True) + config["api_key"] = args.api_key = key + sources["api_key"] = "generated" + config["announce_default_key"] = pending + + +def announce_default_key(config): + if not config.get("announce_default_key"): + return + with terminal_stream() as terminal: + key = config["api_key"] + with config["state_store"].claim_announcement(key) as claimed: + if claimed: + terminal.write(f"\n默认 API key：{key}\n已保存到 control.sqlite3，仅显示这一次；管理登录与 API 请求共用。\n") + terminal.flush() + config["announce_default_key"] = False + + +def run_server(app, config, *, host, port): + if not config.get("announce_default_key"): + return uvicorn.run(app, host=host, port=port, log_level="warning") + + class FirstStartServer(uvicorn.Server): + async def startup(self, sockets=None): + await super().startup(sockets=sockets) + if self.started and not self.should_exit: + try: + announce_default_key(config) + except BaseException: + # A failed announcement/commit must not leave a hidden-key server running. + self.should_exit = True + await self.shutdown(sockets=sockets) + raise + + server = FirstStartServer(uvicorn.Config(app, host=host, port=port, log_level="warning")) + try: + server.run() + except KeyboardInterrupt: + pass + if not server.started: + raise SystemExit(3) diff --git a/app/state_store.py b/app/state_store.py new file mode 100644 index 0000000..ff35031 --- /dev/null +++ b/app/state_store.py @@ -0,0 +1,278 @@ +"""Private SQLite state and one-time imports of legacy runtime snapshots.""" +from __future__ import annotations + +from contextlib import contextmanager +import json +import math +import os +from pathlib import Path +import re +import secrets +import sqlite3 +import stat +import time +import warnings + +MAX_BYTES = 16 * 1024 * 1024 +LEGACY_FILES = { + "sessions": "admin-sessions.json", + "cooldowns": "credential-cooldowns.json", + "model_blocks": "model-site-blocks.json", + "credits": "credits-ledger.json", + "trial": "trial-ledger.json", + "catalog": "model-catalog.json", + "usage": "usage-snapshots.json", +} + + +def strict_json(raw): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError("duplicate state field") + result[key] = value + return result + + def invalid(value): + raise ValueError("non-finite state value") + + return json.loads(raw, object_pairs_hook=pairs, parse_constant=invalid) + +def load_document(path, store, name, limit=MAX_BYTES): + """Read SQLite at runtime; retain bounded legacy readers for import validation.""" + if store is not None: + return store.get(name) + try: + metadata = os.lstat(path) + if not stat.S_ISREG(metadata.st_mode): + return None + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + with os.fdopen(fd, "rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + return None + raw = stream.read(limit + 1) + return strict_json(raw) if len(raw) <= limit else None + except (OSError, ValueError, RecursionError): + return None + + + +def _check_document(name, value): + if name not in LEGACY_FILES or not isinstance(value, dict): + raise ValueError("Invalid runtime state") + # Never promote an accidentally supplied credential file into SQLite. + pending = [value] + while pending: + item = pending.pop() + if isinstance(item, dict): + if any(str(key).lower().replace("_", "") in {"accesstoken", "refreshtoken", "idtoken"} for key in item): + raise ValueError("Credential material is not runtime state") + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) + field, versions = { + "sessions": ("sessions", (1,)), "cooldowns": ("accounts", (1,)), + "model_blocks": ("blocks", (1,)), "credits": ("creds", (1,)), + "trial": ("accounts", (1,)), "catalog": ("groups", (1, 2)), + "usage": ("accounts", (1,)), + }[name] + if type(value.get("version")) is not int or value["version"] not in versions or not isinstance(value.get(field), dict): + raise ValueError("Unsupported runtime state schema") + if name != "sessions" and set(value) != {"version", field}: + raise ValueError("Unexpected runtime state fields") + if name == "sessions": + from .admin_auth import _TOKEN, _FINGERPRINT, MAX_PERSISTED_SESSIONS + if (set(value) != {"version", "fingerprint", "sessions"} + or not isinstance(value.get("fingerprint"), str) or not _FINGERPRINT.fullmatch(value["fingerprint"]) + or len(value[field]) > MAX_PERSISTED_SESSIONS): + raise ValueError("Invalid session snapshot") + for sid, row in value[field].items(): + if (not isinstance(sid, str) or not _TOKEN.fullmatch(sid) or not isinstance(row, dict) + or set(row) != {"csrf_token", "expires"} + or not isinstance(row.get("csrf_token"), str) or not _TOKEN.fullmatch(row["csrf_token"]) + or type(row.get("expires")) not in (int, float) + or not math.isfinite(row["expires"]) or not 0 < row["expires"] < 1e11): + raise ValueError("Invalid persisted session") + elif name == "trial": + from .trial_rewards import _key, _timestamp, _safe_result, _result, _RECORD_FIELDS, _RESULT_FIELDS, _MAX_ACCOUNTS + if len(value[field]) > _MAX_ACCOUNTS: + raise ValueError("Too many trial accounts") + for key, row in value[field].items(): + _key(key) + if not isinstance(row, dict) or set(row) != _RECORD_FIELDS: + raise ValueError("Invalid trial history") + _timestamp(row["attempted_at"]) + if row["finished_at"] is not None: + _timestamp(row["finished_at"]) + if row["finished_at"] < row["attempted_at"]: + raise ValueError("Invalid trial deadline") + result = {key: row[key] for key in _RESULT_FIELDS} + if (type(result["ok"]) is not bool or type(result["already"]) is not bool + or (result["code"] is not None and type(result["code"]) is not int) + or (result["status"] is not None and type(result["status"]) is not int) + or result != _safe_result(result) + or (row["finished_at"] is None and result != _result())): + raise ValueError("Invalid trial result") + elif name == "usage": + from .usage_snapshots import UsageSnapshots, _FIELDS, MAX_ACCOUNTS + if len(value[field]) > MAX_ACCOUNTS: + raise ValueError("Too many usage snapshots") + for path, row in value[field].items(): + if (not isinstance(path, str) or not path or len(path) > 4096 or not isinstance(row, dict) + or set(row) != _FIELDS or UsageSnapshots._valid_row(row) is None): + raise ValueError("Invalid usage snapshot") + elif name == "credits": + for row in value[field].values(): + if (not isinstance(row, dict) or set(row) - {"identity", "checkin", "credits", "error", "travel"} + or not isinstance(row.get("checkin", {}), dict) or not isinstance(row.get("credits", {}), dict)): + raise ValueError("Invalid credit history") + elif name == "cooldowns": + from .credential_cooldowns import _FIELDS, _valid_identity, _valid_token, _number, MAX_ACCOUNTS, MAX_MODELS + if len(value[field]) > MAX_ACCOUNTS: + raise ValueError("Too many cooldown accounts") + for identity, row in value[field].items(): + if (not _valid_identity(identity) or not isinstance(row, dict) or set(row) != _FIELDS + or not _valid_token(row["profile"]) or not isinstance(row["models"], dict) + or len(row["models"]) > MAX_MODELS + or _number(row["fail_until"]) is None or _number(row["failed_at"]) is None + or any(not _valid_token(model) or _number(until) is None for model, until in row["models"].items())): + raise ValueError("Invalid cooldown snapshot") + elif name == "model_blocks": + for models in value[field].values(): + if not isinstance(models, dict): + raise ValueError("Invalid model blocks") + for row in models.values(): + if (not isinstance(row, dict) or type(row.get("until")) not in (int, float) + or not math.isfinite(row["until"]) or type(row.get("hits", 1)) is not int): + raise ValueError("Invalid model backoff") + elif name == "catalog": + for row in value[field].values(): + if (not isinstance(row, dict) or not isinstance(row.get("models"), list) + or not all(isinstance(model, dict) for model in row["models"]) + or type(row.get("fetched_at")) not in (int, float) + or not math.isfinite(row["fetched_at"])): + raise ValueError("Invalid model catalog") + + +class StateStore: + """Share the control connection and lock without exposing secrets in public snapshots.""" + + def __init__(self, control): + self.control = control + self.path = control.path + self._db = control._db + self._lock = control._lock + + @contextmanager + def transaction(self): + with self._lock: + self._db.execute("BEGIN IMMEDIATE") + try: + yield + self._db.execute("COMMIT") + except BaseException: + if self._db.in_transaction: + self._db.execute("ROLLBACK") + raise + + def _get(self, name): + if name not in LEGACY_FILES: + raise ValueError("Unknown state namespace") + row = self._db.execute("SELECT payload FROM runtime_state WHERE name=?", (name,)).fetchone() + if row is None: + return None + value = strict_json(row[0]) + _check_document(name, value) + return value + + def get(self, name): + with self._lock: + return self._get(name) + + def _put(self, name, value): + content = json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + _check_document(name, value) + if len(content.encode()) > MAX_BYTES: + raise ValueError("Runtime state exceeds size limit") + self._db.execute("INSERT INTO runtime_state(name,payload) VALUES(?,?) " + "ON CONFLICT(name) DO UPDATE SET payload=excluded.payload", (name, content)) + + def put(self, name, value): + with self.transaction(): + self._put(name, value) + + def put_cache(self, name, value): + """Cache failures stay visible without replacing the upstream inference outcome.""" + if name not in ("catalog", "model_blocks"): + raise ValueError("Not a rebuildable cache") + try: + self.put(name, value) + return True + except (OSError, ValueError, sqlite3.Error) as error: + warnings.warn(f"SQLite {name} cache write failed ({type(error).__name__}); keeping memory state", + RuntimeWarning, stacklevel=2) + return False + + + def delete(self, name): + if name not in LEGACY_FILES: + raise ValueError("Unknown state namespace") + with self.transaction(): + self._db.execute("DELETE FROM runtime_state WHERE name=?", (name,)) + + + def migrate(self, root): + """Import each legacy source once, including absence, before any maintenance starts.""" + root = Path(root) + with self.transaction(): + for name, filename in LEGACY_FILES.items(): + if self._db.execute("SELECT 1 FROM state_imports WHERE name=?", (name,)).fetchone(): + continue + path = root / filename + try: + metadata = path.lstat() + except FileNotFoundError: + raw = None + else: + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1 or metadata.st_size > MAX_BYTES: + raise ValueError(f"Cannot migrate {filename}: invalid file") + fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + with os.fdopen(fd, "rb") as stream: + opened = os.fstat(stream.fileno()) + if (not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino)): + raise ValueError(f"Cannot migrate {filename}: changed file") + raw = stream.read(MAX_BYTES + 1) + if len(raw) > MAX_BYTES: + raise ValueError(f"Cannot migrate {filename}: size limit") + if raw is not None: + try: + value = strict_json(raw) + _check_document(name, value) + if self._get(name) is None: + self._put(name, value) + except (ValueError, TypeError, KeyError, RecursionError): + raise ValueError(f"Cannot migrate {filename}: invalid state; restore a valid backup") from None + self._db.execute("INSERT INTO state_imports(name,imported,migrated_at) VALUES(?,?,?)", + (name, int(raw is not None), time.time())) + + def default_key(self, *, create=False): + with self.transaction(): + row = self._db.execute("SELECT value,announced FROM gateway_secrets WHERE name='default_api_key'").fetchone() + if row is not None: + if not re.fullmatch(r"cb-[0-9a-f]{32}", row[0]) or row[1] not in (0, 1): + raise ValueError("Invalid saved API key; restore the control database") + return row[0], not bool(row[1]) + if not create: + return None, False + key = "cb-" + secrets.token_hex(16) + self._db.execute("INSERT INTO gateway_secrets(name,value,announced) VALUES('default_api_key',?,0)", (key,)) + return key, True + + @contextmanager + def claim_announcement(self, key): + """Serialize disclosure and roll back the marker if terminal output or commit fails.""" + with self.transaction(): + claimed = self._db.execute("UPDATE gateway_secrets SET announced=1 WHERE name='default_api_key' " + "AND value=? AND announced=0", (key,)).rowcount == 1 + yield claimed diff --git a/app/trial_rewards.py b/app/trial_rewards.py index ea5f76d..356909c 100644 --- a/app/trial_rewards.py +++ b/app/trial_rewards.py @@ -150,8 +150,9 @@ def __init__(self, result): class TrialLedger: """Persist a bounded JSON ledger under one cross-process read-modify-write lock.""" - def __init__(self, path): - path = Path(path) + def __init__(self, path=None, *, store=None): + self._store = store + path = Path(store.path if store is not None else path) if not path.name or path.name in (".", ".."): raise ValueError("Trial ledger requires a file path") # Resolve the parent without following a symlink at the final filename. @@ -160,9 +161,13 @@ def __init__(self, path): self._lock_name = "trial-" + hashlib.sha256(self.path.name.encode()).hexdigest() + ".info" def _lock(self): + if self._store is not None: + return self._store.transaction() return credential_file_lock(self.path.parent, self._lock_name) def _load(self): + if self._store is not None: + return (self._store.get("trial") or {"accounts": {}})["accounts"] flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) try: fd = os.open(self.path, flags) @@ -208,6 +213,9 @@ def _save(self, accounts): separators=(",", ":"), allow_nan=False).encode() if len(content) > _MAX_BYTES: raise ValueError("Trial ledger exceeds size limit") + if self._store is not None: + self._store._put("trial", {"version": 1, "accounts": accounts}) + return fd, temporary = tempfile.mkstemp(prefix=".trial-", suffix=".tmp", dir=self.path.parent) try: with os.fdopen(fd, "wb") as stream: diff --git a/app/usage_snapshots.py b/app/usage_snapshots.py index b4f840d..864ec53 100644 --- a/app/usage_snapshots.py +++ b/app/usage_snapshots.py @@ -17,7 +17,7 @@ import json import os import re -import stat +import sqlite3 import tempfile import threading import time @@ -89,8 +89,9 @@ def no_constants(name): class UsageSnapshots: """Thread-safe, optionally persisted cache of per-account usage snapshots.""" - def __init__(self, path=None): - self.path = str(path) if path else None + def __init__(self, path=None, *, store=None): + self._store = store + self.path = str(store.path) if store is not None else (str(path) if path else None) self._lock = threading.RLock() self._data: dict[str, dict] = {} self.last_error: str | None = None @@ -103,26 +104,8 @@ def __init__(self, path=None): def _load(self): """Adopt only a fully valid snapshot; anything else leaves the cache empty.""" - try: - if not stat.S_ISREG(os.lstat(self.path).st_mode): - return - fd = os.open(self.path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - | getattr(os, "O_NONBLOCK", 0)) - except OSError: - return - try: - with os.fdopen(fd, "rb") as stream: - if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): - return - raw = stream.read(MAX_BYTES + 1) - except OSError: - return - if len(raw) > MAX_BYTES: - return - try: - document = _strict_json(raw) - except (ValueError, UnicodeDecodeError, RecursionError): - return + from .state_store import load_document + document = load_document(self.path, self._store, "usage", MAX_BYTES) if (not isinstance(document, dict) or set(document) != {"version", "accounts"} or type(document["version"]) is not int or document["version"] != VERSION): return @@ -212,6 +195,16 @@ def _save_locked(self) -> bool: self.last_error = "serialize" self._dirty = True return False + if self._store is not None: + try: + self._store.put("usage", json.loads(content)) + except (OSError, ValueError, sqlite3.Error) as error: + self.last_error = type(error).__name__ + self._dirty = True + return False + self.last_error = "capacity" if shed else None + self._dirty = shed + return not shed temporary = None try: directory = os.path.dirname(self.path) or "." diff --git a/converter.py b/converter.py index f485007..fab676b 100644 --- a/converter.py +++ b/converter.py @@ -27,7 +27,7 @@ from fastapi.exception_handlers import http_exception_handler as _default_http_exception_handler from fastapi.responses import JSONResponse, Response, StreamingResponse from starlette.concurrency import run_in_threadpool -import uvicorn +import uvicorn as uvicorn # Keep the existing embedding/test hook. try: from app.desensitize import desensitize_body @@ -72,6 +72,7 @@ def desensitize_body(body, roles=("system",), desensitize_harness_user=False, from app.content_filter import ContentFilterDetector, is_filter_error from app.request_limits import ImageLimitError, apply_image_policy from app.safe_logging import format_log_body, sanitize_log_text +from app.startup import allows_open_noauth, load_startup_env, run_server from app.site_routing import (DOMESTIC, INTERNATIONAL, PROFILE_ENDPOINTS, site_for_auth, site_for_headers, profile_for_auth, profile_for_headers, profile_region, profile_product, profile_site, chat_url_for_headers, refresh_url_for_auth) @@ -466,15 +467,15 @@ class CredentialPool: """Manage credential discovery, reloads, sticky sessions, cooldowns and refresh.""" def __init__(self, paths: list[Path] | None = None, scan: bool = False, - blocks_path: Path | None = None, cooldowns_path: Path | None = None): + blocks_path: Path | None = None, cooldowns_path: Path | None = None, *, state_store=None): self._lock = threading.RLock() self._entries: list[dict] = [] # {id, cm, fail_until} self._sticky: "OrderedDict[str, tuple[str, float]]" = OrderedDict() self._model_fail: dict[tuple[str, str], float] = {} # Per-credential/model 429 expiry # Keep unsupported-model backoff isolated by backend and model. - self._blocks = ModelBlocks(blocks_path, ttl_s=MODEL_SITE_BLOCK_S, max_ttl_s=MODEL_SITE_BLOCK_MAX_S) + self._blocks = ModelBlocks(blocks_path, ttl_s=MODEL_SITE_BLOCK_S, max_ttl_s=MODEL_SITE_BLOCK_MAX_S, store=state_store) # Cooldowns outlive a restart so a backend that just refused is not retried immediately. - self._cooldowns = CredentialCooldowns(cooldowns_path) + self._cooldowns = CredentialCooldowns(cooldowns_path, store=state_store) self._storage_warned = 0.0 # Rate limit for persistence-failure warnings self._rr = {None: 0, "cn": 0, "intl": 0} self._ledger = None # Prefer credits expiring sooner. @@ -1795,48 +1796,19 @@ async def _protocol_http_exception(request: Request, exc: HTTPException): # --------------------------------------------------------------------------- -# File logging +# Runtime audit events # --------------------------------------------------------------------------- -_LOG_LOCK = threading.Lock() -LOG_MAX_BYTES = 50 * 1024 * 1024 # Rotation threshold; a single entry may exceed it. -LOG_BACKUPS = 2 # Retain the two most recent rotated logs. def _log(msg: str): - """Write bounded redacted logs with rotation under a shared lock.""" + """Persist allowlisted runtime events in SQLite, never free-form text or secrets.""" audit = CONFIG.get("audit_store") component = re.match(r"\[(cred|credits|models|usage|trial|checkin|housekeeper)\]", msg) if audit is not None and component: # Persist event codes, not free-form lines which may contain upstream data. code = "cooldown" if "熔断" in msg or "冷却" in msg else "failure" if "失败" in msg or "异常" in msg else "updated" audit.event("runtime", component.group(1), {"code": code}) - path = CONFIG.get("log_path") - if not path: - return - budget = min(max(1024, CONFIG.get("log_body_limit", 65536) + 256), max(0, LOG_MAX_BYTES - 256)) - msg = sanitize_log_text(msg, budget) - line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}\n" - try: - with _LOG_LOCK: - try: - size = os.path.getsize(path) - except FileNotFoundError: - size = 0 - rotated = size > 0 and size + len(line.encode("utf-8")) > LOG_MAX_BYTES - if rotated: - for i in range(LOG_BACKUPS - 1, 0, -1): - old = f"{path}.{i}" - if os.path.exists(old): - os.replace(old, f"{path}.{i + 1}") - os.replace(path, f"{path}.1") - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) - with os.fdopen(fd, "a", encoding="utf-8", newline="\n") as stream: - if rotated: - stream.write(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] ==== 日志轮转 ====\n") - stream.write(line) - except OSError: - pass # Logging failures must not interrupt requests. def _log_json(label: str, value): @@ -3777,6 +3749,7 @@ def _boolean_arg(value): def main(): + dotenv_keys = set() if any(arg in ("-h", "--help") for arg in sys.argv[1:]) else load_startup_env() ap = argparse.ArgumentParser(description="CodeBuddy -> OpenAI 兼容转换器（直连后端）") ap.add_argument("command", nargs="?", choices=("serve", "login"), default="serve", help="serve 启动服务（默认）；login 扫码登录、自动轮询并保存账号") @@ -3787,7 +3760,7 @@ def main(): ap.add_argument("--host", default="127.0.0.1", help="监听地址；覆盖 CODEBUDDY2API_BIND") ap.add_argument("--port", type=int, default=8787, help="监听端口；覆盖 CODEBUDDY2API_PORT") ap.add_argument("--api-key", default=os.environ.get("CODEBUDDY2API_KEY", ""), - help="可选：要求客户端携带的 API key（默认不校验）") + help="管理与推理密钥；未配置时首次本地交互启动生成并保存") ap.add_argument("--admin-csrf", type=_boolean_arg, nargs="?", const=True, default=os.environ.get("CODEBUDDY2API_ADMIN_CSRF", "true"), help="管理 Origin/CSRF 校验，默认 true；仅在受信任本地环境设为 false，鉴权仍启用") @@ -3796,8 +3769,7 @@ def main(): help="额外信任的管理页来源（逗号分隔，支持域名或完整来源，裸域名按 https）；" "反代 HTTPS 域名登录报 Origin 校验失败时设置，也可在 WebUI 配置") ap.add_argument("--log", default=None, metavar="PATH", - help="额外写入兼容文本日志（如 --log converter.log 或 --log /tmp/cb.log）。" - "不传仍记录 SQLite 审计，但不输出文本文件。") + help="已停用：日志统一保存到数据目录中的 logs.sqlite3") ap.add_argument("--desensitize", action="store_true", help="适配固定 CLI 模板、压缩运行时提示并零宽脱敏关键词。默认关闭。") ap.add_argument("--no-compact", action="store_true", @@ -3851,7 +3823,7 @@ def main(): help="按账号模型声明预检图片、工具、思考和输出上限；false 仅关闭新增能力预检") ap.add_argument("--log-body-limit", type=_nonnegative_int, metavar="BYTES", default=os.environ.get("CODEBUDDY2API_LOG_BODY_LIMIT", "65536"), - help="每条正文日志的预览字节上限，默认 64 KiB；0 只记录摘要") + help="旧文本预览兼容项；文本输出已停用，SQLite 诊断使用独立预算") ap.add_argument("--tool-call-max-retry", type=_nonnegative_int, metavar="N", default=os.environ.get("CODEBUDDY2API_TOOL_CALL_MAX_RETRY", "3"), help="工具参数损坏时的额外生成上限，默认 3；0 表示不重试（每次额外生成都消耗额度）") @@ -3888,30 +3860,28 @@ def main(): CONFIG["usd_rate"] = args.usd_rate or None CONFIG["credit_price_usd"] = args.credit_price_usd or None CONFIG["model_guard"] = not args.no_model_guard - # File logging is enabled only when a path is configured. - CONFIG["log_path"] = args.log if args.log else os.environ.get("CODEBUDDY2API_LOG") + if args.log is not None or os.environ.get("CODEBUDDY2API_LOG"): + sys.stderr.write("[log] --log / CODEBUDDY2API_LOG 已停用；请在 WebUI 查看 SQLite 日志。\n") + CONFIG["log_path"] = None from app import runtime_management - runtime_management.initialize(sys.modules[__name__], args, parser=ap) + runtime_management.initialize(sys.modules[__name__], args, parser=ap, dotenv_keys=dotenv_keys) # Validate effective binding and authentication before credential scans or background work. if (args.host not in ("127.0.0.1", "::1", "localhost") and not CONFIG.get("api_key") - and os.environ.get("CODEBUDDY2API_ALLOW_OPEN_NOAUTH", "").lower() not in ("1", "true", "yes")): + and not allows_open_noauth()): runtime_management.close(CONFIG) ap.error("非回环绑定且未设置 API key 会匿名开放推理额度；" "请设置 CODEBUDDY2API_KEY，或确知风险后以 CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true 显式放行") files = [Path(p) for p in args.auth_file] if not files: seed_credentials() # Seed missing desktop credentials into managed storage. - CONFIG["cred_pool"] = CredentialPool(files, scan=not files, - blocks_path=managed_auth_dir() / "model-site-blocks.json", - cooldowns_path=managed_auth_dir() / "credential-cooldowns.json") - CONFIG["usage_snapshots"] = UsageSnapshots(managed_auth_dir() / "usage-snapshots.json") + CONFIG["cred_pool"] = CredentialPool(files, scan=not files, state_store=CONFIG["state_store"]) + CONFIG["usage_snapshots"] = UsageSnapshots(store=CONFIG["state_store"]) CONFIG["cred"] = CONFIG["cred_pool"].first() CONFIG["account_catalogs"] = {} # Disable static fallback before maintenance starts. if credits_mod is not None: - ledger = credits_mod.CreditLedger(managed_auth_dir() / "credits-ledger.json") + ledger = credits_mod.CreditLedger(store=CONFIG["state_store"]) CONFIG["ledger"] = ledger - CONFIG["model_cache"] = credits_mod.ModelCatalogCache( - managed_auth_dir() / "model-catalog.json", ttl=args.model_catalog_ttl) + CONFIG["model_cache"] = credits_mod.ModelCatalogCache(ttl=args.model_catalog_ttl, store=CONFIG["state_store"]) CONFIG["cred_pool"].set_ledger(ledger) # Verify balance ownership before publishing catalogs. _publish_model_cache() # Publish cached usage before maintenance threads start, so the dashboard is populated from @@ -3968,7 +3938,7 @@ def main(): _log(f"==== converter 启动 ====") try: - uvicorn.run(app, host=args.host, port=args.port, log_level="warning") + run_server(app, CONFIG, host=args.host, port=args.port) finally: runtime_management.close(CONFIG) diff --git a/docs/advanced.md b/docs/advanced.md index f1655e0..604ac66 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -6,18 +6,18 @@ Use the [WebUI](webui.md) for everyday management. See [deployment](deployment.m ## Configuration and CLI -Precedence: **explicit CLI flags > environment > persisted WebUI settings > defaults**. Hot settings apply immediately; restart-marked settings require a manual restart. Change locked options in the startup configuration; the WebUI does not edit `.env`. +Precedence: **explicit CLI flags > process environment > `.env` > saved SQLite settings > defaults**. Hot settings apply immediately; restart-marked settings require a manual restart. Change locked options at their source; the WebUI does not edit `.env` or expose API keys. Compose explicitly passes some environment variables and CLI flags, so deleting a line from `.env` may not unlock it. Recreate the container after changing these values; to let the WebUI manage them, also remove the corresponding explicit Compose settings. | Flag | Default | Description | |------|---------|-------------| | `--host` / `--port` | `127.0.0.1` / `8787` | Local listener | -| `--api-key` | none | Shared management and inference key; management is locked without it | +| `--api-key` | saved default | Shared management/inference key; first local interactive startup generates and saves one if absent; explicit empty locks management | | `--admin-csrf [true/false]` | `true` | Startup-only management Origin/CSRF checks; disabling does not bypass API-key or session authentication | | `--admin-allowed-origins` | none | Extra trusted management Origins (comma-separated; bare domains mean HTTPS) for reverse-proxy sign-in; hot and WebUI-editable | | `--auth-file` | scan `auth/` | Explicit credential file, repeatable; disables scanning other files | -| `--log` | none | Additional text logs, 50 MiB rotation and 2 backups; SQLite auditing remains enabled | +| `--log` | retired | Warns without writing a file; use SQLite logs in the WebUI | | `--desensitize` | off | Adapt fixed CLI templates, compact runtime prompts and mask keywords with zero-width characters | | `--no-compact` | off | With desensitization, retain fuller instructions while adapting templates and pruning runtime context; does not disable Responses projection | | `--keep-tool-metadata [true/false]` | `false` | Retain tool descriptions and parameter-schema `description/title`, independently of prompt compaction | @@ -40,7 +40,7 @@ Compose explicitly passes some environment variables and CLI flags, so deleting | `--failover-max` | `0` | Extra credentials tried when a request fails before the first response byte reaches the client; `0` keeps the upstream behaviour of surfacing the failure directly | | `--retry-write-timeout` | `false` | Opt a request-body write timeout into replay (fresh connection and `--failover-max`), accepting that bytes already sent may have been processed | | `--max-request-bytes` | `33554432` | Positive byte limit for the processed upstream JSON | -| `--log-body-limit` | `65536` | Text-log body preview bytes; `0` logs summaries only, not the SQLite diagnostic budget | +| `--log-body-limit` | `65536` | Legacy text-preview option; text output is retired and SQLite diagnostics use their own budget | Environment variables include `CODEBUDDY_AUTH_DIR`, `CODEBUDDY_IMPORT_DIR`, `CODEBUDDY2API_KEY`, `CODEBUDDY2API_ADMIN_CSRF`, `CODEBUDDY2API_ADMIN_ORIGINS`, `CODEBUDDY2API_KEEP_TOOL_METADATA`, `CODEBUDDY2API_LOG`, `CODEBUDDY2API_MAX_IMAGES`, `CODEBUDDY2API_IMAGE_POLICY`, `CODEBUDDY2API_MAX_REQUEST_BYTES`, `CODEBUDDY2API_LOG_BODY_LIMIT`, `CODEBUDDY2API_FAILOVER_MAX` and `CODEBUDDY2API_RETRY_WRITE_TIMEOUT`. See [deployment](deployment.md) for startup examples. @@ -82,7 +82,7 @@ Manual `POST /admin/credentials/{id}/travel` returns `buddy_confirmation` with o Travel claims and departures share an account-scoped write reservation. Uncertain results do not expire or replay; fresh status reads reconcile them without issuing upstream writes. Store failures stop claims and departures, and local readback updates preserve receipt ownership across processes. -Trial credits are manual-only for eligible `intl-work` accounts: use the credential row's claim drawer or `POST /admin/credentials/{id}/trial`. Startup, periodic maintenance and balance sync never claim. Results expose safe error categories, HTTP/business codes and retry time; response bodies are capped at 64 KiB and never returned to the browser. Success/already-claimed records persist in `auth/trial-ledger.json`; failures wait at least 24 hours before another manual attempt. Keep this file when upgrading. +Trial credits are manual-only for eligible `intl-work` accounts through the credential drawer or `POST /admin/credentials/{id}/trial`; startup and maintenance never claim. Safe results, successful claims and reservations persist in `control.sqlite3`; failed attempts retain the 24-hour backoff. Response bodies are capped at 64 KiB and never returned to the browser. Preserve the database when upgrading. `CODEBUDDY2API_AUTO_TRIAL` and `--auto-trial` are retired: old startup options warn and do nothing; saved Boolean `auto_trial` settings are ignored on load. Remove them from deployment configuration. Before reverting to older code, check these old settings to avoid re-enabling automatic claims. @@ -147,7 +147,7 @@ The WebUI supports direct uploads; these rules concern path imports through `POS ## Models and scheduling -Select client models from the WebUI or `GET /v1/models`. Raw catalogs remain cached by account/tenant, region, product and client version in `auth/model-catalog.json`, with a default 6-hour TTL. New credentials trigger synchronization; failed refreshes retain that account's trusted cache. Legacy unscoped caches do not become international sharing sources. +Select client models from the WebUI or `GET /v1/models`. Raw catalogs remain cached by account/tenant, region, product and client version in `auth/control.sqlite3`, with a default 6-hour TTL. New credentials trigger synchronization; failed refreshes retain that account's trusted cache. Legacy unscoped caches do not become international sharing sources. International CLI and WorkBuddy use a deduplicated shared view from enabled, catalog-ready international accounts. A target account must have its own synchronized catalog; its existing model declarations win unchanged. Missing IDs inherit shared declarations, retaining `catalog_source` and safe `source_variants`. Conflicting inherited rates use the higher known rate, limits the smaller known value, reasoning options their intersection and differing descriptive fields are omitted; unknown prices never mean free. Domestic catalogs, credentials, balances, bindings and `auto` defaults remain independent. Shared rates are catalog references, not billing or permission guarantees. @@ -191,11 +191,11 @@ Both international profiles merge image-bearing consecutive `user` runs only aft - Valid upstream `Retry-After` values (0–86400 seconds or equivalent HTTP dates) are returned as seconds before streaming starts; 429 only cools the selected account/model. Invalid or expired values fall back to the body's reset time or 600 seconds. Pool-generated 429 responses include the remaining wait. - Chat and Responses preserve an explicit client `prompt_cache_key` without generating one; cache hits and savings depend on the upstream. - Unsupported capabilities are rejected rather than silently degraded: chat `n` other than 1 and the Responses state fields `previous_response_id`/`conversation` (this gateway keeps no server-side response state) return 400; length-truncated or content-filtered Responses are reported as `incomplete`, never disguised as `completed`. -- Text logs and SQLite auditing have separate budgets. Logs contain bounded, redacted previews, not complete original requests. Treat logs, credential exports and backups as private data. +- SQLite auditing retains only bounded, redacted diagnostics, not complete original requests. Treat logs, credential exports and backups as private data. ## Deployment exposure and credential intake -- The compose port mapping binds loopback by default (`CODEBUDDY2API_BIND` defaults to 127.0.0.1); after resolving CLI, environment and saved settings, a native non-loopback bind with an empty effective API key refuses to start unless `CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true` is set explicitly. +- Compose maps loopback by default. Without a configured or saved key, `CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true` explicitly permits headless/non-loopback inference without generating a key; management stays locked. The shipped image sets this compatibility opt-in, so configure `CODEBUDDY2API_KEY` before exposing it. The opt-in never disables an existing key. - When a key is configured, generation and token-count POSTs verify request headers before buffering bodies or reserving inference capacity; invalid keys return 401 even while generation slots are full. Other routes retain their existing authentication and routing behavior. - Credential imports/uploads persist the normalized form (token aliases folded into the canonical fields); strict JSON parsing rejects NaN/Infinity, and `expiresAt`/`lastRefreshTime` must be plausible finite millisecond timestamps. diff --git a/docs/advanced.zh-CN.md b/docs/advanced.zh-CN.md index 4c9cc6b..2a09e06 100644 --- a/docs/advanced.zh-CN.md +++ b/docs/advanced.zh-CN.md @@ -6,18 +6,18 @@ ## 配置与 CLI -配置优先级：**显式 CLI 参数 > 环境变量 > WebUI 持久化配置 > 默认值**。WebUI 中可热更新的设置立即生效，标记为重启生效的设置需手动重启；锁定项须在启动配置中修改，WebUI 不改写 `.env`。 +配置优先级：**显式 CLI 参数 > 进程环境变量 > `.env` > SQLite 保存值 > 默认值**。热更新设置立即生效，标记为重启的项需手动重启；锁定项请在对应来源修改，WebUI 不改写 `.env` 或返回 API key 明文。 Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的一行不一定解除锁定。修改这些值后重建容器；若要由 WebUI 接管，还需取消 Compose 中对应的显式设置。 | 参数 | 默认值 | 说明 | |------|--------|------| | `--host` / `--port` | `127.0.0.1` / `8787` | 本地监听地址与端口 | -| `--api-key` | 无 | 管理与推理共用密钥；未设置时管理锁定 | +| `--api-key` | 已保存的默认 key | 管理与推理共用；缺失时首次本地交互启动生成并保存；显式空值锁定管理 | | `--admin-csrf [true/false]` | `true` | 管理 Origin/CSRF 校验；仅启动配置可关闭，API key 和会话鉴权不变 | | `--admin-allowed-origins` | 无 | 额外信任的管理页来源（逗号分隔，裸域名按 HTTPS），用于反代登录；热生效，可在 WebUI 配置 | | `--auth-file` | 扫描 `auth/` | 指定凭据文件，可重复传入；不再扫描其他文件 | -| `--log` | 无 | 额外文本日志，50 MiB 轮转、保留 2 份；不影响默认 SQLite 审计 | +| `--log` | 已停用 | 提示弃用且不写文件；日志统一在 WebUI 查看 SQLite 记录 | | `--desensitize` | 关 | 适配固定 CLI 模板、压缩运行时提示、零宽脱敏关键词 | | `--no-compact` | 关 | 配合脱敏保留主要行为指令，仍适配模板及裁剪运行时上下文；不关闭 Responses 投影 | | `--keep-tool-metadata [true/false]` | `false` | 保留工具描述及参数 schema 的 `description/title`，与提示词压缩独立 | @@ -40,7 +40,7 @@ Compose 会显式传入部分环境变量及 CLI 参数，删除 `.env` 中的 | `--failover-max` | `0` | 请求在「一个字节都还没发给下游」之前失败时，最多再换几个凭证就地重放；`0` 表示如实把失败回给下游 | | `--retry-write-timeout` | `false` | 让「写请求体超时」也参与重放（换新连接与 `--failover-max` 换凭证），代价是已发出的那半截正文可能已被上游处理 | | `--max-request-bytes` | `33554432` | 处理后的上游 JSON 字节上限，须为正整数 | -| `--log-body-limit` | `65536` | 兼容文本日志正文预览字节；`0` 只记摘要，不控制 SQLite 诊断预算 | +| `--log-body-limit` | `65536` | 旧文本预览兼容项；文本输出已停用，SQLite 诊断使用独立预算 | 环境变量包括 `CODEBUDDY_AUTH_DIR`、`CODEBUDDY_IMPORT_DIR`、`CODEBUDDY2API_KEY`、`CODEBUDDY2API_ADMIN_CSRF`、`CODEBUDDY2API_ADMIN_ORIGINS`、`CODEBUDDY2API_KEEP_TOOL_METADATA`、`CODEBUDDY2API_LOG`，以及 `CODEBUDDY2API_MAX_IMAGES`、`CODEBUDDY2API_IMAGE_POLICY`、`CODEBUDDY2API_MAX_REQUEST_BYTES`、`CODEBUDDY2API_LOG_BODY_LIMIT`、`CODEBUDDY2API_FAILOVER_MAX`、`CODEBUDDY2API_RETRY_WRITE_TIMEOUT`。启动示例见[部署指南](deployment.zh-CN.md)。 @@ -82,7 +82,7 @@ scoped 模式可选传入 `X-Codebuddy-Session-ID`、`metadata.conversation_id` 旅行奖励领取与派遣共用按账号隔离的写入预留，不确定结果不超时重放；状态查询只回查并更新本地确认记录，不发上游写请求。存储失败停止领取和派遣，跨进程确认不覆盖其他请求的记录。 -体验积分仅供符合官方资格的 `intl-work` 账号手动领取：使用凭证行的领取抽屉或 `POST /admin/credentials/{id}/trial`。启动、定时维护、余额同步均不领取。结果显示安全错误类别、HTTP 状态／业务码及重试时间，响应正文限制为 64 KiB 且不返回浏览器。成功或已领取记录保存在 `auth/trial-ledger.json`，失败至少等待 24 小时才能再次手动申请；升级时保留该文件。 +体验积分仅供符合资格的 `intl-work` 账号通过凭证抽屉或 `POST /admin/credentials/{id}/trial` 手动领取，启动和维护不领取。安全结果、成功历史与请求预留保存在 `control.sqlite3`，失败保留 24 小时退避。响应正文上限 64 KiB，且不返回浏览器；升级时保留数据库。 `CODEBUDDY2API_AUTO_TRIAL` 和 `--auto-trial` 已停用：旧启动选项仅提示、不触发任务；控制库中的旧布尔 `auto_trial` 设置在加载时忽略。请从部署配置中移除；回滚旧代码前也须核对这些旧设置，避免重新启用自动领取。 @@ -147,7 +147,7 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` ## 模型与调度 -以 WebUI 和 `GET /v1/models` 为客户端选择依据。原始目录仍按账号/租户、地域、产品与客户端版本缓存到 `auth/model-catalog.json`，默认有效期 6 小时；新凭据触发同步，刷新失败保留该账号的可信旧缓存。旧未隔离缓存不作为国际共享来源。 +以 WebUI 和 `GET /v1/models` 为客户端选择依据。原始目录按账号/租户、地域、产品与客户端版本缓存到 `auth/control.sqlite3`，默认有效期 6 小时；新凭据触发同步，刷新失败保留该账号的可信旧缓存。旧未隔离缓存不作为国际共享来源。 国际 CLI／WorkBuddy 使用已启用、目录已就绪的国际账号生成去重共享视图。目标账号须完成自身目录同步；已有型号保留自己的完整声明，缺失型号才继承，并通过 `catalog_source`、安全 `source_variants` 标明来源。继承声明冲突时倍率取较高值、上限取较小值、思考选项取交集、描述类字段不一致即省略；未知倍率不当零。国内目录、凭据、余额、绑定及 `auto` 默认模型保持独立；共享倍率只是目录参考，不保证权限或实际扣分。 @@ -191,11 +191,11 @@ WebUI 可以直接上传文件；以下限制针对 `POST /admin/credentials` - 上游有效 `Retry-After`（0–86400 秒或对应 HTTP 日期）规范化为秒并在开流前返回；429 仅冷却对应账号/模型。无效或过期值回落正文重置时间或默认 600 秒；本地全凭据冷却的 429 返回剩余等待秒数。 - Chat 与 Responses 保留客户端显式 `prompt_cache_key`，不自动生成；缓存命中和节费取决于上游。 - 不支持的能力显式拒绝而非静默降级：Chat 的 `n≠1`、Responses 的 `previous_response_id`/`conversation`（本网关不保存服务端响应状态）返回 400；长度截断或审核过滤的 Responses 标记为 `incomplete`，不伪装为 `completed`。 -- 兼容文本日志和 SQLite 审计使用独立预算；日志仅记录有界、脱敏预览，不是完整原始请求。日志、凭证导出和备份仍须按私有数据保管。 +- SQLite 审计只保留有界、脱敏诊断，不是完整原始请求。日志、凭证导出和备份仍须按私有数据保管。 ## 部署暴露与凭据导入 -- Compose 端口映射默认只绑回环（`CODEBUDDY2API_BIND` 默认 127.0.0.1）；原生运行在合并 CLI、环境和持久化设置后，若实际地址非回环且生效 key 为空则拒绝启动，须显式设 `CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true` 放行。 +- Compose 默认映射回环地址。没有配置或已保存的 key 时，`CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true` 显式允许后台／非回环匿名推理，不生成密钥，管理仍锁定。镜像保留此兼容开关，对外暴露前请配置 `CODEBUDDY2API_KEY`；该开关不会停用已有 key。 - 配置 key 时，生成及 token 估算 POST 在缓冲请求体、预留推理名额之前校验请求头；即使名额已满，无效 key 仍返回 401。其他路由保留原有鉴权和路由行为。 - 凭据导入/上传在落盘前把 token 别名归一化为官方字段名；严格 JSON 解析拒绝 NaN/Infinity，`expiresAt`/`lastRefreshTime` 必须是合理的有限毫秒时间戳。 diff --git a/docs/deployment.md b/docs/deployment.md index 95b5ac6..40c4a32 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -63,14 +63,28 @@ Requires Python 3.12+, uv, and Node.js with the vp CLI to build the interface: ```bash uv sync --locked --no-build --python 3.12 (cd web && vp install --frozen-lockfile && vp build) -uv run --locked --no-build --env-file .env converter.py --desensitize +uv run converter.py ``` -Configure `.env` as above before starting, then open `/dashboard` to add accounts. Rebuild the WebUI after changing frontend source. +Local startup does not require `.env`. Without an explicit key, the first loopback startup generates a `cb-…` default key in `auth/control.sqlite3` and displays it once in the interactive terminal after listening succeeds. Restarts reuse it without printing it again. Keep it safe: management and API requests share this key. Configure a key explicitly for first-time headless or non-loopback deployment. -Without uv, run `python3 -m venv .venv`, activate it, install dependencies with `pip install --require-hashes --only-binary=:all: -r requirements.txt`, and start with `python3 converter.py --desensitize`. **Plain Python does not load `.env`**; export environment variables or pass CLI flags explicitly. +One-time disclosure goes directly to the controlling terminal (`/dev/tty` or Windows `CONOUT$`), not stdout/stderr; redirecting standard output does not capture the key. Terminal recording remains outside the gateway's control. -Native binding follows explicit `--host/--port` > `CODEBUDDY2API_BIND/PORT` > saved WebUI values > defaults. Remove explicit flags if `.env` should control the listener; changes require restart. `CODEBUDDY2API_IMAGE/AUTH_PATH` remain Compose-only; use `CODEBUDDY_AUTH_DIR` for native data. +A failed terminal write or database commit leaves the same key pending for the next interactive start. A crash between display and commit can therefore repeat the notice; a successfully committed display is not repeated. + +Without uv, run `python3 -m venv .venv`, activate it, install dependencies with `pip install --require-hashes --only-binary=:all: -r requirements.txt`, then run `python3 converter.py`. Distribution archives include the WebUI; source installs still need the frontend build. + +Both commands optionally read `.env` in the current working directory, never parent directories. Precedence: explicit CLI > process environment > `.env` > saved SQLite values > defaults. Overrides do not replace the saved default key; an explicitly empty key still locks management. Listener changes require restart; `CODEBUDDY2API_IMAGE/AUTH_PATH` are Compose-only. + +### Upgrade and state migration + +Stop the gateway and back up the entire data directory first. Startup imports legacy JSON sessions, cooldowns, credit/trial history, catalogs and usage into `control.sqlite3` once. Old files remain as backups and are no longer read or updated. Invalid critical state or a failed migration stops startup; repair it or restore a backup rather than deleting ledgers to bypass validation. + +SQLite state read/validation failures stop startup without deleting sessions or replacing state with empty defaults. Repair the control database before retrying; only rebuildable cache write failures may retain the current in-memory data with a warning. + +All runtime logs use `logs.sqlite3`; legacy `--log` / `CODEBUDDY2API_LOG` settings warn and no longer write text files. Generated keys enter private configuration, never logs. Restrict access to the data directory; use a private user directory on Windows. + +Older versions cannot read the upgraded control database. Downgrades require a stopped gateway and matching state migration, not merely old code reading stale JSON. Never overwrite new claims, dispatches or session revocations with an outdated backup. ## Dependency locks diff --git a/docs/deployment.zh-CN.md b/docs/deployment.zh-CN.md index 6c96056..569756a 100644 --- a/docs/deployment.zh-CN.md +++ b/docs/deployment.zh-CN.md @@ -63,14 +63,28 @@ docker compose up -d ```bash uv sync --locked --no-build --python 3.12 (cd web && vp install --frozen-lockfile && vp build) -uv run --locked --no-build --env-file .env converter.py --desensitize +uv run converter.py ``` -按上文配置 `.env` 后启动，打开 `/dashboard` 添加账号。修改前端源码后需重新构建 WebUI。 +本地启动无需 `.env`。没有显式 key 时，首次回环启动会生成 `cb-…` 默认密钥，保存到 `auth/control.sqlite3`，监听成功后仅在交互终端显示一次；以后重启复用且不再打印。请妥善保存，管理登录与 API 请求共用。首次后台或非回环部署请显式配置 key。 -没有 uv 时：运行 `python3 -m venv .venv` 并激活，用 `pip install --require-hashes --only-binary=:all: -r requirements.txt` 安装依赖，以 `python3 converter.py --desensitize` 启动。**纯 Python 不会加载 `.env`**；请显式导出环境变量或传递 CLI 参数。 +密码提示直接写入控制终端（`/dev/tty` 或 Windows `CONOUT$`），不经过 stdout/stderr，重定向标准输出不会收集密码；外部终端录制不在程序控制范围内。 -本地监听地址优先级为：显式 `--host/--port` > `CODEBUDDY2API_BIND/PORT` > WebUI 已保存值 > 默认值。希望由 `.env` 控制监听时去掉显式参数；修改需重启。`CODEBUDDY2API_IMAGE/AUTH_PATH` 仅用于 Compose；本地数据目录用 `CODEBUDDY_AUTH_DIR`。 +终端写入或数据库提交失败时，同一密钥保留为待显示，下次交互启动可重试。显示后、提交前崩溃可能再次提示；成功提交后不重复显示。 + +没有 uv 时：运行 `python3 -m venv .venv` 并激活，用 `pip install --require-hashes --only-binary=:all: -r requirements.txt` 安装依赖，再执行 `python3 converter.py`。发行包已包含 WebUI；源码安装仍需构建界面。 + +两种启动方式均可选读取当前工作目录的 `.env`，不搜索父目录。优先级：显式 CLI > 进程环境变量 > `.env` > SQLite 保存值 > 默认值。覆盖不改写已保存的默认 key；显式空 key 仍锁定管理。修改监听需重启，`CODEBUDDY2API_IMAGE/AUTH_PATH` 仅用于 Compose。 + +### 升级与数据迁移 + +升级前停止网关并备份整个数据目录。首次启动将旧 JSON 会话、冷却、积分/领取、目录和用量状态一次性导入 `control.sqlite3`；原文件保留作备份，不再参与读写。关键数据损坏或迁移失败会阻止启动，请修复或恢复备份，不要删除账本绕过检查。 + +SQLite 状态读取或校验失败会阻止启动，不删除会话、不回退为空状态；修复控制库后再启动。仅可重建缓存的写入失败允许保留当前内存数据并告警。 + +所有运行日志使用 `logs.sqlite3`；旧 `--log` / `CODEBUDDY2API_LOG` 仅提示弃用，不再输出文本文件。自动生成的 key 只进私有配置，不进任何日志。限制数据目录访问权限；Windows 应使用当前用户的私有目录。 + +旧版本不能读取升级后的控制库。回滚须停机并迁回匹配的状态，不能只切代码后复用旧 JSON；升级后发生的领取、派遣或会话撤销不能用过期备份覆盖。 ## 依赖锁定 diff --git a/docs/webui.md b/docs/webui.md index 27e4978..70af9e3 100644 --- a/docs/webui.md +++ b/docs/webui.md @@ -6,7 +6,7 @@ Start the gateway using the [deployment guide](deployment.md), then open `http://127.0.0.1:8787/dashboard` and sign in with the current API key. Source installs need a frontend build; Docker builds include it. Use HTTPS unless connecting locally. -Management is locked without a key. After changing it, sign in again and restart any unfinished OAuth login. +The first unconfigured local startup generates and saves a default key, shown once in the terminal and reused afterward. An explicitly empty key locks management. After changing the effective key, sign in again and restart unfinished OAuth logins. ## Overview @@ -74,15 +74,15 @@ Data defaults to `auth/`, or `/data/auth` inside Docker. Local installs can set | File | Contents | |------|----------| | `*.info` | Official plaintext credentials; never migrated into SQLite | -| `control.sqlite3` | Gateway settings, model rules, credential metadata and first-Buddy reservations | +| `control.sqlite3` | Settings, private default key, model rules, sessions, cooldowns, credit/reward state, usage and catalogs | | `logs.sqlite3` | Request details and independent aggregate statistics | Auditing defaults to 30-day detail retention and a 256 MiB logical detail budget, **not a hard limit on database or directory disk usage**. Detail cleanup and eviction preserve aggregates. SQLite failure diagnostics have a separate budget, defaulting to 8192 bytes. Existing text logs are retained, not backfilled as precise statistics. Mount the whole data directory on writable local storage, not just a single database file, and do not share it between gateway instances. -Stop the gateway before copying the entire directory, including databases, any WAL/SHM files, credentials and catalog/credit state files; do not back up only `.info` files. Keep this private data secure. +Stop the gateway before copying the entire directory, including databases, WAL/SHM files, credentials and migration backups; do not back up only `.info`. Protect the control database: it contains the default key and management sessions. -Back up control metadata before using new model rules or automation preferences. Reverting to older code requires the matching control-database snapshot, including its WAL/SHM state without mixing files; older readers reject the new fields. Rollback cannot undo completed upstream check-ins, claims or travel dispatches. +Legacy JSON is imported once; SQLite is authoritative afterward. Downgrades require a stopped gateway and matching state migration, never stale JSON overwriting new claims or revoked sessions. See [upgrade and state migration](deployment.md#upgrade-and-state-migration). See [client configuration](clients.md) for API keys and URLs. diff --git a/docs/webui.zh-CN.md b/docs/webui.zh-CN.md index 0a6f2db..4a5e471 100644 --- a/docs/webui.zh-CN.md +++ b/docs/webui.zh-CN.md @@ -6,7 +6,7 @@ 按[部署指南](deployment.zh-CN.md)启动网关后，打开 `http://127.0.0.1:8787/dashboard`，使用当前 API key 登录。源码安装需要先构建前端；Docker 镜像已包含。非本机访问请使用 HTTPS。 -未设置 key 时管理功能锁定。更换 key 后需重新登录，并重新开始未完成的 OAuth 登录。 +本地零配置首次启动会生成并保存默认 key，仅在终端显示一次；以后复用。显式空 key 锁定管理。更换生效 key 后需重新登录，并重新开始未完成的 OAuth 登录。 ## 概览 @@ -74,15 +74,15 @@ | 文件 | 内容 | |------|------| | `*.info` | 官方明文凭证；绝不迁移进 SQLite | -| `control.sqlite3` | 网关设置、模型规则、凭证元数据与首次领养预留 | +| `control.sqlite3` | 网关设置、私有默认 key、模型规则、会话、冷却、积分/领取、用量与目录缓存 | | `logs.sqlite3` | 请求明细与独立的聚合统计 | 审计默认保留 30 天明细与 256 MiB 逻辑明细预算，**不是数据库或目录磁盘占用的硬上限**。明细清理与淘汰都保留聚合。SQLite 失败诊断有独立预算，默认 8192 字节。已有文本日志保留原样，不回填为精确统计。 将整个数据目录挂载到可写本地存储，而不是只挂载单个数据库文件；不要在多个网关实例间共享。 -复制前停止网关，并复制整个目录——包括数据库、WAL/SHM 文件、凭证以及目录/积分状态文件；不要只备份 `.info`。这些数据属于隐私，妥善保管。 +复制前停止网关，并复制整个目录，包括数据库、WAL/SHM、凭证及迁移备份；不要只备份 `.info`。控制库包含默认密钥与登录会话，必须妥善保管。 -使用新的模型规则或自动化开关前备份控制元数据。回退到旧代码需要匹配的控制数据库快照（含 WAL/SHM，不混用文件）；旧版本会拒绝新字段。回滚无法撤销已完成的上游签到、领取或旅行派出。 +旧 JSON 只在首次升级时导入；迁移后的运行状态以 SQLite 为准。回滚须停机并迁回匹配状态，不能用旧 JSON 覆盖新的领取或会话撤销记录，见[升级与数据迁移](deployment.zh-CN.md#升级与数据迁移)。 API key 与地址见[客户端配置](clients.zh-CN.md)。 diff --git a/pyproject.toml b/pyproject.toml index 0a7312f..2537642 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ dependencies = [ "fastapi>=0.141.1", "httpx>=0.28.1", "pytest>=9.1.1", + "python-dotenv>=1.2.3", "uvicorn[standard]>=0.52.4", ] diff --git a/requirements.in b/requirements.in index 13a8f8d..46362e5 100644 --- a/requirements.in +++ b/requirements.in @@ -2,4 +2,5 @@ fastapi>=0.141.1 httpx>=0.28.1 pytest>=9.1.1 +python-dotenv>=1.2.3 uvicorn[standard]>=0.52.4 diff --git a/requirements.txt b/requirements.txt index 64fa7c5..82e7f1d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -178,7 +178,9 @@ pytest==9.1.1 \ python-dotenv==1.2.3 \ --hash=sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9 \ --hash=sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35 - # via uvicorn + # via + # codebuddy2api + # uvicorn pyyaml==6.0.3 \ --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ diff --git a/tests/test_control_store.py b/tests/test_control_store.py index b16ae07..0bb1176 100644 --- a/tests/test_control_store.py +++ b/tests/test_control_store.py @@ -23,7 +23,7 @@ def test_reopen_and_schema(self): state = self.store.update_settings({"max_images": 3}, 0) self.assertEqual(state["revision"], 1) with sqlite3.connect(self.path) as db: - self.assertEqual(db.execute("PRAGMA user_version").fetchone()[0], 1) + self.assertEqual(db.execute("PRAGMA user_version").fetchone()[0], ControlStore.SCHEMA_VERSION) other = ControlStore(self.path) self.addCleanup(other.close) self.assertEqual(other.snapshot()["settings"], {"max_images": 3}) diff --git a/tests/test_environment_config.py b/tests/test_environment_config.py index 3057c18..469e906 100644 --- a/tests/test_environment_config.py +++ b/tests/test_environment_config.py @@ -36,6 +36,7 @@ def start(self, environ=None, cli=(), saved=None): env.update(environ or {}) with patch.dict(os.environ, env, clear=True), patch.dict(converter.CONFIG, dict(converter.CONFIG), clear=True), \ patch.object(sys, 'argv', ['converter.py', '--skip-check', *cli]), \ + patch.object(converter, 'load_startup_env', return_value=set()), \ patch.object(converter, 'seed_credentials') as seed, patch.object(converter, 'CredentialPool') as pool, \ patch.object(converter, '_publish_model_cache'), patch.object(runtime_management, 'install'), \ patch.object(converter.threading, 'Thread'), patch.object(converter, '_log'), \ diff --git a/tests/test_identity_sync.py b/tests/test_identity_sync.py index 4a1da3c..eda4be0 100644 --- a/tests/test_identity_sync.py +++ b/tests/test_identity_sync.py @@ -52,6 +52,7 @@ def setUp(self): self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) self.enterContext(patch.dict(os.environ, {"CODEBUDDY_AUTH_DIR": str(self.root), "CODEBUDDY2API_LOG": "", "CODEBUDDY2API_KEY": ""})) + self.enterContext(patch.object(c, "load_startup_env", return_value=set())) self.enterContext(patch.dict(c.CONFIG, {"cred_pool": None, "cred": None, "ledger": None, "model_cache": None, "model_catalogs": {}, "account_catalogs": None, "models_remote": None, "models_intl": None, "model_guard": True, "log_path": None})) @@ -403,8 +404,12 @@ def test_main_publishes_bound_empty_or_nonempty_cache_before_any_thread_or_serve ledger = credits.CreditLedger(self.root / "credits-ledger.json") ledger.bind_identity(str(path), identity) ledger.update_credits(str(path), balance("", domain=DOMAINS["intl-cli"])) + from app.control_store import ControlStore + control = ControlStore(self.root / "control.sqlite3") + self.addCleanup(control.close) + control.state.migrate(self.root) for models in ([], [model("shared")]): - cache = credits.ModelCatalogCache(self.root / "model-catalog.json") + cache = credits.ModelCatalogCache(store=control.state) cache.put(key, models) cache.put("international", [model("legacy-forbidden")]) observed = [] diff --git a/tests/test_login.py b/tests/test_login.py index c18f7ca..b565873 100644 --- a/tests/test_login.py +++ b/tests/test_login.py @@ -26,6 +26,7 @@ def setUp(self): self.addCleanup(temporary.cleanup) self.directory = Path(temporary.name) self.enterContext(patch.dict(os.environ, {"CODEBUDDY_AUTH_DIR": temporary.name})) + self.enterContext(patch.object(converter, "load_startup_env", return_value=set())) self.enterContext(patch.dict(converter.CONFIG, {"cred_pool": None, "api_key": "", "log_path": None})) self.stdout = self.enterContext(contextlib.redirect_stdout(io.StringIO())) self.stderr = self.enterContext(contextlib.redirect_stderr(io.StringIO())) diff --git a/tests/test_native_noauth.py b/tests/test_native_noauth.py new file mode 100644 index 0000000..6083ad2 --- /dev/null +++ b/tests/test_native_noauth.py @@ -0,0 +1,57 @@ +"""Exercise the shipped image's no-TTY startup contract with disposable state.""" +import contextlib +import json +import os +from pathlib import Path +import signal +import socket +import sqlite3 +import subprocess +import sys +import tempfile +import unittest + +import httpx + +ROOT = Path(__file__).resolve().parents[1] + + +class NativeNoAuthTests(unittest.TestCase): + def test_image_default_starts_without_key_but_keeps_management_locked(self): + dockerfile = (ROOT / 'Dockerfile').read_text() + command = json.loads(next(line[4:] for line in dockerfile.splitlines() if line.startswith('CMD '))) + self.assertIn('ENV CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true', dockerfile) + with tempfile.TemporaryDirectory() as directory, socket.socket() as probe: + root = Path(directory) + probe.bind(('127.0.0.1', 0)) + port = probe.getsockname()[1] + probe.close() + env = {name: os.environ[name] for name in ('PATH', 'SYSTEMROOT') if name in os.environ} + env.update(HOME=directory, USERPROFILE=directory, TMPDIR=directory, TEMP=directory, + CODEBUDDY_AUTH_DIR=str(root / 'auth'), CODEBUDDY2API_ALLOW_OPEN_NOAUTH='true', + PYTHONDONTWRITEBYTECODE='1') + process = subprocess.Popen([sys.executable, '-B', str(ROOT / command[1]), *command[2:], '--port', str(port)], + cwd=root, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True) + try: + with httpx.Client(base_url=f'http://127.0.0.1:{port}', transport=httpx.HTTPTransport(retries=5), + timeout=5, trust_env=False) as client: + self.assertEqual(client.get('/health').json(), {'status': 'ok'}) + self.assertEqual(client.get('/admin/session').status_code, 503) + self.assertEqual(client.get('/v1/models').status_code, 200) + with contextlib.closing(sqlite3.connect(root / 'auth/control.sqlite3')) as db: + self.assertEqual(db.execute('SELECT count(*) FROM gateway_secrets').fetchone()[0], 0) + finally: + if process.poll() is None: + process.send_signal(signal.SIGINT) if os.name == 'posix' else process.terminate() + try: + output, error = process.communicate(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=5) + self.fail('Owned no-auth fixture did not shut down') + self.assertNotIn(b'cb-', output + error) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_runtime_endpoints.py b/tests/test_runtime_endpoints.py index 3b8dab5..201fee2 100644 --- a/tests/test_runtime_endpoints.py +++ b/tests/test_runtime_endpoints.py @@ -786,7 +786,8 @@ class StartupUsageHydrationTests(unittest.TestCase): def setUp(self): self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) - self.enterContext(patch.dict(os.environ, {"CODEBUDDY_AUTH_DIR": str(self.root)}, clear=False)) + self.enterContext(patch.dict(os.environ, {"CODEBUDDY_AUTH_DIR": str(self.root), "CODEBUDDY2API_KEY": "synthetic-key"}, clear=True)) + self.enterContext(patch.object(converter, "load_startup_env", return_value=set())) self.path = self.root / "usage-snapshots.json" def credential(self, uid="synthetic-uid"): @@ -819,9 +820,6 @@ def test_main_publishes_cached_usage_before_serving(self): stack.enter_context(patch.object(converter, "credits_mod", None)) # No network. stack.enter_context(patch.object(converter, "seed_credentials")) stack.enter_context(patch.object(converter, "preflight", return_value=True)) - # Keep this test on startup ordering: skip the SQLite-backed stores so no file - # handle is left open on Windows. - stack.enter_context(patch.object(runtime_management, "initialize")) stack.enter_context(patch.object(runtime_management, "install")) def capture(*args, **kwargs): @@ -850,9 +848,6 @@ def test_main_without_a_cache_leaves_usage_empty(self): stack.enter_context(patch.object(converter, "credits_mod", None)) stack.enter_context(patch.object(converter, "seed_credentials")) stack.enter_context(patch.object(converter, "preflight", return_value=True)) - # Keep this test on startup ordering: skip the SQLite-backed stores so no file - # handle is left open on Windows. - stack.enter_context(patch.object(runtime_management, "initialize")) stack.enter_context(patch.object(runtime_management, "install")) stack.enter_context(patch.object(converter.uvicorn, "run", side_effect=lambda *a, **k: observed.setdefault("usage", @@ -878,9 +873,6 @@ def test_main_starts_no_maintenance_thread_before_hydration(self): stack.enter_context(patch.object(converter, "credits_mod", None)) stack.enter_context(patch.object(converter, "seed_credentials")) stack.enter_context(patch.object(converter, "preflight", return_value=True)) - # Keep this test on startup ordering: skip the SQLite-backed stores so no file - # handle is left open on Windows. - stack.enter_context(patch.object(runtime_management, "initialize")) stack.enter_context(patch.object(runtime_management, "install")) real_thread = converter.threading.Thread @@ -908,7 +900,8 @@ def configure(self, env=None, flags=(), invalid=False, stored=None, expected_hos store.close() stack.enter_context(patch.object(converter, "managed_auth_dir", return_value=Path(directory))) stack.enter_context(patch.object(converter, "app", FastAPI())) - stack.enter_context(patch.dict(os.environ, env or {}, clear=True)) + stack.enter_context(patch.dict(os.environ, {"CODEBUDDY2API_KEY": "", **(env or {})}, clear=True)) + stack.enter_context(patch.object(converter, "load_startup_env", return_value=set())) stack.enter_context(patch.dict(converter.CONFIG)) stack.enter_context(patch("sys.argv", ["converter.py", "--skip-check", *flags])) stack.enter_context(contextlib.redirect_stderr(io.StringIO())) @@ -1030,41 +1023,41 @@ def test_invalid_config_fails_before_side_effects(self): class LogIntegrationTests(unittest.TestCase): - def test_log_rotation_is_bounded_and_thread_safe(self): + def test_runtime_events_use_sqlite_audit_not_text_files(self): + from app.audit_store import AuditStore with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "test.log" - with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 256}), \ - patch.object(converter, "LOG_MAX_BYTES", 2048): - with ThreadPoolExecutor(max_workers=8) as executor: - list(executor.map(converter._log, [f"event={i} " + "汉字" * 1000 for i in range(30)])) - files = list(Path(directory).glob("test.log*")) - self.assertLessEqual(len(files), 3) - self.assertTrue(all(file.stat().st_size <= 2048 for file in files)) - for file in files: - file.read_text(encoding="utf-8", errors="strict") - - def test_body_previews_do_not_log_image_data_or_tokens(self): + with contextlib.closing(AuditStore(Path(directory) / "logs.sqlite3")) as audit: + with patch.dict(converter.CONFIG, {"log_path": str(path), "audit_store": audit}): + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(converter._log, ["[cred] updated" for _ in range(30)])) + self.assertEqual(len(audit.list_records("runtime", limit=100)["items"]), 30) + self.assertFalse(path.exists()) + + def test_body_previews_and_secret_hints_never_reach_persistent_logs(self): + from app.audit_store import AuditStore with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "test.log" - with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 1024}): - converter._log_json("body", {"accessToken": "synthetic-private", "image": "data:image/png;base64," + "Z" * 10000}) - converter._log("ReadTimeout: Authorization: Bearer synthetic-secret") - text = path.read_text() - self.assertNotIn("synthetic-private", text) - self.assertNotIn("synthetic-secret", text) - self.assertNotIn("Z" * 20, text) - self.assertIn("ReadTimeout", text) - self.assertLess(path.stat().st_size, 1500) - - def test_zero_body_budget_keeps_summary_only(self): + database = Path(directory) / "logs.sqlite3" + key = "cb-" + "a" * 32 + with contextlib.closing(AuditStore(database)) as audit: + with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 1024, "audit_store": audit}): + converter._log_json("body", {"accessToken": "synthetic-private", "image": "data:image/png;base64," + "Z" * 10000}) + converter._log("[cred] failure " + key) + converter._log("ReadTimeout: Authorization: Bearer synthetic-secret") + self.assertFalse(path.exists()) + raw = b"".join(file.read_bytes() for file in Path(directory).glob("logs.sqlite3*")) + for secret in (key.encode(), b"synthetic-private", b"synthetic-secret", b"Z" * 20): + self.assertNotIn(secret, raw) + + def test_retired_text_setting_cannot_reenable_file_output(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "test.log" - with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 0}): + with patch.dict(converter.CONFIG, {"log_path": str(path), "log_body_limit": 0, "audit_store": None}): converter._log_json("body", {"text": "must-not-be-logged"}) converter._log_text_body("raw", "must-not-be-logged") converter._log("request summary") - self.assertIn("request summary", path.read_text()) - self.assertNotIn("must-not-be-logged", path.read_text()) + self.assertFalse(path.exists()) if __name__ == "__main__": diff --git a/tests/test_sqlite_state.py b/tests/test_sqlite_state.py new file mode 100644 index 0000000..0e6d607 --- /dev/null +++ b/tests/test_sqlite_state.py @@ -0,0 +1,581 @@ +"""Exercise SQLite migration, persisted defaults and authentication using synthetic state.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import concurrent.futures +import contextlib +import io +import json +import os +import sqlite3 +import tempfile +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from app.admin_auth import AdminAuth, SessionStoreError +from app.control_store import ControlStore +from app.credential_cooldowns import CredentialCooldowns +from app.credits import CreditLedger, ModelCatalogCache +from app.model_blocks import ModelBlocks +from app.safe_logging import sanitize_log_text +from app.settings import resolve_settings +from app.startup import announce_default_key, load_startup_env, resolve_startup_key +from app.state_store import LEGACY_FILES +from app.trial_rewards import TrialLedger, attempt_trial +from app.usage_snapshots import UsageSnapshots + +IDENTITY = "a" * 64 +FIXED_KEY = "synthetic-fixed-key" + + +class Terminal(io.StringIO): + def isatty(self): + return True + + +class SQLiteStateTests(unittest.TestCase): + def setUp(self): + self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.control = self.open_control() + self.state = self.control.state + self.now = time.time() + self.console_open = self.enterContext(patch("app.startup.terminal_stream", side_effect=OSError("no fixture terminal"))) + + def console(self): + terminal = Terminal() + self.console_open.side_effect = lambda: contextlib.nullcontext(terminal) + return terminal + + def open_control(self): + control = ControlStore(self.root / "control.sqlite3") + self.addCleanup(control.close) + return control + + def create_legacy(self): + ledger = CreditLedger(self.root / LEGACY_FILES["credits"]) + ledger.bind_identity("fixture.info", IDENTITY) + ledger.mark_checkin("fixture.info", "2026-09-22", True, 0, "ok") + ledger.update_credits("fixture.info", {"credits": 12, "segments": [], "partial": True}) + cache = ModelCatalogCache(self.root / LEGACY_FILES["catalog"]) + cache.put("account:version", [{"id": "model"}]) + blocks = ModelBlocks(self.root / LEGACY_FILES["model_blocks"]) + blocks.note("https://example.invalid/chat", "model", now=self.now) + cooldowns = CredentialCooldowns(self.root / LEGACY_FILES["cooldowns"]) + cooldowns.note_credential(IDENTITY, "cn-work", self.now + 300) + cooldowns.note_model(IDENTITY, "cn-work", "model", self.now + 600) + usage = UsageSnapshots(self.root / LEGACY_FILES["usage"]) + usage.store("fixture.info", IDENTITY, "domestic", {"total_credits": 12, "requests": 2, + "by_day": {"2026-09-22": {"model": 12}}}, partial=True) + trial = TrialLedger(self.root / LEGACY_FILES["trial"]) + self.assertTrue(trial.begin(IDENTITY)) + trial.finish(IDENTITY, {"ok": True, "already": False, "code": 0, "status": 200}) + auth = AdminAuth({"api_key": FIXED_KEY, "session_path": self.root / LEGACY_FILES["sessions"]}) + result, status = auth.login(SimpleNamespace(client=SimpleNamespace(host="fixture"), cookies={}), FIXED_KEY) + self.assertEqual(status, 200) + return result[0] + + def test_all_legacy_sources_migrate_once_and_runtime_never_updates_json(self): + sid = self.create_legacy() + before = {name: (self.root / file).read_bytes() for name, file in LEGACY_FILES.items()} + self.state.migrate(self.root) + ledger = CreditLedger(store=self.state) + self.assertTrue(ledger.checkin_done("fixture.info", "2026-09-22")) + self.assertTrue(ledger.entry("fixture.info")["credits"]["partial"]) + self.assertTrue(ModelCatalogCache(store=self.state).fresh("account:version")) + self.assertTrue(ModelBlocks(store=self.state).blocked("https://example.invalid/chat", "model")) + cooldowns = CredentialCooldowns(store=self.state) + self.assertGreater(cooldowns.credential_until(IDENTITY, "cn-work"), self.now) + self.assertGreater(cooldowns.model_until(IDENTITY, "cn-work", "model"), self.now) + self.assertTrue(UsageSnapshots(store=self.state).accounts()["fixture.info"]["partial"]) + trial = TrialLedger(store=self.state) + self.assertFalse(trial.begin(IDENTITY, self.now + 3 * 86400)) + auth = AdminAuth({"api_key": FIXED_KEY, "state_store": self.state}) + auth.reconcile() + self.assertIn(sid, auth.sessions) + ledger.note_error("fixture.info", "fixture error") + cooldowns.clear_credential(IDENTITY, "cn-work") + UsageSnapshots(store=self.state).forget("fixture.info") + auth.config["api_key"] = "rotated-fixture" + auth.reconcile() + self.state.migrate(self.root) + self.assertFalse(CredentialCooldowns(store=self.state).credential_until(IDENTITY, "cn-work")) + self.assertFalse(UsageSnapshots(store=self.state).accounts()) + auth.config["api_key"] = FIXED_KEY + auth.reconcile() + self.assertNotIn(sid, auth.sessions) + self.assertEqual(before, {name: (self.root / file).read_bytes() for name, file in LEGACY_FILES.items()}) + self.assertEqual(self.control._db.execute("SELECT count(*) FROM state_imports").fetchone()[0], 7) + + def test_absent_legacy_file_cannot_resurrect_later(self): + self.state.migrate(self.root) + (self.root / LEGACY_FILES["credits"]).write_text('{"version":1,"creds":{"old":{}}}') + self.state.migrate(self.root) + self.assertIsNone(self.state.get("credits")) + + def test_invalid_migration_rolls_back_every_document_and_marker(self): + self.create_legacy() + (self.root / LEGACY_FILES["trial"]).write_text('{"version":1,"accounts":{"bad":{}}}') + with self.assertRaisesRegex(ValueError, "trial-ledger"): + self.state.migrate(self.root) + for table in ("state_imports", "runtime_state"): + self.assertEqual(self.control._db.execute(f"SELECT count(*) FROM {table}").fetchone()[0], 0) + + def test_duplicate_fields_and_upstream_tokens_are_rejected(self): + file = self.root / LEGACY_FILES["credits"] + for payload in ('{"version":1,"version":1,"creds":{}}', + '{"version":1,"creds":{"f":{"accessToken":"fixture"}}}'): + file.write_text(payload) + with self.assertRaises(ValueError): + self.state.migrate(self.root) + + def test_symlink_legacy_source_is_rejected(self): + target = self.root / "target" + target.write_text('{"version":1,"creds":{}}') + (self.root / LEGACY_FILES["credits"]).symlink_to(target) + with self.assertRaises(ValueError): + self.state.migrate(self.root) + + def test_upgrade_v1_preserves_settings_and_rejects_old_readers(self): + self.control.update_settings({"max_images": 3}, 0) + self.control._db.execute("PRAGMA user_version=1") + other = self.open_control() + self.assertEqual(other.snapshot()["settings"]["max_images"], 3) + self.assertEqual(other._db.execute("PRAGMA user_version").fetchone()[0], 2) + + def test_trial_reservation_is_atomic_across_connections_and_failures_are_closed(self): + other = self.open_control() + ledgers = [TrialLedger(store=self.state), TrialLedger(store=other.state)] + with concurrent.futures.ThreadPoolExecutor(2) as executor: + results = list(executor.map(lambda ledger: ledger.begin(IDENTITY), ledgers)) + self.assertEqual(sorted(results), [False, True]) + self.assertFalse(ledgers[0].begin(IDENTITY)) + with patch.object(self.state, "_put", side_effect=sqlite3.OperationalError("fixture")), \ + patch("app.trial_rewards.claim_trial") as claim: + with self.assertRaises(sqlite3.OperationalError): + attempt_trial(ledgers[0], "b" * 64, {"X-Domain": "www.workbuddy.ai"}) + claim.assert_not_called() + + def test_cooldown_write_failure_is_visible_and_retried(self): + cooldowns = CredentialCooldowns(store=self.state) + with patch.object(self.state, "put", side_effect=sqlite3.OperationalError("fixture")): + self.assertFalse(cooldowns.note_credential(IDENTITY, "cn-work", self.now + 300)) + self.assertEqual(cooldowns.last_error, "OperationalError") + self.assertTrue(cooldowns.note_credential(IDENTITY, "cn-work", self.now + 300)) + self.assertGreater(CredentialCooldowns(store=self.state).credential_until(IDENTITY, "cn-work"), self.now) + + def test_rebuildable_cache_write_failure_keeps_in_memory_routing(self): + blocks = ModelBlocks(store=self.state) + catalogs = ModelCatalogCache(store=self.state) + with patch.object(self.state, "put", side_effect=sqlite3.OperationalError("fixture")): + with self.assertWarns(RuntimeWarning): + blocks.note("https://example.invalid/chat", "model", now=self.now) + with self.assertWarns(RuntimeWarning): + catalogs.put("account:version", [{"id": "model"}]) + self.assertTrue(blocks.blocked("https://example.invalid/chat", "model")) + self.assertTrue(catalogs.fresh("account:version")) + self.assertEqual(catalogs.models("account:version"), [{"id": "model"}]) + + + def test_sqlite_session_read_failure_preserves_record_and_retries_epoch(self): + self.create_legacy() + self.state.migrate(self.root) + before = self.state.get("sessions") + for error in (sqlite3.OperationalError("locked"), sqlite3.DatabaseError("malformed"), + OSError("unreadable"), ValueError("invalid state")): + auth = AdminAuth({"api_key": FIXED_KEY, "state_store": self.state}) + with self.subTest(error=type(error).__name__), \ + patch.object(self.state, "get", side_effect=error), \ + patch.object(self.state, "delete") as delete: + for attempt in (auth.reconcile, auth.enabled): + with self.assertRaises(SessionStoreError): + attempt() + self.assertIsNone(auth._configured_key) + self.assertFalse(auth.sessions) + delete.assert_not_called() + self.assertEqual(auth.storage()["last_error"], type(error).__name__) + self.assertEqual(self.state.get("sessions"), before) + auth.reconcile() + self.assertEqual(auth._configured_key, FIXED_KEY) + self.assertEqual(set(auth.sessions), set(before["sessions"])) + self.assertFalse(auth.storage()["degraded"]) + + def test_invalid_sqlite_sessions_stop_startup_without_deleting_evidence(self): + self.control._db.execute("INSERT INTO runtime_state VALUES('sessions', 'invalid-json')") + auth = AdminAuth({"api_key": FIXED_KEY, "state_store": self.state}) + with self.assertRaises(SessionStoreError): + auth.reconcile() + self.assertIsNone(auth._configured_key) + self.assertEqual(self.control._db.execute("SELECT payload FROM runtime_state WHERE name='sessions'").fetchone()[0], + "invalid-json") + + def test_all_sqlite_loaders_propagate_read_failures_without_empty_fallbacks(self): + loaders = (lambda: CreditLedger(store=self.state), lambda: ModelCatalogCache(store=self.state), + lambda: ModelBlocks(store=self.state), lambda: CredentialCooldowns(store=self.state), + lambda: UsageSnapshots(store=self.state), lambda: TrialLedger(store=self.state).snapshot()) + for error in (sqlite3.OperationalError("locked"), ValueError("invalid state"), OSError("unreadable")): + for loader in loaders: + with self.subTest(error=type(error).__name__, loader=loader), \ + patch.object(self.state, "get", side_effect=error): + with self.assertRaises(type(error)): + loader() + + + def test_session_revocation_failure_does_not_activate_new_key(self): + auth = AdminAuth({"api_key": FIXED_KEY, "state_store": self.state}) + auth.reconcile() + auth.config["api_key"] = "rotated-fixture" + with patch.object(self.state, "put", side_effect=sqlite3.OperationalError("fixture")), \ + patch.object(self.state, "delete", side_effect=sqlite3.OperationalError("fixture")): + with self.assertRaises(SessionStoreError): + auth.reconcile() + self.assertEqual(auth._configured_key, FIXED_KEY) + + def test_default_key_is_atomic_private_and_survives_restart(self): + other = self.open_control() + with concurrent.futures.ThreadPoolExecutor(2) as executor: + results = list(executor.map(lambda state: state.default_key(create=True), [self.state, other.state])) + self.assertEqual(results[0], results[1]) + key, pending = results[0] + self.assertRegex(key, r"^cb-[0-9a-f]{32}$") + self.assertTrue(pending) + self.assertNotIn(key, json.dumps(self.control.snapshot())) + self.assertEqual(self.open_control().state.default_key(), (key, True)) + with self.state.claim_announcement(key) as claimed: + self.assertTrue(claimed) + with other.state.claim_announcement(key) as claimed: + self.assertFalse(claimed) + self.assertEqual(other.state.default_key(), (key, False)) + + def test_broken_saved_key_is_not_regenerated(self): + self.state.default_key(create=True) + self.control._db.execute("UPDATE gateway_secrets SET value='broken'") + with self.assertRaises(ValueError): + self.state.default_key(create=True) + self.assertEqual(self.control._db.execute("SELECT value FROM gateway_secrets").fetchone()[0], "broken") + + def config(self, source="default", key=""): + return {"state_store": self.state, "control_store": self.control, "api_key": key, + "host": "127.0.0.1", "settings_sources": {"api_key": source}} + + def test_successful_announcement_is_committed_and_not_repeated(self): + config = self.config() + terminal = self.console() + with patch("sys.stderr", io.StringIO()) as output: + resolve_startup_key(config, SimpleNamespace(api_key="")) + key = config["api_key"] + self.assertEqual(terminal.getvalue(), "") + announce_default_key(config) + announce_default_key(config) + again = self.config() + resolve_startup_key(again, SimpleNamespace(api_key="")) + announce_default_key(again) + self.assertEqual(terminal.getvalue().count(key), 1) + self.assertNotIn(key, output.getvalue()) + self.assertEqual(again["api_key"], key) + self.assertIsNone(next(row for row in resolve_settings(config) if row["key"] == "api_key")["value"]) + self.assertNotIn(key, sanitize_log_text("key leaked " + key)) + with patch("sys.stderr", io.StringIO()): + resolve_startup_key(self.config(), SimpleNamespace(api_key="")) + + def test_missing_terminal_at_disclosure_does_not_consume_announcement(self): + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + self.console_open.side_effect = OSError("terminal gone") + with self.assertRaises(OSError): + announce_default_key(config) + self.assertEqual(self.state.default_key(), (config["api_key"], True)) + self.assertEqual(terminal.getvalue(), "") + + def test_announcement_reservation_failure_never_writes_terminal(self): + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + with patch.object(self.state, "claim_announcement", side_effect=sqlite3.OperationalError("locked")), \ + self.assertRaises(sqlite3.OperationalError): + announce_default_key(config) + self.assertEqual(terminal.getvalue(), "") + self.assertEqual(self.state.default_key(), (config["api_key"], True)) + + + def test_terminal_write_and_flush_failures_leave_default_key_pending(self): + for operation in ("write", "flush"): + with self.subTest(operation=operation): + self.control._db.execute("DELETE FROM gateway_secrets") + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + key = config["api_key"] + with patch.object(terminal, operation, side_effect=BrokenPipeError("fixture terminal lost")), \ + self.assertRaises(BrokenPipeError): + announce_default_key(config) + self.assertEqual(self.state.default_key(), (key, True)) + self.assertTrue(config["announce_default_key"]) + recovered = self.console() + announce_default_key(config) + self.assertEqual(recovered.getvalue().count(key), 1) + self.assertEqual(self.state.default_key(), (key, False)) + + def test_process_crash_during_disclosure_rolls_back_the_marker(self): + import subprocess + key, _ = self.state.default_key(create=True) + script = ''' +import os, sys +from app.control_store import ControlStore +control = ControlStore(sys.argv[1]) +key, _ = control.state.default_key() +with control.state.claim_announcement(key) as claimed: + assert claimed + os._exit(17) +''' + result = subprocess.run([sys.executable, "-B", "-c", script, str(self.control.path)], + capture_output=True, timeout=15) + self.assertEqual(result.returncode, 17) + self.assertEqual((result.stdout, result.stderr), (b"", b"")) + self.assertEqual(self.open_control().state.default_key(), (key, True)) + + + def test_commit_failure_after_disclosure_keeps_recovery_pending(self): + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + key = config["api_key"] + def deny_commit(action, arg1, arg2, database, trigger): + return sqlite3.SQLITE_DENY if action == sqlite3.SQLITE_TRANSACTION and arg1 == "COMMIT" else sqlite3.SQLITE_OK + self.control._db.set_authorizer(deny_commit) + try: + with self.assertRaises(sqlite3.DatabaseError): + announce_default_key(config) + finally: + self.control._db.set_authorizer(None) + self.assertFalse(self.control._db.in_transaction) + self.assertEqual(terminal.getvalue().count(key), 1) + self.assertEqual(self.state.default_key(), (key, True)) + self.assertTrue(config["announce_default_key"]) + recovered = self.console() + announce_default_key(config) + self.assertEqual(recovered.getvalue().count(key), 1) + self.assertEqual(self.state.default_key(), (key, False)) + + def test_concurrent_disclosures_print_only_after_acquiring_exclusive_claim(self): + import threading + terminal = self.console() + key, _ = self.state.default_key(create=True) + other = self.open_control() + configs = [{**self.config(), "state_store": state, "api_key": key, "announce_default_key": True} + for state in (self.state, other.state)] + barrier = threading.Barrier(2, timeout=5) + def announce(config): + barrier.wait() + announce_default_key(config) + with concurrent.futures.ThreadPoolExecutor(2) as executor: + list(executor.map(announce, configs)) + self.assertEqual(terminal.getvalue().count(key), 1) + self.assertEqual(self.state.default_key(), (key, False)) + self.assertTrue(all(not config["announce_default_key"] for config in configs)) + + def test_terminal_failure_after_binding_shuts_down_with_pending_key(self): + import asyncio + import uvicorn + from fastapi import FastAPI + from unittest.mock import AsyncMock + from app.startup import run_server + terminal = self.console() + config = self.config() + resolve_startup_key(config, SimpleNamespace(api_key="")) + async def ready(server, sockets=None): + server.started = True + with patch.object(uvicorn.Server, "startup", ready), \ + patch.object(uvicorn.Server, "shutdown", new=AsyncMock()) as shutdown, \ + patch.object(uvicorn.Server, "run", lambda server: asyncio.run(server.startup())), \ + patch.object(terminal, "flush", side_effect=BrokenPipeError("fixture")), \ + self.assertRaises(BrokenPipeError): + run_server(FastAPI(), config, host="127.0.0.1", port=8787) + shutdown.assert_awaited_once() + self.assertEqual(self.state.default_key(), (config["api_key"], True)) + + + def test_explicit_sources_do_not_overwrite_saved_default_or_print(self): + key, _ = self.state.default_key(create=True) + for source in ("cli", "environment", "dotenv"): + for supplied in ("", "override-fixture"): + config = self.config(source, supplied) + with patch("sys.stderr", io.StringIO()) as output: + resolve_startup_key(config, SimpleNamespace(api_key=supplied)) + announce_default_key(config) + self.assertEqual(config["api_key"], supplied) + self.assertEqual(output.getvalue(), "") + self.assertEqual(self.state.default_key()[0], key) + + def test_nonterminal_and_public_first_start_do_not_generate_hidden_key(self): + with patch("sys.stderr", io.StringIO()), self.assertRaises(ValueError): + resolve_startup_key(self.config(), SimpleNamespace(api_key="")) + config = self.config() + config["host"] = "0.0.0.0" + with patch("sys.stderr", Terminal()), self.assertRaises(ValueError): + resolve_startup_key(config, SimpleNamespace(api_key="")) + self.assertEqual(self.state.default_key(), (None, False)) + + def test_explicit_noauth_opt_in_skips_first_key_generation_without_a_terminal(self): + for value in ("1", "true", "yes", "TRUE"): + for host in ("127.0.0.1", "0.0.0.0"): + config = {**self.config(), "host": host} + with self.subTest(value=value, host=host), \ + patch.dict(os.environ, {"CODEBUDDY2API_ALLOW_OPEN_NOAUTH": value}, clear=True): + resolve_startup_key(config, SimpleNamespace(api_key="")) + self.assertFalse(config["api_key"]) + self.assertFalse(config["announce_default_key"]) + self.assertEqual(self.state.default_key(), (None, False)) + self.console_open.assert_not_called() + + def test_disabled_noauth_opt_in_keeps_first_start_protection(self): + for value in ("", "0", "false", "no", "invalid"): + with self.subTest(value=value), \ + patch.dict(os.environ, {"CODEBUDDY2API_ALLOW_OPEN_NOAUTH": value}, clear=True), \ + self.assertRaises(ValueError): + resolve_startup_key({**self.config(), "host": "0.0.0.0"}, SimpleNamespace(api_key="")) + self.assertEqual(self.state.default_key(), (None, False)) + + def test_noauth_opt_in_never_downgrades_an_existing_default_or_override(self): + key, _ = self.state.default_key(create=True) + with patch.dict(os.environ, {"CODEBUDDY2API_ALLOW_OPEN_NOAUTH": "true"}, clear=True): + with self.assertRaises(ValueError): + resolve_startup_key(self.config(), SimpleNamespace(api_key="")) + with self.state.claim_announcement(key): + pass + config = {**self.config(), "host": "0.0.0.0"} + resolve_startup_key(config, SimpleNamespace(api_key="")) + self.assertEqual(config["api_key"], key) + for source in ("cli", "environment", "dotenv"): + config = self.config(source, "configured-fixture") + resolve_startup_key(config, SimpleNamespace(api_key="configured-fixture")) + self.assertEqual(config["api_key"], "configured-fixture") + + def test_docker_default_without_key_or_terminal_reaches_server(self): + import converter + from fastapi import FastAPI + dockerfile = (Path(__file__).resolve().parents[1] / "Dockerfile").read_text() + command = json.loads(next(line[4:] for line in dockerfile.splitlines() if line.startswith("CMD "))) + self.assertIn("ENV CODEBUDDY2API_ALLOW_OPEN_NOAUTH=true", dockerfile) + with contextlib.chdir(self.root), contextlib.ExitStack() as stack: + stack.enter_context(patch.dict(os.environ, {"HOME": str(self.root), "CODEBUDDY_AUTH_DIR": str(self.root), + "CODEBUDDY2API_ALLOW_OPEN_NOAUTH": "true"}, clear=True)) + stack.enter_context(patch.dict(converter.CONFIG)) + stack.enter_context(patch.object(converter, "app", FastAPI())) + stack.enter_context(patch.object(sys, "argv", command[1:])) + stack.enter_context(patch.object(converter, "seed_credentials")) + stack.enter_context(patch.object(converter, "CredentialPool")) + stack.enter_context(patch.object(converter, "_publish_model_cache")) + stack.enter_context(patch.object(converter.threading, "Thread")) + stack.enter_context(patch("sys.stderr", io.StringIO())) + captured = {} + def serve(app, config, **kwargs): + captured.update(kwargs, key=config["api_key"], announcement=config["announce_default_key"]) + stack.enter_context(patch.object(converter, "run_server", side_effect=serve)) + converter.main() + self.assertEqual((captured["host"], captured["port"]), ("0.0.0.0", 8787)) + self.assertFalse(captured["key"]) + self.assertFalse(captured["announcement"]) + self.assertEqual(self.state.default_key(), (None, False)) + self.console_open.assert_not_called() + + + def test_dotenv_is_optional_and_process_environment_wins(self): + path = self.root / ".env" + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(load_startup_env(path), set()) + path.write_text("CODEBUDDY2API_PORT=9090\nCODEBUDDY2API_KEY=file-fixture\n") + os.environ["CODEBUDDY2API_PORT"] = "9091" + changed = load_startup_env(path) + self.assertEqual(os.environ["CODEBUDDY2API_PORT"], "9091") + self.assertEqual(os.environ["CODEBUDDY2API_KEY"], "file-fixture") + self.assertEqual(changed, {"CODEBUDDY2API_KEY"}) + config = self.config("environment", "file-fixture") + resolve_startup_key(config, SimpleNamespace(api_key="file-fixture"), changed) + self.assertEqual(config["settings_sources"]["api_key"], "dotenv") + + def test_invalid_dotenv_fails_without_loading_partial_values(self): + path = self.root / ".env" + path.write_text('CODEBUDDY2API_PORT=9090\nCODEBUDDY2API_KEY="unterminated\n') + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(ValueError, "第 2 行") as caught: + load_startup_env(path) + self.assertNotIn("unterminated", str(caught.exception)) + self.assertNotIn("CODEBUDDY2API_PORT", os.environ) + + + def test_full_startup_precedence_and_default_restore(self): + import converter + from app import runtime_management + from fastapi import FastAPI + + def boot(env=None, flags=()): + observed = {} + terminal = self.console() + with contextlib.chdir(self.root), contextlib.ExitStack() as stack: + stack.enter_context(patch.dict(os.environ, {"HOME": str(self.root), "CODEBUDDY_AUTH_DIR": str(self.root), **(env or {})}, clear=True)) + stack.enter_context(patch.dict(converter.CONFIG)) + stack.enter_context(patch.object(converter, "app", FastAPI())) + stack.enter_context(patch.object(sys, "argv", ["converter.py", "--skip-check", *flags])) + stack.enter_context(patch.object(converter, "seed_credentials")) + stack.enter_context(patch.object(converter, "CredentialPool")) + stack.enter_context(patch.object(converter, "_publish_model_cache")) + stack.enter_context(patch.object(converter.threading, "Thread")) + stack.enter_context(patch.object(runtime_management, "install")) + output = stack.enter_context(patch("sys.stderr", Terminal())) + def serve(app, config, **kwargs): + observed.update(kwargs, key=config["api_key"], sources=dict(config["settings_sources"])) + announce_default_key(config) + stack.enter_context(patch.object(converter, "run_server", side_effect=serve)) + converter.main() + observed["output"] = output.getvalue() + observed["terminal"] = terminal.getvalue() + return observed + + first = boot() + key = first["key"] + self.assertEqual(first["terminal"].count(key), 1) + self.assertNotIn(key, first["output"]) + self.assertEqual(first["sources"]["api_key"], "generated") + envfile = self.root / ".env" + envfile.write_text("CODEBUDDY2API_PORT=9092\nCODEBUDDY2API_KEY=file-fixture\n") + file_run = boot() + self.assertEqual((file_run["port"], file_run["key"], file_run["sources"]["api_key"]), (9092, "file-fixture", "dotenv")) + environment = boot({"CODEBUDDY2API_PORT": "9093", "CODEBUDDY2API_KEY": "env-fixture"}) + self.assertEqual((environment["port"], environment["key"]), (9093, "env-fixture")) + cli = boot({"CODEBUDDY2API_PORT": "9093", "CODEBUDDY2API_KEY": "env-fixture"}, + ("--port", "9094", "--api-key", "cli-fixture")) + self.assertEqual((cli["port"], cli["key"]), (9094, "cli-fixture")) + envfile.unlink() + resumed = boot() + self.assertEqual(resumed["key"], key) + self.assertNotIn(key, resumed["output"]) + self.assertEqual(resumed["terminal"], "") + self.assertEqual(self.state.default_key(), (key, False)) + self.assertFalse(list(self.root.glob("*.json"))) + + def test_listener_failure_does_not_consume_announcement(self): + import asyncio + import uvicorn + from fastapi import FastAPI + from unittest.mock import AsyncMock + from app.startup import run_server + config = self.config() + terminal = self.console() + with patch("sys.stderr", io.StringIO()): + resolve_startup_key(config, SimpleNamespace(api_key="")) + key = config["api_key"] + with patch.object(uvicorn.Server, "startup", new=AsyncMock(side_effect=SystemExit(3))), \ + patch.object(uvicorn.Server, "run", lambda server: asyncio.run(server.startup())): + with self.assertRaises(SystemExit): + run_server(FastAPI(), config, host="127.0.0.1", port=8787) + self.assertNotIn(key, terminal.getvalue()) + self.assertEqual(self.state.default_key(), (key, True)) + + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_terminal_output.py b/tests/test_terminal_output.py new file mode 100644 index 0000000..d9988e5 --- /dev/null +++ b/tests/test_terminal_output.py @@ -0,0 +1,114 @@ +"""Verify secret disclosure uses an actual terminal instead of stdout/stderr.""" +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import contextlib +import io +import os +import sqlite3 +import stat +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from app.startup import terminal_stream + + +class TerminalOutputTests(unittest.TestCase): + def test_regular_file_and_non_tty_device_are_rejected_and_closed(self): + for mode, tty in ((stat.S_IFREG, True), (stat.S_IFCHR, False)): + with self.subTest(mode=mode, tty=tty), \ + patch('app.startup.os.open', return_value=123), \ + patch('app.startup.os.fstat', return_value=type('Stat', (), {'st_mode': mode})()), \ + patch('app.startup.os.isatty', return_value=tty), \ + patch('app.startup.os.close') as close, \ + patch('app.startup.os.fdopen') as fdopen: + with self.assertRaises(OSError): + terminal_stream() + close.assert_called_once_with(123) + fdopen.assert_not_called() + + def test_platform_console_path_is_fixed(self): + for platform, device in (('nt', 'CONOUT$'), ('posix', '/dev/tty')): + with self.subTest(platform=platform), \ + patch('app.startup.os.name', platform), \ + patch('app.startup.os.open', return_value=123) as opened, \ + patch('app.startup.os.fstat', return_value=type('Stat', (), {'st_mode': stat.S_IFCHR})()), \ + patch('app.startup.os.isatty', return_value=True), \ + patch('app.startup.os.fdopen', return_value=io.StringIO()): + with terminal_stream(): + pass + self.assertEqual(opened.call_args.args[0], device) + self.assertFalse(opened.call_args.args[1] & os.O_CREAT) + + @unittest.skipUnless(os.name == 'posix', 'Real controlling PTY requires POSIX') + def test_real_pty_with_redirected_standard_streams_discloses_once(self): + import pty + with tempfile.TemporaryDirectory() as directory: + database = Path(directory) / 'control.sqlite3' + for attempt in range(2): + master, slave = pty.openpty() + try: + script = ''' +import fcntl, os, sys, termios +from types import SimpleNamespace +from app.control_store import ControlStore +from app.startup import resolve_startup_key, announce_default_key +os.setsid() +fcntl.ioctl(int(sys.argv[1]), termios.TIOCSCTTY, 0) +control = ControlStore(sys.argv[2]) +try: + config = {"settings_sources": {}, "state_store": control.state, "host": "127.0.0.1"} + resolve_startup_key(config, SimpleNamespace(api_key="")) + announce_default_key(config) + announce_default_key(config) +finally: + control.close() +''' + result = subprocess.run([sys.executable, '-B', '-c', script, str(slave), str(database)], + pass_fds=(slave,), stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=20, check=True) + with contextlib.closing(sqlite3.connect(database)) as db: + key, announced = db.execute('SELECT value,announced FROM gateway_secrets').fetchone() + self.assertEqual(announced, 1) + os.set_blocking(master, False) + try: + output = os.read(master, 16384) + except BlockingIOError: + output = b'' + self.assertEqual(output.count(key.encode()), 1 if attempt == 0 else 0) + self.assertEqual((result.stdout, result.stderr), (b'', b'')) + finally: + os.close(slave) + os.close(master) + + @unittest.skipUnless(os.name == 'posix', 'Detached process requires POSIX') + def test_no_controlling_terminal_cannot_create_hidden_key(self): + with tempfile.TemporaryDirectory() as directory: + database = Path(directory) / 'control.sqlite3' + script = ''' +import sys +from types import SimpleNamespace +from app.control_store import ControlStore +from app.startup import resolve_startup_key +control = ControlStore(sys.argv[1]) +try: + try: + resolve_startup_key({"settings_sources": {}, "state_store": control.state, "host": "127.0.0.1"}, + SimpleNamespace(api_key="")) + except ValueError: + assert control.state.default_key() == (None, False) + else: + raise AssertionError("Headless startup unexpectedly generated a key") +finally: + control.close() +''' + result = subprocess.run([sys.executable, '-B', '-c', script, str(database)], + start_new_session=True, capture_output=True, timeout=20, check=True) + self.assertEqual((result.stdout, result.stderr), (b'', b'')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/webui_fixture.py b/tests/webui_fixture.py index 236d292..dcc0740 100644 --- a/tests/webui_fixture.py +++ b/tests/webui_fixture.py @@ -48,17 +48,19 @@ def main(): root = Path(directory) os.environ["CODEBUDDY_AUTH_DIR"] = directory gateway.CONFIG.update(api_key="synthetic-e2e-key", log_path=None, - trial_ledger=gateway.trial_rewards.TrialLedger(root / "trial-ledger.json"), control_store=ControlStore(root / "control.sqlite3"), audit_store=AuditStore(root / "logs.sqlite3")) + state = gateway.CONFIG["control_store"].state + state.migrate(root) + gateway.CONFIG.update(state_store=state, trial_ledger=gateway.trial_rewards.TrialLedger(store=state)) apply_persisted_settings(gateway.CONFIG, explicit=("api_key",), environ={}) credential = {"account": {"uid": "fixture-account", "nickname": "集成测试凭证"}, "auth": {"domain": "www.workbuddy.cn", "accessToken": "synthetic-browser-access", "refreshToken": "synthetic-browser-refresh", "expiresAt": (time.time() + 86400) * 1000}} path = gateway.atomic_write_credential(root, "fixture.info", json.dumps(credential).encode()) - pool = gateway.CredentialPool([path]) + pool = gateway.CredentialPool([path], state_store=state) gateway.CONFIG["cred_pool"], gateway.CONFIG["cred"] = pool, pool.first() - ledger = gateway.credits_mod.CreditLedger(root / "credits-ledger.json") + ledger = gateway.credits_mod.CreditLedger(store=state) pool.set_ledger(ledger) ledger.update_credits(str(path), {"credits": 125, "intl": False, "segments": []}) identity = pool.entries()[0]["account_key"] diff --git a/uv.lock b/uv.lock index 008e420..bfde9bf 100644 --- a/uv.lock +++ b/uv.lock @@ -59,6 +59,7 @@ dependencies = [ { name = "fastapi" }, { name = "httpx" }, { name = "pytest" }, + { name = "python-dotenv" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -67,6 +68,7 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.141.1" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "pytest", specifier = ">=9.1.1" }, + { name = "python-dotenv", specifier = ">=1.2.3" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.52.4" }, ] diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index 6c1b4b2..26595f3 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -28,6 +28,8 @@ const sourceLabels: Record = { cli: "命令行", environment: "环境变量", env: "环境变量", + dotenv: ".env 文件", + generated: "本地默认密钥", management: "管理界面", default: "默认值", internal: "内置",