diff --git a/.air.toml b/.air.toml index ad58ea4e..c14df604 100644 --- a/.air.toml +++ b/.air.toml @@ -22,7 +22,7 @@ bin = "tmp/ai-cloudops" cmd = "go build -o ./tmp/ai-cloudops main.go" # 自定义二进制文件 -full_bin = "APP_ENV=development ./tmp/ai-cloudops" +full_bin = "ENV=development ./tmp/ai-cloudops" # SEC-27：后端只读 ENV，APP_ENV 不生效；未设置/未知取值拒绝启动 # 监控的文件扩展名 include_ext = ["go", "yaml", "yml", "json", "toml"] diff --git a/.githooks/README.md b/.githooks/README.md index 778b950a..bad04ccb 100644 --- a/.githooks/README.md +++ b/.githooks/README.md @@ -7,7 +7,7 @@ ### 执行步骤 1. **`go generate ./...`** - 自动生成代码（如Wire依赖注入、Swagger文档等） -2. **代码格式化** - 自动运行`go fmt`并将格式化后的文件添加到暂存区 +2. **代码格式检查** - 对暂存的 `.go` 文件的暂存版本运行 `gofmt -l`，不规范则拒绝提交（钩子不会修改或暂存任何文件） 3. **静态分析** - 运行`go vet`检查潜在问题 4. **编译检查** - 运行`go build`确保代码可以正常编译 @@ -43,7 +43,8 @@ wire: no provider found for invalid type 4. 临时跳过：`SKIP_GENERATE=1 git commit -m "你的消息"` #### 格式化问题 -钩子会自动格式化代码并添加到暂存区，无需手动处理。 +钩子只检查、不自动格式化，也不会执行 `git add`（避免把未经 gitleaks 扫描的工作区改动带进提交）。 +按提示执行 `gofmt -w <文件>`，确认改动后重新 `git add` 再提交。 #### 编译错误 解决所有编译错误后重新提交，或使用`SKIP_BUILD=1`临时跳过。 diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 663db32f..ce89c32e 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -40,6 +40,137 @@ if [ -n "$SKIP_GENERATE" ] || [ -n "$SKIP_VET" ] || [ -n "$SKIP_BUILD" ]; then [ "$SKIP_BUILD" = "1" ] && echo " - SKIP_BUILD=1 (跳过编译检查)" fi +# --------------------------------------------------------------------------- +# gitleaks: block commits that stage secrets (SEC-01A) +# --------------------------------------------------------------------------- +# gitleaks 最低版本 8.25.0（review3g N6）：.gitleaks.toml 使用多个 [[allowlists]]（含 targetRules/condition）， +# 该写法自 8.25.0 起才支持；更低版本的策略语义不同，且 8.19 之前没有 `gitleaks git --staged`。 +# 只接受 gitleaks version 输出的严格形式 X.Y.Z（可带前缀 v；每段为不带前导零、最多 6 位的十进制数字），无法解析一律视为不满足。 +GITLEAKS_MIN_MAJOR=8 +GITLEAKS_MIN_MINOR=25 +gitleaks_version_check() { + gl_ver_raw="$(gitleaks version 2>/dev/null | head -n 1)" + gl_ver="${gl_ver_raw#v}" + case "$gl_ver" in *.*.*) ;; *) return 2 ;; esac + gl_major="${gl_ver%%.*}"; gl_rest="${gl_ver#*.}" + gl_minor="${gl_rest%%.*}"; gl_patch="${gl_rest#*.}" + # review3h P3-i：三段都只能是十进制数字，不允许前导零（单独的 0 除外），每段最多 6 位； + # 过长的数字直接视为无法解析，不交给 [ ] 做整数比较（溢出时 [ ] 报错，会被误判成“版本过低”） + for gl_c in "$gl_major" "$gl_minor" "$gl_patch"; do + case "$gl_c" in ''|*[!0-9]*|0?*|???????*) return 2 ;; esac + done + if [ "$gl_major" -gt "$GITLEAKS_MIN_MAJOR" ]; then return 0; fi + if [ "$gl_major" -eq "$GITLEAKS_MIN_MAJOR" ] && [ "$gl_minor" -ge "$GITLEAKS_MIN_MINOR" ]; then return 0; fi + return 1 +} +run_gitleaks_staged() { + if ! command -v gitleaks >/dev/null 2>&1; then + echo "[PRE-COMMIT ERROR] 未找到 gitleaks，已阻止提交。" >&2 + echo " 安装: https://github.com/gitleaks/gitleaks/releases (校验 sha256 后放入 PATH)" >&2 + echo " 然后执行: scripts/setup-git-hooks.sh" >&2 + exit 1 + fi + gl_vrc=0; gitleaks_version_check || gl_vrc=$? + if [ "$gl_vrc" -ne 0 ]; then + if [ "$gl_vrc" -eq 2 ]; then + echo "[PRE-COMMIT ERROR] 无法解析 gitleaks 版本（gitleaks version 输出: ${gl_ver_raw:-<空>}），需要 >= ${GITLEAKS_MIN_MAJOR}.${GITLEAKS_MIN_MINOR}.0，已阻止提交。" >&2 + else + echo "[PRE-COMMIT ERROR] gitleaks 版本 ${gl_ver_raw} 低于 ${GITLEAKS_MIN_MAJOR}.${GITLEAKS_MIN_MINOR}.0，已阻止提交。" >&2 + fi + echo " 请升级: https://github.com/gitleaks/gitleaks/releases (校验 sha256 后放入 PATH)" >&2 + exit 1 + fi + local repo_root tmpd f + repo_root="$(git rev-parse --show-toplevel)" || { echo "[PRE-COMMIT ERROR] 无法确定仓库根目录" >&2; exit 1; } + # 策略文件只取暂存区（index）版本，与被扫描的暂存内容一致（review3g P3-1）。 + # gitleaks 默认读取工作区的 .gitleaks.toml，且总会自动读取“源目录/.gitleaksignore”（工作区文件）： + # 未跟踪或与暂存区不一致的放行配置/指纹会在提交时生效却不进入提交。因此这两个文件只要是未跟踪、 + # 或工作区与暂存区不一致（含工作区中删除、换成符号链接），一律拒绝提交，请先暂存（git add）或还原（git restore）。 + # review3h P3-a：git diff 不比较被标记为 skip-worktree / assume-unchanged 的文件（会被绕过），这里改为： + # 1) git ls-files -v 的标记必须是 "H"（S=skip-worktree，小写=assume-unchanged，M=未合并，一律拒绝）； + # 2) 直接比对暂存区 blob 与工作区文件的字节（git show ":" | cmp），不依赖 git 的索引状态缓存。 + # （启用 core.autocrlf / clean 过滤器时字节比对可能误拒，属于 fail-closed，按提示处理即可。） + tmpd="$(mktemp -d "${TMPDIR:-/tmp}/gitleaks-hook.XXXXXX")" || { echo "[PRE-COMMIT ERROR] mktemp 失败" >&2; exit 1; } + for f in .gitleaks.toml .gitleaksignore; do + if git -C "$repo_root" ls-files --error-unmatch -- "$f" >/dev/null 2>&1; then + git -C "$repo_root" ls-files -v -- "$f" >"$tmpd/tag" 2>/dev/null || { rm -rf "$tmpd"; echo "[PRE-COMMIT ERROR] 无法读取 $f 的索引标记" >&2; exit 1; } + gl_tag="$(sed -n '1s/ .*//p' "$tmpd/tag")" + if [ "$gl_tag" != "H" ]; then + rm -rf "$tmpd" + echo "[PRE-COMMIT ERROR] $f 在索引中带有 skip-worktree / assume-unchanged / 未合并标记（git ls-files -v: ${gl_tag:-<空>}），工作区修改会被 git 忽略，已阻止提交。" >&2 + echo " 请执行 git update-index --no-skip-worktree --no-assume-unchanged -- $f，确认工作区与暂存区一致后重新提交。" >&2 + exit 1 + fi + if [ -L "$repo_root/$f" ] || [ ! -f "$repo_root/$f" ]; then + rm -rf "$tmpd" + echo "[PRE-COMMIT ERROR] $f 有未暂存的修改（在工作区中被删除或不是普通文件），gitleaks 策略只认暂存区版本，已阻止提交。" >&2 + echo " 请先 git add $f 或 git restore $f 后重新提交。" >&2 + exit 1 + fi + # 先写文件再 cmp（不用管道：pipefail 下 cmp 提前退出会让 git show 收到 SIGPIPE） + if ! git -C "$repo_root" show ":$f" >"$tmpd/index-copy" 2>/dev/null || ! cmp -s "$tmpd/index-copy" "$repo_root/$f"; then + rm -rf "$tmpd" + echo "[PRE-COMMIT ERROR] $f 有未暂存的修改（工作区与暂存区内容不一致），gitleaks 策略只认暂存区版本，已阻止提交。" >&2 + echo " 请先 git add $f 或 git restore $f 后重新提交。" >&2 + exit 1 + fi + elif [ -e "$repo_root/$f" ] || [ -L "$repo_root/$f" ]; then + rm -rf "$tmpd" + echo "[PRE-COMMIT ERROR] $f 未跟踪（不在暂存区），会被 gitleaks 读取却不进入提交，已阻止提交。" >&2 + echo " 请先 git add $f 或删除该文件后重新提交。" >&2 + exit 1 + fi + done + rm -f "$tmpd/tag" "$tmpd/index-copy" + if git -C "$repo_root" cat-file -e ":.gitleaks.toml" 2>/dev/null; then + git -C "$repo_root" show ":.gitleaks.toml" >"$tmpd/gitleaks.toml" || { rm -rf "$tmpd"; echo "[PRE-COMMIT ERROR] 无法读取暂存区的 .gitleaks.toml" >&2; exit 1; } + # review3h P3-a：与 CI 相同，用 gitleaks-config-check.py 检查暂存区配置（[extend] path 会读取工作区文件，拒绝）。 + # 检查脚本同样取暂存区版本（工作区里未暂存的改动不能让检查放水）；缺脚本、缺 python3 / tomllib（3.11+）均拒绝。 + if ! git -C "$repo_root" show ":.github/scripts/gitleaks-config-check.py" >"$tmpd/gitleaks-config-check.py" 2>/dev/null; then + rm -rf "$tmpd" + echo "[PRE-COMMIT ERROR] 暂存区缺少 .github/scripts/gitleaks-config-check.py，无法检查 .gitleaks.toml，已阻止提交。" >&2 + exit 1 + fi + if ! command -v python3 >/dev/null 2>&1; then + rm -rf "$tmpd" + echo "[PRE-COMMIT ERROR] 缺少 python3（需要 3.11+ 的 tomllib）检查 .gitleaks.toml，已阻止提交。" >&2 + exit 1 + fi + if ! python3 "$tmpd/gitleaks-config-check.py" "$tmpd/gitleaks.toml"; then + rm -rf "$tmpd" + echo "[PRE-COMMIT ERROR] 暂存区的 .gitleaks.toml 未通过 gitleaks-config-check.py 检查（见上一行），已阻止提交。" >&2 + exit 1 + fi + else + # 暂存区没有配置：显式使用默认规则（不受 GITLEAKS_CONFIG 等环境变量影响） + printf '[extend]\nuseDefault = true\n' >"$tmpd/gitleaks.toml" + fi + if git -C "$repo_root" cat-file -e ":.gitleaksignore" 2>/dev/null; then + git -C "$repo_root" show ":.gitleaksignore" >"$tmpd/gitleaksignore" || { rm -rf "$tmpd"; echo "[PRE-COMMIT ERROR] 无法读取暂存区的 .gitleaksignore" >&2; exit 1; } + else + : >"$tmpd/gitleaksignore" + fi + # [extend] path 已由上面的 gitleaks-config-check.py 拒绝（review3h P3-a；此前为残余风险）。 + # --ignore-gitleaks-allow：源码里的 "gitleaks:allow" 行内注释不再豁免任何发现（任何人都能在密钥行后加这条注释）； + # 误报只能在 .gitleaks.toml 中按值精确放行（SEC-01A 第三轮 P3-1）。 + # 已保证 >= 8.25.0，直接使用 `gitleaks git --staged`（不再回退到旧的 `gitleaks protect --staged`，review3g N6）。 + gitleaks git --pre-commit --staged --redact --no-banner --verbose --ignore-gitleaks-allow \ + --config "$tmpd/gitleaks.toml" --gitleaks-ignore-path "$tmpd/gitleaksignore" "$repo_root" + local rc=$? + rm -rf "$tmpd" + if [ $rc -ne 0 ]; then + echo "" >&2 + echo "[PRE-COMMIT ERROR] gitleaks 在暂存内容中发现疑似密钥（以上输出已脱敏），提交已被阻止。" >&2 + echo " 请移除密钥并改为环境变量/密钥管理；若确属误报，请在 .gitleaks.toml 中添加带注释的窄范围 allowlist。" >&2 + exit 1 + fi + echo "[PRE-COMMIT] gitleaks: 暂存内容未发现密钥" +} +run_gitleaks_staged + +# 记录 gitleaks 扫描时的暂存区快照。后续步骤不得修改暂存区（否则新暂存的内容未经扫描）。 +INDEX_TREE_SCANNED="$(git write-tree)" || error_exit "无法读取暂存区（git write-tree 失败）" + # 检查是否有Go文件变更 if git diff --cached --quiet -- '*.go'; then warning_msg "未检测到 Go 文件变更，跳过Go相关检查" @@ -70,20 +201,24 @@ else success_msg "go generate 执行成功" fi -# 2. 代码格式化检查和自动修复 -info_msg "步骤 2/4: 检查代码格式..." - -# 查找所有需要格式化的Go文件 -UNFORMATTED_FILES=$(go fmt ./...) -if [ -n "$UNFORMATTED_FILES" ]; then - warning_msg "以下文件已自动格式化:" - echo "$UNFORMATTED_FILES" - - # 将格式化后的文件添加到暂存区 - for file in $UNFORMATTED_FILES; do - git add "$file" - done - warning_msg "格式化后的文件已自动添加到暂存区" +# 2. 代码格式检查（只检查，不修改、不暂存） +# 只检查“暂存的” .go 文件的“暂存版本”内容（git cat-file blob :path），与 gitleaks 扫描的内容一致。 +# 旧实现在 gitleaks 之后执行 `go fmt ./...` 并 `git add`，会把工作区中未暂存的改动（甚至从未暂存过的 +# .go 文件）连同其中未经扫描的密钥一起加入提交（SEC-01A 审查 G1/G2），已移除。 +info_msg "步骤 2/4: 检查暂存的 Go 文件格式 (gofmt)..." +command -v gofmt >/dev/null 2>&1 || error_exit "未找到 gofmt，无法检查代码格式" +UNFORMATTED_FILES=() +while IFS= read -r -d '' file; do + if [ -n "$(git cat-file blob ":$file" | gofmt -l 2>&1)" ]; then + UNFORMATTED_FILES+=("$file") + fi +done < <(git diff --cached --name-only --diff-filter=ACMR -z -- '*.go') +if [ ${#UNFORMATTED_FILES[@]} -gt 0 ]; then + echo -e "${RED}[PRE-COMMIT ERROR]${NC} 以下暂存的 Go 文件未按 gofmt 格式化（检查的是暂存版本）:" + printf ' %s\n' "${UNFORMATTED_FILES[@]}" + echo "请自行格式化并重新暂存后再提交，例如:" + echo " gofmt -w <文件> && git add -p <文件> # 注意只暂存你确认要提交的内容" + exit 1 fi success_msg "代码格式检查完成" @@ -124,5 +259,10 @@ if [ -n "$GENERATED_FILES" ]; then warning_msg "请检查这些文件是否应该被提交，如需要请手动 git add" fi +# 防御性检查：gitleaks 扫描之后暂存区不得被任何步骤修改（如 go generate 的副作用），否则拒绝提交。 +if [ "$(git write-tree)" != "$INDEX_TREE_SCANNED" ]; then + error_exit "gitleaks 扫描后暂存区发生了变化，新暂存的内容未经密钥扫描，已阻止提交。请检查后重新 git commit。" +fi + success_msg "所有检查通过，准备提交" exit 0 \ No newline at end of file diff --git a/.github/scripts/gitleaks-base-config.sh b/.github/scripts/gitleaks-base-config.sh new file mode 100755 index 00000000..e7465a2d --- /dev/null +++ b/.github/scripts/gitleaks-base-config.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# 从“基准提交”取出 gitleaks 配置与指纹忽略列表，供 CI 扫描使用（SEC-01A 第三轮 P3-1）。 +# +# 用法: bash .github/scripts/gitleaks-base-config.sh <扫描范围> <输出目录> +# <扫描范围> 为 gitleaks-range.sh 的输出（"A..B" 或 "B"）。 +# 输出: <输出目录>/gitleaks.toml、<输出目录>/gitleaksignore（总是生成，供 --config / --gitleaks-ignore-path 使用） +# 环境变量（与 gitleaks-range.sh 相同，来自 GitHub 事件上下文，必填 EVENT_NAME / DEFAULT_BRANCH）： +# EVENT_NAME、DEFAULT_BRANCH、REF_NAME、REF_TYPE（push 必填，只接受 branch / tag）、PR_BASE_REF（github.base_ref）、REMOTE（默认 origin） +# +# 策略来源（取哪个提交上的 .gitleaks.toml / .gitleaksignore；review3g N2：扫描起点 ≠ 策略来源）： +# push 到默认分支 → 范围起点 A（默认分支上一次的状态）；"B"（全量）→ 默认规则、空忽略列表 +# push 到其他分支 / tag → /<默认分支> 的最新提交（不用 before：before 是同一分支上一次推送， +# 攻击者可以先推一个全部放行的 .gitleaks.toml，再推密钥） +# pull_request 目标为默认分支 → 范围起点 A（PR base sha，在默认分支上） +# pull_request 目标为其他分支 / 未知 → /<默认分支> 的最新提交（同上，非默认分支不受保护） +# 缺少 EVENT_NAME / DEFAULT_BRANCH，或需要 /<默认分支> 但不存在 → 失败（不回退到范围起点） +# +# 为什么：PR 可以同时提交密钥并修改 .gitleaks.toml / .gitleaksignore 放行它。CI 只信任基准提交上的这两个文件： +# - 策略提交上没有该文件时使用默认配置（[extend] useDefault = true）/ 空忽略列表，绝不回退到 HEAD 的文件； +# - gitleaks 总会额外读取“扫描源目录/.gitleaksignore”（即 HEAD 工作区中的文件，8.30.1 实测）， +# 因此这里把工作区中的 .gitleaksignore 移走（CI 检出目录是一次性的；该文件的内容仍会被提交扫描覆盖）； +# - 基准配置中的 [extend] path 会从当前工作区（HEAD）读取文件，视为不可信，直接失败； +# 检查由 gitleaks-config-check.py（tomllib，Python 3.11+）完成，大小写/点号/内联表等写法都算，缺 tomllib 也失败。 +# 局限：本脚本与工作流本身仍来自 PR 的 HEAD，修改 .github/ 的 PR 可以绕过——需要 CODEOWNERS/分支保护（见 README-ops 待办）。 +set -euo pipefail + +die() { echo "[gitleaks-base-config] ERROR: $*" >&2; exit 1; } +note() { echo "[gitleaks-base-config] $*" >&2; } + +[ "$#" -eq 2 ] || die "用法: $0 <扫描范围> <输出目录>" +range="$1"; out="$2" +[ -n "$range" ] || die "扫描范围为空" +REMOTE="${REMOTE:-origin}" +[ -n "${EVENT_NAME:-}" ] || die "缺少 EVENT_NAME，无法确定策略来源（不回退到扫描范围起点）" +[ -n "${DEFAULT_BRANCH:-}" ] || die "缺少 DEFAULT_BRANCH，无法确定策略来源" +mkdir -p "$out" +cfg="$out/gitleaks.toml"; ign="$out/gitleaksignore" + +start="" +case "$range" in + *...*) die "不支持的范围格式: $range" ;; + *..*) start="${range%%..*}" ;; +esac +if [ -n "$start" ]; then + git rev-parse --verify --quiet "${start}^{commit}" >/dev/null || die "范围起点提交不存在: $start" +fi + +# 策略提交 base：只取默认分支上的提交（见文件头） +use_default_tip=0 +case "$EVENT_NAME" in + push) + # review3h P3-i：REF_TYPE 只接受 branch / tag（区分大小写），空值或其他写法失败，不再默认当作 branch + case "${REF_TYPE:-}" in + branch|tag) ;; + *) die "REF_TYPE=${REF_TYPE:-<空>} 无效（只接受 branch / tag），无法确定策略来源" ;; + esac + if [ "$REF_TYPE" = "branch" ] && [ "${REF_NAME:-}" = "$DEFAULT_BRANCH" ]; then + base="$start" + else + use_default_tip=1 + fi ;; + pull_request|pull_request_target) + if [ "${PR_BASE_REF:-}" = "$DEFAULT_BRANCH" ]; then base="$start"; else use_default_tip=1; fi ;; + *) die "不支持的事件类型: $EVENT_NAME" ;; +esac +if [ "$use_default_tip" -eq 1 ]; then + default_ref="refs/remotes/${REMOTE}/${DEFAULT_BRANCH}" + base="$(git rev-parse --verify -q "${default_ref}^{commit}")" \ + || die "推送/PR 目标不是默认分支，策略须取自 ${REMOTE}/${DEFAULT_BRANCH}，但本地没有该引用（actions/checkout 需 fetch-depth: 0）" + note "目标不是默认分支（${REF_TYPE:-?} ${REF_NAME:-?}${PR_BASE_REF:+，PR 目标 $PR_BASE_REF}）：策略取自 ${REMOTE}/${DEFAULT_BRANCH}（${base}），不用扫描起点 ${start:-<无>}" +else + note "策略取自默认分支上的范围起点 ${base:-（无，全量扫描）}" +fi + +if [ -n "$base" ] && git cat-file -e "${base}:.gitleaks.toml" 2>/dev/null; then + git show "${base}:.gitleaks.toml" >"$cfg" + note "使用基准 ${base} 上的 .gitleaks.toml" + # [extend] path 会从工作区（HEAD）读取，不可信；用 TOML 解析器按 gitleaks（viper）的规则检查（review3g N1） + command -v python3 >/dev/null 2>&1 || die "缺少 python3（需要 3.11+ 的 tomllib）检查基准 .gitleaks.toml 的 [extend]，拒绝继续" + python3 "$(dirname "$0")/gitleaks-config-check.py" "$cfg" \ + || die "基准 .gitleaks.toml 未通过 [extend] 检查（见上一行），拒绝使用" +else + printf '# 基准提交上没有 .gitleaks.toml：使用 gitleaks 默认规则（不使用 HEAD 的配置）\n[extend]\nuseDefault = true\n' >"$cfg" + note "基准 ${base:-（无，全量扫描）} 上没有 .gitleaks.toml，使用 gitleaks 默认规则" +fi + +if [ -n "$base" ] && git cat-file -e "${base}:.gitleaksignore" 2>/dev/null; then + git show "${base}:.gitleaksignore" >"$ign" + note "使用基准 ${base} 上的 .gitleaksignore" +else + : >"$ign" + note "基准 ${base:-（无，全量扫描）} 上没有 .gitleaksignore，使用空忽略列表" +fi + +if [ -e .gitleaksignore ]; then + mv .gitleaksignore "$out/head.gitleaksignore" + note "已移走工作区中的 .gitleaksignore（HEAD 版本，不作为忽略列表使用）" +fi diff --git a/.github/scripts/gitleaks-config-check.py b/.github/scripts/gitleaks-config-check.py new file mode 100755 index 00000000..60c80e57 --- /dev/null +++ b/.github/scripts/gitleaks-config-check.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""检查基准 .gitleaks.toml 是否使用了 [extend] path（review3g N1）。 + +gitleaks 用 viper 读配置：键名不区分大小写，带点的键（"extend.path"、["extend.path"]）按 "." 拆成层级， +\\u 转义在 TOML 层就被还原。因此用 TOML 解析器（tomllib，Python 3.11+）读出完整结构，把所有键按 viper +的方式（小写、按 "." 拆分）展开成路径，只要有路径以 extend.path 开头就拒绝；同时拒绝： + - 多个大小写不同的 extend 写法（viper 合并行为未定义）； + - extend 不是表（字符串/数组表等）； + - TOML 解析失败； + - 没有 tomllib（Python < 3.11）。 +退出码：0 = 通过；1 = 拒绝/出错（原因写到 stderr）。 +用法：python3 gitleaks-config-check.py <配置文件> +""" +import sys + + +def fail(msg): + sys.stderr.write("[gitleaks-config-check] ERROR: %s\n" % msg) + sys.exit(1) + + +def walk(node, prefix, out): + if isinstance(node, dict): + for k, v in node.items(): + parts = prefix + [p.lower() for p in str(k).split(".")] + out.append(parts) + walk(v, parts, out) + elif isinstance(node, list): + for v in node: + walk(v, prefix, out) + + +def main(argv): + if len(argv) != 2: + fail("用法: gitleaks-config-check.py <配置文件>") + try: + import tomllib + except ImportError: + fail("缺少 tomllib（需要 Python 3.11+），无法可靠检查 [extend]，拒绝使用该配置") + try: + with open(argv[1], "rb") as fh: + data = tomllib.load(fh) + except Exception as exc: # 解析失败一律拒绝 + fail("配置无法按 TOML 解析（%s），拒绝使用" % exc.__class__.__name__) + + ext_keys = [k for k in data if str(k).split(".")[0].lower() == "extend"] + if len(ext_keys) > 1: + fail("配置中有多个 extend 写法（%d 个，大小写/点号不同），拒绝使用" % len(ext_keys)) + for k in ext_keys: + if "." not in str(k) and not isinstance(data[k], dict): + fail("配置中的 extend 不是表（%s），拒绝使用" % type(data[k]).__name__) + + paths = [] + walk(data, [], paths) + for p in paths: + if len(p) >= 2 and p[0] == "extend" and p[1] == "path": + fail("配置的 [extend] 使用了 path（会读取 HEAD 工作区中的文件），拒绝使用") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.github/scripts/gitleaks-range.sh b/.github/scripts/gitleaks-range.sh new file mode 100755 index 00000000..98c76f75 --- /dev/null +++ b/.github/scripts/gitleaks-range.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# 计算 CI 中 gitleaks 要扫描的提交范围（SEC-01A 审查 P2-3），供 .github/workflows/gitleaks.yml 调用。 +# stdout 只输出一个 git log 范围参数；说明信息写 stderr；无法确定完整范围时 exit 1（宁可失败，不可漏扫）。 +# +# 输入（环境变量，均来自 GitHub 事件上下文）： +# EVENT_NAME github.event_name +# PUSH_BEFORE github.event.before（push） +# PR_BASE_SHA github.event.pull_request.base.sha（pull_request） +# HEAD_SHA github.sha +# DEFAULT_BRANCH github.event.repository.default_branch +# REF_NAME github.ref_name +# REF_TYPE github.ref_type（非 PR 事件必填，只接受 branch / tag；空值、Tag 等其他写法一律失败，review3h P3-i） +# REMOTE 远端名，默认 origin（actions/checkout 使用 origin，且需 fetch-depth: 0） +# +# 规则： +# pull_request -> ..；base 不在历史中则失败 +# push，before 在历史中 -> ..（force-push 但旧提交仍可达时同样适用） +# push，before 为全零或不在历史中（首推 / force-push 后旧提交不可达）： +# - 推的是默认分支本身 -> 全量扫描 HEAD 可达的全部历史（无更早的基线可比较） +# - 其他分支 / tag -> HEAD>.. +# - 缺少默认分支信息、没有 origin/、无共同祖先、浅克隆 -> 失败 +# 注意：本脚本只决定“扫描哪些提交”。范围起点（如 push 的 before）不是可信的策略来源—— +# gitleaks 配置 / 忽略列表由 gitleaks-base-config.sh 另行从默认分支选取（review3g N2）。 +set -euo pipefail + +ZERO=0000000000000000000000000000000000000000 +REMOTE="${REMOTE:-origin}" +die() { echo "[gitleaks-range] ERROR: $*" >&2; exit 1; } +note() { echo "[gitleaks-range] $*" >&2; } +is_commit() { [ -n "${1:-}" ] && [ "$1" != "$ZERO" ] && git cat-file -e "$1^{commit}" 2>/dev/null; } + +[ -n "${HEAD_SHA:-}" ] || die "HEAD_SHA 为空" +is_commit "$HEAD_SHA" || die "HEAD_SHA=$HEAD_SHA 不在本地历史中" +[ "$(git rev-parse --is-shallow-repository)" = "false" ] || die "浅克隆无法确定完整范围（actions/checkout 需设置 fetch-depth: 0）" + +case "${EVENT_NAME:-}" in + pull_request|pull_request_target) + is_commit "${PR_BASE_SHA:-}" || die "PR base sha（${PR_BASE_SHA:-<空>}）不在历史中，无法确定扫描范围" + echo "${PR_BASE_SHA}..${HEAD_SHA}" + exit 0 ;; +esac + +# review3h P3-i：非 PR 事件按 push 处理，REF_TYPE 只接受 branch / tag（区分大小写）； +# 以前的 ${REF_TYPE:-branch} 会把空值当 branch，把 "Tag" 之类的写法当成分支，现在一律失败 +case "${REF_TYPE:-}" in + branch|tag) ;; + *) die "REF_TYPE=${REF_TYPE:-<空>} 无效（只接受 branch / tag），无法确定扫描范围" ;; +esac + +if is_commit "${PUSH_BEFORE:-}"; then + echo "${PUSH_BEFORE}..${HEAD_SHA}" + exit 0 +fi +note "before=${PUSH_BEFORE:-<空>} 为全零或不在历史中（首推或 force-push），改用默认分支确定范围" + +[ -n "${DEFAULT_BRANCH:-}" ] || die "缺少默认分支信息（DEFAULT_BRANCH 为空），无法确定扫描范围" + +if [ "$REF_TYPE" = "branch" ] && [ "${REF_NAME:-}" = "$DEFAULT_BRANCH" ]; then + note "推送的是默认分支 ${DEFAULT_BRANCH} 本身且无可用 before：全量扫描 HEAD 可达的全部历史" + echo "${HEAD_SHA}" + exit 0 +fi + +default_ref="refs/remotes/${REMOTE}/${DEFAULT_BRANCH}" +git rev-parse --verify -q "${default_ref}^{commit}" >/dev/null \ + || die "缺少 ${REMOTE}/${DEFAULT_BRANCH}，无法确定扫描范围（actions/checkout 需设置 fetch-depth: 0）" +base="$(git merge-base "$default_ref" "$HEAD_SHA" 2>/dev/null)" \ + || die "${HEAD_SHA} 与 ${REMOTE}/${DEFAULT_BRANCH} 没有共同祖先，无法确定扫描范围" +[ "$base" = "$HEAD_SHA" ] && note "HEAD 已包含在 ${REMOTE}/${DEFAULT_BRANCH} 中，没有新增提交" +echo "${base}..${HEAD_SHA}" diff --git a/.github/tests/lib-gitleaks-workflow.sh b/.github/tests/lib-gitleaks-workflow.sh new file mode 100755 index 00000000..78fe1425 --- /dev/null +++ b/.github/tests/lib-gitleaks-workflow.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# .github/workflows/gitleaks.yml 回归测试的公共函数（SEC-01A 审查 P2/P3）。由同目录的 test-*.sh source。 +# 所有场景在 mktemp -d 的临时仓库中运行；密钥为运行时拼出的合成值，不是真实凭据。 +# 兼容 macOS 自带 bash 3.2 与 BSD 工具：不用 \s / grep -P / sed -i / readlink -f / mapfile。 +set -euo pipefail + +GL_REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +GL_REV="" +GL_WORKFLOW=".github/workflows/gitleaks.yml" +GL_STEP="scan" +gl_parse_args() { + while [ $# -gt 0 ]; do + case "$1" in + --rev) GL_REV="$2"; shift 2 ;; + *) echo "未知参数: $1（用法: $0 [--rev ]，默认测试当前工作区文件）" >&2; exit 2 ;; + esac + done +} +export PATH="$HOME/.local/bin:$PATH" +GL_TMP="$(mktemp -d)" +trap 'rm -rf "$GL_TMP"' EXIT +export GIT_CONFIG_GLOBAL="$GL_TMP/gitconfig" GIT_CONFIG_NOSYSTEM=1 +git config --global user.name test +git config --global user.email test@example.invalid +git config --global init.defaultBranch main +git config --global advice.detachedHead false +GL_FAILS=0 + +# 读取目标版本（--rev 或工作区）中的文件 +gl_src() { if [ -n "$GL_REV" ]; then git -C "$GL_REPO" show "$GL_REV:$1"; else cat "$GL_REPO/$1"; fi; } +gl_has() { if [ -n "$GL_REV" ]; then git -C "$GL_REPO" cat-file -e "$GL_REV:$1" 2>/dev/null; else [ -e "$GL_REPO/$1" ]; fi; } + +# 合成密钥：运行时拼接，源码中不出现完整 token +gl_rand() { LC_ALL=C tr -dc "$1" "；按缩进解析，不依赖 PyYAML） +gl_extract_run() { + gl_src "$GL_WORKFLOW" | python3 -c ' +import re, sys +name = sys.argv[1]; lines = sys.stdin.read().splitlines() +i = next(k for k, l in enumerate(lines) if re.match(r"\s*-?\s*name:\s*" + re.escape(name) + r"\s*$", l)) +step_indent = len(lines[i]) - len(lines[i].lstrip(" -")) +for k in range(i + 1, len(lines)): + m = re.match(r"(\s*)run:\s*\|\s*$", lines[k]) + if m: + ind = len(m.group(1)); out = [] + for l in lines[k + 1:]: + if l.strip() and len(l) - len(l.lstrip()) <= ind: break + out.append(l) + body_ind = min(len(l) - len(l.lstrip()) for l in out if l.strip()) + print("\n".join(l[body_ind:] for l in out)); sys.exit(0) + m = re.match(r"\s*run:\s*(\S.*)$", lines[k]) + if m: + print(m.group(1)); sys.exit(0) + if re.match(r"\s*-\s", lines[k]) and len(lines[k]) - len(lines[k].lstrip(" -")) <= step_indent: break +sys.exit("run block not found for step: " + name) +' "$2" >"$1" +} + +# 初始化"远端"裸仓库 + 开发者仓库（含目标版本的 .gitleaks.toml） +gl_new_origin() { + local name="$1" + GL_ORIGIN="$GL_TMP/$name-origin.git"; GL_DEV="$GL_TMP/$name-dev" + git init -q --bare "$GL_ORIGIN" + git init -q "$GL_DEV"; git -C "$GL_DEV" remote add origin "$GL_ORIGIN" + if gl_has .gitleaks.toml; then gl_src .gitleaks.toml >"$GL_DEV/.gitleaks.toml"; else : >"$GL_DEV/.gitleaks.toml"; fi + # review3h P3-a 起 pre-commit 用暂存区的 gitleaks-config-check.py 检查 .gitleaks.toml，缺失即拒绝； + # 夹具仓库与真实仓库一致，带上目标版本的检查器（目标版本没有时不放，旧版本钩子也不需要） + if gl_has .github/scripts/gitleaks-config-check.py; then + mkdir -p "$GL_DEV/.github/scripts"; gl_src .github/scripts/gitleaks-config-check.py >"$GL_DEV/.github/scripts/gitleaks-config-check.py" + fi +} +gl_commit() { # gl_commit + mkdir -p "$(dirname "$GL_DEV/$1")" + printf '%s\n' "$2" >"$GL_DEV/$1"; git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm "$3" +} + +# 模拟 actions/checkout(fetch-depth: 0)：fetch refs/heads/* 到 origin/*，不设置 origin/HEAD，detached 到 HEAD_SHA +gl_checkout() { # gl_checkout [extra refspec] + GL_WS="$GL_TMP/ws-$RANDOM$RANDOM"; git init -q "$GL_WS" + git -C "$GL_WS" remote add origin "$GL_ORIGIN" + git -C "$GL_WS" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' ${2:-} + git -C "$GL_WS" checkout -q --detach "$1" + # 目标版本的辅助脚本（如 .github/scripts/）放到工作区（未跟踪，不影响 gitleaks git 扫描的提交） + if gl_has .github/scripts; then + mkdir -p "$GL_WS/.github" + if [ -n "$GL_REV" ]; then git -C "$GL_REPO" archive "$GL_REV" .github/scripts | tar -x -C "$GL_WS" + else cp -R "$GL_REPO/.github/scripts" "$GL_WS/.github/"; fi + fi +} + +# 选择被判定的步骤：scan = "Scan new commits"；tree = 先跑 "Scan new commits"（准备 base 配置，结果不计）， +# 再跑并判定 "Scan working tree (HEAD)" +gl_step() { GL_STEP="$1"; } + +# 运行目标版本的步骤（与 GitHub 的 bash --noprofile --norc -eo pipefail 一致）；每次使用新的 RUNNER_TEMP +# 用法: gl_run_scan EVENT_NAME=push PUSH_BEFORE=... HEAD_SHA=... DEFAULT_BRANCH=main REF_NAME=main +gl_run_scan() { + local f="$GL_TMP/scan-step.sh" t="$GL_TMP/tree-step.sh" rc=0 + GL_RT="$GL_TMP/rt-$RANDOM$RANDOM"; mkdir -p "$GL_RT"; rm -f "$GL_TMP/scan-commits.out" + gl_extract_run "$f" "Scan new commits" + set +e + ( cd "$GL_WS" && env -u PR_BASE_SHA -u GITLEAKS_CONFIG -u GITLEAKS_CONFIG_TOML RUNNER_TEMP="$GL_RT" \ + EVENT_NAME= PUSH_BEFORE= PR_BASE_SHA= PR_BASE_REF= HEAD_SHA= DEFAULT_BRANCH= REF_NAME= REF_TYPE= "$@" \ + bash --noprofile --norc -eo pipefail "$f" ) >"$GL_TMP/scan.out" 2>&1 + rc=$? + if [ "$GL_STEP" = "tree" ]; then + mv "$GL_TMP/scan.out" "$GL_TMP/scan-commits.out"; rm -f "$GL_RT"/gitleaks-commits*.json + gl_extract_run "$t" "Scan working tree (HEAD)" + ( cd "$GL_WS" && env -u GITLEAKS_CONFIG -u GITLEAKS_CONFIG_TOML RUNNER_TEMP="$GL_RT" "$@" \ + bash --noprofile --norc -eo pipefail "$t" ) >"$GL_TMP/scan.out" 2>&1 + rc=$? + fi + set -e + return "$rc" +} + +# 列出本次运行检出的发现（每行 "RuleIDFile"）：优先读步骤写出的 JSON 报告（$RUNNER_TEMP/gitleaks-*.json）， +# 目标版本没有报告文件时（旧版本工作流）回退解析 --verbose 输出中的 RuleID/File 字段 +gl_findings() { + local reports; reports="$(ls "$GL_RT"/gitleaks-*.json 2>/dev/null || true)" + if [ -n "$reports" ]; then + python3 -c ' +import json, sys +for p in sys.argv[1:]: + for f in json.load(open(p)) or []: + print("%s\t%s" % (f.get("RuleID", ""), f.get("File", ""))) +' $reports + else + awk '$1 == "RuleID:" { r = $2 } $1 == "File:" { print r "\t" $2 }' "$GL_TMP/scan.out" + fi +} + +# 期望：步骤失败，且失败原因与预期一致 +# gl_expect_block <名称> leak:: 检出密钥（报告中须有该 RuleID+File） +# gl_expect_block <名称> error: 因错误失败（无任何检出，输出匹配 ERE） +gl_expect_block() { + local name="$1" want="$2" rc=0; shift 2 + gl_run_scan "$@" || rc=$? + local range; range="$(cat "$GL_TMP/scan-commits.out" "$GL_TMP/scan.out" 2>/dev/null | grep -m1 'gitleaks scan range' || true)" + local found; found="$(gl_findings | sort -u)" + local summary; summary="$(printf '%s' "$found" | tr '\t\n' ': ' )" + local err; err="$(grep -m1 -E 'ERROR|ERR |无法|FTL' "$GL_TMP/scan.out" || true)" + if [ "$rc" -eq 0 ]; then + echo "FAIL [$name] 漏扫：步骤 rc=0，未拦下 | ${range:-<无范围输出>}"; GL_FAILS=$((GL_FAILS+1)); return 0 + fi + case "$want" in + leak:*) + local rule file; rule="${want#leak:}"; file="${rule#*:}"; rule="${rule%%:*}" + if printf '%s\n' "$found" | grep -qxF "$(printf '%s\t%s' "$rule" "$file")"; then + echo "PASS [$name] 检出 ${rule} @ ${file}（rc=${rc}；全部发现: ${summary}）${range:+ | $range}" + elif [ -z "$found" ]; then + echo "FAIL [$name] 步骤失败（rc=${rc}）但不是检出密钥，而是出错: ${err:-<无错误信息>}"; GL_FAILS=$((GL_FAILS+1)) + else + echo "FAIL [$name] 检出内容与预期不符（期望 ${rule} @ ${file}，实际: ${summary}）"; GL_FAILS=$((GL_FAILS+1)) + fi ;; + error:*) + local pat="${want#error:}" + if [ -n "$found" ]; then + echo "FAIL [$name] 期望因错误失败，实际检出密钥: ${summary}"; GL_FAILS=$((GL_FAILS+1)) + elif grep -qE "$pat" "$GL_TMP/scan.out"; then + echo "PASS [$name] 按预期出错失败（rc=${rc}）: $(grep -m1 -E "$pat" "$GL_TMP/scan.out")" + else + echo "FAIL [$name] 失败原因不符（期望匹配 /${pat}/）: ${err:-$(tail -1 "$GL_TMP/scan.out")}"; GL_FAILS=$((GL_FAILS+1)) + fi ;; + *) echo "测试用例写法错误: 期望值须为 leak:: 或 error:（${want}）" >&2; exit 2 ;; + esac +} +gl_expect_pass() { # 对照：无密钥时步骤应成功 + local name="$1" rc=0; shift + gl_run_scan "$@" || rc=$? + if [ "$rc" -eq 0 ]; then echo "PASS [$name] 无密钥，步骤成功（rc=0）" + else echo "FAIL [$name] 无密钥但步骤失败（rc=${rc}）"; tail -5 "$GL_TMP/scan.out" | sed 's/^/ /'; GL_FAILS=$((GL_FAILS+1)); fi +} +gl_finish() { echo "目标版本: ${GL_REV:-工作区}；失败 $GL_FAILS 项"; [ "$GL_FAILS" -eq 0 ]; } diff --git a/.github/tests/test-gitleaks-actions-pinned.sh b/.github/tests/test-gitleaks-actions-pinned.sh new file mode 100755 index 00000000..bc725f6b --- /dev/null +++ b/.github/tests/test-gitleaks-actions-pinned.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# SEC-01A 审查 P3：gitleaks 工作流中所有 action 必须固定到完整 commit SHA（注释写版本号）， +# actions/checkout 必须设置 persist-credentials: false（不把 GITHUB_TOKEN 留在 .git/config 里）。 +# review3h P3-i：必须有 actions/setup-python（python-version >= 3.11）位于所有 run 步骤之前。 +# 用法: .github/tests/test-gitleaks-actions-pinned.sh [--rev ] [--verify-remote] +# --verify-remote 额外用 git ls-remote 核对注释中的版本 tag 确实指向该 SHA（需要网络） +# grep/sed 只用 POSIX 字符类 [[:space:]]：BSD grep/sed（macOS）不认识反斜杠 s 简写。 +VERIFY_REMOTE=0; ARGS=() +for a in "$@"; do if [ "$a" = --verify-remote ]; then VERIFY_REMOTE=1; else ARGS+=("$a"); fi; done +source "$(dirname "${BASH_SOURCE[0]}")/lib-gitleaks-workflow.sh" +gl_parse_args "${ARGS[@]+"${ARGS[@]}"}" + +gl_src "$GL_WORKFLOW" >"$GL_TMP/wf.yml" +# 不用进程替换 "< <(...)"：macOS bash 3.2 的 --posix 模式下是语法错误（且脚本以 rc=0、0 个 PASS 结束，review3h 遗留项） +grep -E '^[[:space:]]*-?[[:space:]]*uses:' "$GL_TMP/wf.yml" >"$GL_TMP/uses.txt" || true +[ -s "$GL_TMP/uses.txt" ] || { echo "FAIL [uses] 工作流中没有找到任何 uses: 行"; GL_FAILS=$((GL_FAILS+1)); } +while IFS= read -r line; do + spec="$(sed -E 's/^[[:space:]]*-?[[:space:]]*uses:[[:space:]]*([^ #]+).*/\1/' <<<"$line")" + action="${spec%@*}"; ref="${spec#*@}"; tag="$(grep -oE '#[[:space:]]*v[0-9][^ ]*' <<<"$line" | tr -d '# ' || true)" + if [[ "$ref" =~ ^[0-9a-f]{40}$ ]]; then + echo "PASS [uses] ${action}@${ref} ${tag:+(${tag})}" + if [ "$VERIFY_REMOTE" = 1 ]; then + [ -n "$tag" ] || { echo "FAIL [verify] ${action} 缺少版本注释"; GL_FAILS=$((GL_FAILS+1)); continue; } + remote="$(git ls-remote "https://github.com/${action}" "refs/tags/${tag}" "refs/tags/${tag}^{}" | awk '{print $1}' | tail -1)" + if [ "$remote" = "$ref" ]; then echo "PASS [verify] ${action} ${tag} -> ${remote}" + else echo "FAIL [verify] ${action} ${tag} -> ${remote:-<不存在>}，与 ${ref} 不符"; GL_FAILS=$((GL_FAILS+1)); fi + fi + else + echo "FAIL [uses] ${action}@${ref} 未固定到完整 commit SHA"; GL_FAILS=$((GL_FAILS+1)) + fi +done <"$GL_TMP/uses.txt" + +# actions/checkout 步骤的 with: 中需要 persist-credentials: false +if python3 - "$GL_TMP/wf.yml" <<'PY'; then echo "PASS [checkout] persist-credentials: false"; else echo "FAIL [checkout] 未设置 persist-credentials: false"; GL_FAILS=$((GL_FAILS+1)); fi +import re, sys +lines = open(sys.argv[1], encoding="utf-8").read().splitlines() +for i, l in enumerate(lines): + if re.match(r"\s*-?\s*uses:\s*actions/checkout@", l): + ind = len(l) - len(l.lstrip(" -")) + for m in lines[i + 1:]: + if re.match(r"\s*-\s", m) and len(m) - len(m.lstrip(" -")) <= ind: break + if re.match(r"\s*persist-credentials:\s*false\s*(#.*)?$", m): sys.exit(0) + sys.exit(1) +sys.exit(1) +PY + +# review3h P3-i：gitleaks-config-check.py 需要 tomllib，工作流必须用 actions/setup-python 显式选择 Python 3.11+， +# 且该步骤位于 checkout 之后、第一个 run 步骤之前（不依赖 runner 镜像自带的 python3） +if python3 - "$GL_TMP/wf.yml" <<'PY'; then :; else echo "FAIL [setup-python] 缺少 actions/setup-python（python-version >= 3.11，且在所有 run 步骤之前）"; GL_FAILS=$((GL_FAILS+1)); fi +import re, sys +lines = open(sys.argv[1], encoding="utf-8").read().splitlines() +first_run = next((i for i, l in enumerate(lines) if re.match(r"\s*-?\s*run:", l)), len(lines)) +for i, l in enumerate(lines): + if re.match(r"\s*-?\s*uses:\s*actions/setup-python@", l): + if i > first_run: sys.exit(1) + ind = len(l) - len(l.lstrip(" -")) + for m in lines[i + 1:]: + if re.match(r"\s*-\s", m) and len(m) - len(m.lstrip(" -")) <= ind: break + v = re.match(r"""\s*python-version:\s*["']?(3)\.([0-9]+)["']?\s*(#.*)?$""", m) + if v and int(v.group(2)) >= 11: + print("PASS [setup-python] python-version %s.%s（>= 3.11，位于第一个 run 步骤之前）" % (v.group(1), v.group(2))); sys.exit(0) + sys.exit(1) +sys.exit(1) +PY +gl_finish diff --git a/.github/tests/test-gitleaks-allow-bypass.sh b/.github/tests/test-gitleaks-allow-bypass.sh new file mode 100755 index 00000000..cf8cc323 --- /dev/null +++ b/.github/tests/test-gitleaks-allow-bypass.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# SEC-01A 第三轮审查 P3-1：行内 "gitleaks:allow" 注释、以及 PR 自带的 .gitleaks.toml / .gitleaksignore 都不能放行密钥。 +# +# 15 pre-commit：暂存 k.env 中的合成 GitHub PAT 行尾带 "# gitleaks:allow" → 必须拦下 +# 15b pre-commit：app.js 中 "// gitleaks:allow" 变体 → 必须拦下 +# 16 CI Scan new commits：新提交中带 gitleaks:allow 的合成 PAT → 必须拦下 +# 17 CI Scan working tree：base 中已存在、带 gitleaks:allow 的合成 PAT（范围内无新密钥）→ 必须拦下 +# 18 CI：PR 同时提交密钥并把 .gitleaks.toml 改成全部放行 → 必须拦下（用 base 的配置） +# 18b CI：PR 同时提交密钥并在 .gitleaksignore 加入它的指纹 → 必须拦下（用 base 的忽略列表） +# 19 CI：base 上没有 .gitleaks.toml，PR 新增一个全部放行的 .gitleaks.toml → 必须拦下（用默认规则） +# U1/U2 gitleaks-base-config.sh：无基准时用默认规则；基准配置 [extend] path 拒绝（review3h：断言检查器的拒绝原因） +# 对照 U2c：基准配置合法时通过 +# 对照 C1/C2：base 上的 allowlist / .gitleaksignore 确实生效（同一合成值在 CI 中放行） +# +# 用法: .github/tests/test-gitleaks-allow-bypass.sh [--rev ] （对修复前版本运行应报 FAIL） +# 合成密钥均在运行时随机生成；所有仓库都在临时目录中创建。 +source "$(dirname "${BASH_SOURCE[0]}")/lib-gitleaks-workflow.sh" +gl_parse_args "$@" +export SKIP_VET=1 SKIP_BUILD=1 SKIP_GENERATE=1 + +ALLOW_ALL="$(printf '[extend]\nuseDefault = true\n\n[[allowlists]]\ndescription = "allow everything"\npaths = [%s]\n' "'''.*'''")" + +# ---------- pre-commit（目标版本的 .githooks/pre-commit） ---------- +hook_case() { # hook_case <名称> <文件> <内容> + local name="$1" f="$2" content="$3" rc=0 before after + gl_new_origin "hk$RANDOM" + mkdir -p "$GL_DEV/.githooks"; gl_src .githooks/pre-commit >"$GL_DEV/.githooks/pre-commit"; chmod +x "$GL_DEV/.githooks/pre-commit" + gl_commit README.md hello init + git -C "$GL_DEV" config core.hooksPath .githooks + before="$(git -C "$GL_DEV" rev-parse HEAD)" + mkdir -p "$(dirname "$GL_DEV/$f")"; printf '%s\n' "$content" >"$GL_DEV/$f"; git -C "$GL_DEV" add "$f" + git -C "$GL_DEV" commit -qm "$name" >"$GL_TMP/hook.out" 2>&1 || rc=$? + after="$(git -C "$GL_DEV" rev-parse HEAD)" + if [ "$rc" -ne 0 ] && [ "$after" = "$before" ] \ + && awk -v f="$f" '$1=="RuleID:"{r=$2} $1=="File:"{ if (r=="github-pat" && $2==f) ok=1 } END{exit !ok}' "$GL_TMP/hook.out"; then + echo "PASS [$name] pre-commit 拦下（github-pat @ ${f}）" + elif [ "$rc" -ne 0 ] && [ "$after" = "$before" ]; then + echo "FAIL [$name] 提交被拒但不是因为检出 github-pat @ ${f}: $(grep -m1 -E 'ERROR|RuleID' "$GL_TMP/hook.out")"; GL_FAILS=$((GL_FAILS+1)) + else + echo "FAIL [$name] pre-commit 放行（rc=${rc}），带 gitleaks:allow 的密钥进入提交"; GL_FAILS=$((GL_FAILS+1)) + fi +} +hook_case "15 pre-commit：# gitleaks:allow" k.env "GITHUB_TOKEN=$(gl_fake_ghp) # gitleaks:allow" +hook_case "15b pre-commit：// gitleaks:allow（JS）" web/app.js "const token = \"$(gl_fake_ghp)\"; // gitleaks:allow" + +# ---------- CI ---------- +# pr_case <名称> -- 之后为 PR 中的 文件 内容 对 +# PR 提交：feature 分支上一个提交，事件为 pull_request（PR_BASE_SHA = base） +setup_pr() { # setup_pr + gl_new_origin "$1" + if [ "$2" = no ]; then rm -f "$GL_DEV/.gitleaks.toml"; fi +} +base_push() { + git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm base + git -C "$GL_DEV" push -q origin HEAD:main; BASE_SHA="$(git -C "$GL_DEV" rev-parse HEAD)" + git -C "$GL_DEV" checkout -q -b feature +} +pr_push() { + git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm pr + git -C "$GL_DEV" push -q origin HEAD:feature; HEAD_SHA="$(git -C "$GL_DEV" rev-parse HEAD)" + gl_checkout "$HEAD_SHA" + PR_ARGS="EVENT_NAME=pull_request PR_BASE_SHA=$BASE_SHA PR_BASE_REF=main HEAD_SHA=$HEAD_SHA DEFAULT_BRANCH=main REF_NAME=1/merge REF_TYPE=branch" +} + +# 16：新提交中带 gitleaks:allow +setup_pr c16 yes; printf 'hello\n' >"$GL_DEV/README.md"; base_push +printf 'GITHUB_TOKEN=%s # gitleaks:allow\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env"; pr_push +gl_step scan; gl_expect_block "16 CI Scan new commits：gitleaks:allow 不豁免" leak:github-pat:k.env $PR_ARGS + +# 17：base 中已有带 gitleaks:allow 的密钥（绕过钩子提交），PR 只改 README +setup_pr c17 yes; printf 'GITHUB_TOKEN=%s # gitleaks:allow\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env"; base_push +printf 'hello 2\n' >"$GL_DEV/README.md"; pr_push +gl_step scan; gl_expect_pass "17 前提：范围内无新增密钥，Scan new commits 通过" $PR_ARGS +gl_step tree; gl_expect_block "17 CI Scan working tree：gitleaks:allow 不豁免" leak:github-pat:k.env $PR_ARGS + +# 18：PR 把 .gitleaks.toml 改为全部放行 +setup_pr c18 yes; printf 'hello\n' >"$GL_DEV/README.md"; base_push +printf '%s\n' "$ALLOW_ALL" >"$GL_DEV/.gitleaks.toml"; printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env"; pr_push +gl_step scan; gl_expect_block "18 CI：PR 修改 .gitleaks.toml 全部放行（范围扫描）" leak:github-pat:k.env $PR_ARGS +gl_step tree; gl_expect_block "18 CI：PR 修改 .gitleaks.toml 全部放行（工作树扫描）" leak:github-pat:k.env $PR_ARGS + +# 18b：PR 在 .gitleaksignore 加入密钥指纹（path:rule:line 对 git 与 dir 扫描都生效） +setup_pr c18b yes; printf 'hello\n' >"$GL_DEV/README.md"; base_push +printf 'k.env:github-pat:1\n' >"$GL_DEV/.gitleaksignore"; printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env"; pr_push +gl_step scan; gl_expect_block "18b CI：PR 新增 .gitleaksignore 指纹（范围扫描）" leak:github-pat:k.env $PR_ARGS +gl_step tree; gl_expect_block "18b CI：PR 新增 .gitleaksignore 指纹（工作树扫描）" leak:github-pat:k.env $PR_ARGS + +# 19：base 无 .gitleaks.toml，PR 新增全部放行的配置 +setup_pr c19 no; printf 'hello\n' >"$GL_DEV/README.md"; base_push +printf '%s\n' "$ALLOW_ALL" >"$GL_DEV/.gitleaks.toml"; printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env"; pr_push +gl_step scan; gl_expect_block "19 CI：base 无配置、PR 新增全部放行配置" leak:github-pat:k.env $PR_ARGS + +# 对照 C1：base 的 .gitleaks.toml 按值放行某个合成值 → PR 中出现该值应放行（证明确实加载了 base 的配置） +setup_pr cc1 yes; tok="$(gl_fake_ghp)" +printf '\n[[allowlists]]\ndescription = "test control"\nregexTarget = "secret"\nregexes = [%s]\n' "'''^${tok}\$'''" >>"$GL_DEV/.gitleaks.toml" +printf 'hello\n' >"$GL_DEV/README.md"; base_push +printf 'GITHUB_TOKEN=%s\n' "$tok" >"$GL_DEV/k.env"; pr_push +gl_step scan; gl_expect_pass "对照 C1：base 配置中的 allowlist 生效（范围扫描）" $PR_ARGS +gl_step tree; gl_expect_pass "对照 C1：base 配置中的 allowlist 生效（工作树扫描）" $PR_ARGS + +# 对照 C2：base 的 .gitleaksignore 指纹生效 +setup_pr cc2 yes; printf 'k.env:github-pat:1\n' >"$GL_DEV/.gitleaksignore"; printf 'hello\n' >"$GL_DEV/README.md"; base_push +printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env"; pr_push +gl_step scan; gl_expect_pass "对照 C2：base 的 .gitleaksignore 生效（范围扫描）" $PR_ARGS +gl_step tree; gl_expect_pass "对照 C2：base 的 .gitleaksignore 生效（工作树扫描）" $PR_ARGS +gl_step scan + +# U1/U2：gitleaks-base-config.sh 的边界（目标版本没有该脚本时判 FAIL） +if gl_has .github/scripts/gitleaks-base-config.sh; then + gl_new_origin u1; printf '%s\n' "$ALLOW_ALL" >"$GL_DEV/.gitleaks.toml"; printf 'x:y:1\n' >"$GL_DEV/.gitleaksignore" + git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm base; h="$(git -C "$GL_DEV" rev-parse HEAD)" + gl_src .github/scripts/gitleaks-base-config.sh >"$GL_TMP/bc.sh" + # review3h P3-g：bc.sh 按 $(dirname "$0")/gitleaks-config-check.py 调检查器，必须一起复制； + # 否则检查器缺失也会让 bc.sh 失败，U2 会“因为缺文件”而误判通过 + rm -f "$GL_TMP/gitleaks-config-check.py" + if gl_has .github/scripts/gitleaks-config-check.py; then gl_src .github/scripts/gitleaks-config-check.py >"$GL_TMP/gitleaks-config-check.py"; fi + rc=0; (cd "$GL_DEV" && EVENT_NAME=push DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch bash "$GL_TMP/bc.sh" "$h" "$GL_TMP/u1out") >"$GL_TMP/u1.log" 2>&1 || rc=$? + if [ "$rc" -eq 0 ] && ! grep -q allowlists "$GL_TMP/u1out/gitleaks.toml" && grep -q 'useDefault = true' "$GL_TMP/u1out/gitleaks.toml" \ + && [ ! -s "$GL_TMP/u1out/gitleaksignore" ] && [ ! -e "$GL_DEV/.gitleaksignore" ]; then + echo "PASS [U1 无基准（全量扫描）时用默认规则、空忽略列表，不用 HEAD 的文件，并移走工作区 .gitleaksignore]" + else echo "FAIL [U1 无基准时应使用默认规则/空忽略列表] rc=${rc}: $(tail -1 "$GL_TMP/u1.log")"; GL_FAILS=$((GL_FAILS+1)); fi + gl_new_origin u2; printf '[extend]\npath = "evil.toml"\n' >"$GL_DEV/.gitleaks.toml"; printf 'a\n' >"$GL_DEV/a" + git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm base; b="$(git -C "$GL_DEV" rev-parse HEAD)" + printf 'b\n' >"$GL_DEV/a"; git -C "$GL_DEV" commit -qam two; h="$(git -C "$GL_DEV" rev-parse HEAD)" + rc=0; (cd "$GL_DEV" && EVENT_NAME=push DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch bash "$GL_TMP/bc.sh" "$b..$h" "$GL_TMP/u2out") >"$GL_TMP/u2.log" 2>&1 || rc=$? + # 断言具体的拒绝原因（检查器的输出），不只看退出码和 "extend" 字样 + if [ ! -f "$GL_TMP/gitleaks-config-check.py" ]; then + echo "FAIL [U2 基准配置使用 [extend] path 时应失败] 目标版本没有 .github/scripts/gitleaks-config-check.py"; GL_FAILS=$((GL_FAILS+1)) + elif [ "$rc" -ne 0 ] && grep -F -q '配置的 [extend] 使用了 path' "$GL_TMP/u2.log"; then + echo "PASS [U2 基准配置 [extend] path 指向工作区文件时拒绝（检查器报 [extend] 使用了 path）]" + else echo "FAIL [U2 基准配置使用 [extend] path 时应失败且由检查器拒绝] rc=${rc}: $(tail -1 "$GL_TMP/u2.log")"; GL_FAILS=$((GL_FAILS+1)); fi + # 对照 U2c：同样的流程，基准配置合法（useDefault + 一条自定义规则）时通过，输出就是基准上的配置 + gl_new_origin u2c; printf '[extend]\nuseDefault = true\n\n[[rules]]\nid = "u2c-rule"\nregex = %s\n' "'''u2c-[0-9]{8}'''" >"$GL_DEV/.gitleaks.toml"; printf 'a\n' >"$GL_DEV/a" + git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm base; b="$(git -C "$GL_DEV" rev-parse HEAD)" + printf 'b\n' >"$GL_DEV/a"; git -C "$GL_DEV" commit -qam two; h="$(git -C "$GL_DEV" rev-parse HEAD)" + rc=0; (cd "$GL_DEV" && EVENT_NAME=push DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch bash "$GL_TMP/bc.sh" "$b..$h" "$GL_TMP/u2cout") >"$GL_TMP/u2c.log" 2>&1 || rc=$? + git -C "$GL_DEV" show "$b:.gitleaks.toml" >"$GL_TMP/u2c-base.toml" + if [ "$rc" -eq 0 ] && cmp -s "$GL_TMP/u2c-base.toml" "$GL_TMP/u2cout/gitleaks.toml"; then + echo "PASS [对照 U2c 基准配置合法（无 [extend] path）时通过，使用基准配置]" + else echo "FAIL [对照 U2c 基准配置合法时应通过] rc=${rc}: $(tail -1 "$GL_TMP/u2c.log")"; GL_FAILS=$((GL_FAILS+1)); fi +else + echo "FAIL [U1/U2] 目标版本没有 .github/scripts/gitleaks-base-config.sh（CI 仍使用 HEAD 的配置）"; GL_FAILS=$((GL_FAILS+1)) +fi +gl_finish diff --git a/.github/tests/test-gitleaks-concurrency.sh b/.github/tests/test-gitleaks-concurrency.sh new file mode 100755 index 00000000..56d159f2 --- /dev/null +++ b/.github/tests/test-gitleaks-concurrency.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# SEC-01A 审查 P2-2：push 事件不得被 concurrency 取消（否则被取消的 push 永远不会补扫），PR 事件可以取消。 +# 解析工作流 concurrency.cancel-in-progress，并对 push / pull_request 两种事件求值 +# （支持字面量 true/false 和 ${{ github.event_name == '...' }} / != 形式）。 +# 另外：GitHub 对同一 concurrency group 只保留 1 个运行中 + 1 个排队中的运行，新排队的会取消旧的 +# 排队运行（与 cancel-in-progress 无关）。因此 push 事件的 group 还必须每次运行唯一（含 github.run_id）， +# 否则连续 3 次 push 时中间那次仍会被取消、其提交永不补扫。 +# 用法: .github/tests/test-gitleaks-concurrency.sh [--rev ] [--workflow ] （对修复前版本运行应报 FAIL） +# --workflow 被检查的工作流文件（默认 .github/workflows/gitleaks.yml；web 仓库也用它检查 verify.yml） +# group 必须与下列表达式完全一致（只忽略 ${{ }} 内多余空白），不接受“包含 run_id 片段”之类的宽松匹配： +# ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && 'pr' || github.run_id }} +CC_WORKFLOW=""; CC_ARGS=() +while [ $# -gt 0 ]; do + case "$1" in --workflow) CC_WORKFLOW="$2"; shift 2 ;; *) CC_ARGS+=("$1"); shift ;; esac +done +source "$(dirname "${BASH_SOURCE[0]}")/lib-gitleaks-workflow.sh" +gl_parse_args "${CC_ARGS[@]+"${CC_ARGS[@]}"}" +if [ -n "$CC_WORKFLOW" ]; then GL_WORKFLOW="$CC_WORKFLOW"; fi +echo "工作流: ${GL_WORKFLOW}" + +eval_cancel() { # eval_cancel -> 打印 true/false + gl_src "$GL_WORKFLOW" | python3 -c ' +import re, sys +event = sys.argv[1]; text = sys.stdin.read() +m = re.search(r"^concurrency:\s*\n((?:[ \t]+.*\n?)+)", text, re.M) +if not m: print("false"); sys.exit(0) # 无 concurrency：不取消 +v = re.search(r"^\s+cancel-in-progress:\s*(.+?)\s*$", m.group(1), re.M) +if not v: print("false"); sys.exit(0) +val = v.group(1).strip().strip("\"\x27") +if val in ("true", "false"): print(val); sys.exit(0) +e = re.fullmatch(r"\$\{\{\s*github\.event_name\s*(==|!=)\s*\x27([^\x27]+)\x27\s*\}\}", val) +if not e: sys.exit("无法求值的表达式: " + val) +eq = (event == e.group(2)); print(str(eq if e.group(1) == "==" else not eq).lower()) +' "$1" +} +expect() { # expect <期望值> + local got; got="$(eval_cancel "$1")" + if [ "$got" = "$2" ]; then echo "PASS [$1] cancel-in-progress=${got}" + else echo "FAIL [$1] cancel-in-progress=${got}（期望 ${2}）"; GL_FAILS=$((GL_FAILS+1)); fi +} +echo "concurrency 段: $(gl_src "$GL_WORKFLOW" | grep -A2 '^concurrency:' | tr -s ' ' | tr '\n' ' ')" +group_unique_for_push() { + gl_src "$GL_WORKFLOW" | python3 -c ' +import re, sys +text = sys.stdin.read() +m = re.search(r"^concurrency:\s*\n((?:[ \t]+.*\n?)+)", text, re.M) +if not m: print("true"); sys.exit(0) # 无 concurrency：互不影响 +gm = re.search(r"^\s+group:\s*(.+?)\s*$", m.group(1), re.M) +if not gm: print("false"); sys.exit(0) +g = gm.group(1).strip().strip("\"\x27") +# 精确匹配：每个 ${{ ... }} 内的空白归一化后，与期望表达式逐字比较 +norm = lambda x: re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda e: "${{ " + re.sub(r"\s+", " ", e.group(1)) + " }}", x) +want = "${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == \x27pull_request\x27 && \x27pr\x27 || github.run_id }}" +print("true" if norm(g) == want else "false") +' +} +expect push false +expect pull_request true +got="$(group_unique_for_push)" +if [ "$got" = true ]; then echo "PASS [push] concurrency group 每次运行唯一（含 github.run_id），排队中的 push 不会被取消" +else echo "FAIL [push] concurrency group 与期望表达式不一致（push 需每次运行唯一、且按工作流 + ref 区分）: $(gl_src "$GL_WORKFLOW" | grep -m1 -E '^[[:space:]]+group:' | sed -E 's/^[[:space:]]+//')"; GL_FAILS=$((GL_FAILS+1)); fi +gl_finish diff --git a/.github/tests/test-gitleaks-extend-check.sh b/.github/tests/test-gitleaks-extend-check.sh new file mode 100755 index 00000000..2a377a6c --- /dev/null +++ b/.github/tests/test-gitleaks-extend-check.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# review3g N1：基准 .gitleaks.toml 的 [extend] path 检查改用 TOML 解析器（gitleaks-config-check.py，tomllib）。 +# gitleaks 8.30.1 实测（见 review3g/n1-gitleaks-extend-probe.log）：键名不区分大小写，带点的键按层级展开， +# 下列写法都会让 gitleaks 读取 path 指向的文件；旧的 awk 只认 "[extend]" + 行首 "path ="，可以绕过。 +# +# 每个用例：main 上一次推送的 .gitleaks.toml 用该写法指向 evil.toml；下一次推送加入“全部放行”的 evil.toml +# 和合成 PAT（before = 上一次推送）→ 必须因 [extend] 检查失败（修复前：gitleaks 读入 evil.toml，漏扫）。 +# X1..X13 各种 extend.path 写法（含大小写、点号键、内联表、引号键、\u 转义、子表） +# M1 多个大小写不同的 extend 表 M2 extend 不是表 M3 TOML 解析失败 +# M4 没有 tomllib（PYTHONPATH 中放一个抛 ModuleNotFoundError 的 tomllib.py 模拟 Python < 3.11） +# M5 PATH 中没有 python3（直接调用 gitleaks-base-config.sh） +# 对照 P1 目标版本自己的 .gitleaks.toml、P2 useDefault + disabledRules：检查通过，合成 PAT 照常检出 +# +# 用法: .github/tests/test-gitleaks-extend-check.sh [--rev ] （对修复前版本运行，X*/M* 应报 FAIL） +# 合成密钥均在运行时随机生成；所有仓库都在临时目录中创建。 +source "$(dirname "${BASH_SOURCE[0]}")/lib-gitleaks-workflow.sh" +gl_parse_args "$@" +ALLOW_ALL="$(printf '[extend]\nuseDefault = true\n\n[[allowlists]]\ndescription = "allow everything"\npaths = [%s]\n' "'''.*'''")" +P='error:\[extend\] 使用了 path' + +# setup_case <配置行...>：main 推送该配置（H1），再推 evil.toml(全部放行) + 合成 PAT（H2），检出 H2 +setup_case() { + local name="$1"; shift + gl_new_origin "$name" + if [ "$#" -gt 0 ]; then printf '%s\n' "$@" >"$GL_DEV/.gitleaks.toml"; fi + printf 'hello\n' >"$GL_DEV/README.md" + git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm base; git -C "$GL_DEV" push -q origin HEAD:main + H1="$(git -C "$GL_DEV" rev-parse HEAD)" + printf '%s\n' "$ALLOW_ALL" >"$GL_DEV/evil.toml" + printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env" + git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm secret; git -C "$GL_DEV" push -q origin HEAD:main + H2="$(git -C "$GL_DEV" rev-parse HEAD)" + gl_checkout "$H2" + ARGS="EVENT_NAME=push PUSH_BEFORE=$H1 HEAD_SHA=$H2 DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch" +} +xcase() { # xcase <名称> <期望 error:ERE> <配置行...> + local name="$1" want="$2"; shift 2 + setup_case "x$GL_N" "$@"; GL_N=$((GL_N+1)) + # shellcheck disable=SC2086 + gl_expect_block "$name" "$want" $ARGS +} +GL_N=1 +gl_step scan +xcase 'X1 [extend] path' "$P" '[extend]' 'path = "evil.toml"' +xcase 'X2 [extend] 后带注释' "$P" '[extend] # x' 'path = "evil.toml"' +xcase 'X3 点号键 extend.path' "$P" 'extend.path = "evil.toml"' +xcase 'X4 内联表 extend = { path }' "$P" 'extend = { path = "evil.toml" }' +xcase 'X5 引号表名 ["extend"]' "$P" '["extend"]' 'path = "evil.toml"' +xcase 'X6 引号键 "path"' "$P" '[extend]' '"path" = "evil.toml"' +xcase 'X7 [Extend] Path（大小写）' "$P" '[Extend]' 'Path = "evil.toml"' +xcase 'X8 [EXTEND]' "$P" '[EXTEND]' 'path = "evil.toml"' +xcase 'X9 引号点号键 "extend.path"' "$P" '"extend.path" = "evil.toml"' +xcase 'X10 引号点号表 ["extend.path"]' "$P" '["extend.path"]' 'x = 1' +xcase 'X11 子表 [extend.path]' "$P" '[extend.path]' 'x = "evil.toml"' +xcase 'X12 \u 转义表名' "$P" '["\u0065xtend"]' 'path = "evil.toml"' +xcase 'X13 [extend] path.x' "$P" '[extend]' 'path.x = "evil.toml"' +xcase 'M1 多个大小写不同的 extend 表' 'error:多个 extend 写法' '[extend]' 'useDefault = true' '[Extend]' 'useDefault = true' +xcase 'M2 extend 不是表' 'error:extend 不是表' 'extend = "evil.toml"' +xcase 'M3 TOML 解析失败' 'error:无法按 TOML 解析' '[extend' 'path = "evil.toml"' + +# M4：模拟没有 tomllib（目标版本自己的配置，本应通过） +SHADOW="$GL_TMP/shadow-py"; mkdir -p "$SHADOW" +printf 'raise ModuleNotFoundError("tomllib shadowed by test (simulating Python < 3.11)")\n' >"$SHADOW/tomllib.py" +setup_case m4 +# shellcheck disable=SC2086 +gl_expect_block 'M4 没有 tomllib（Python < 3.11）' 'error:缺少 tomllib' $ARGS PYTHONPATH="$SHADOW" + +# M5：PATH 中没有 python3（只放 base-config 需要的命令），直接调用脚本 +setup_case m5 +MINBIN="$GL_TMP/minbin"; mkdir -p "$MINBIN" +for c in git dirname mkdir mv cat; do ln -sf "$(command -v "$c")" "$MINBIN/$c"; done +rc=0; ( cd "$GL_WS" && env PATH="$MINBIN" EVENT_NAME=push DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch \ + "$(command -v bash)" .github/scripts/gitleaks-base-config.sh "$H1..$H2" "$GL_TMP/m5-out" ) >"$GL_TMP/m5.log" 2>&1 || rc=$? +if [ "$rc" -ne 0 ] && grep -q '缺少 python3' "$GL_TMP/m5.log"; then echo "PASS [M5 PATH 中没有 python3 时失败] $(grep -m1 '缺少 python3' "$GL_TMP/m5.log")" +else echo "FAIL [M5 PATH 中没有 python3 时应失败] rc=${rc}: $(tail -1 "$GL_TMP/m5.log")"; GL_FAILS=$((GL_FAILS+1)); fi + +# 对照：合法配置通过检查，合成 PAT 照常检出 +setup_case p1 +# shellcheck disable=SC2086 +gl_expect_block '对照 P1 目标版本的 .gitleaks.toml 通过检查' leak:github-pat:k.env $ARGS +setup_case p2 '[extend]' 'useDefault = true' 'disabledRules = ["generic-api-key"]' +# shellcheck disable=SC2086 +gl_expect_block '对照 P2 useDefault + disabledRules 通过检查' leak:github-pat:k.env $ARGS +gl_finish diff --git a/.github/tests/test-gitleaks-history-fullscan.sh b/.github/tests/test-gitleaks-history-fullscan.sh new file mode 100755 index 00000000..e863b368 --- /dev/null +++ b/.github/tests/test-gitleaks-history-fullscan.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# SEC-01A 第三轮补修（review3b 第 3 项）：历史中 RFC 6455 示例 nonce（24b8315 / 125bc27）要不要加带提交号的指纹。 +# 结论：不加。本脚本把依据固化为可重复的检查： +# T1 本地全量历史扫描 + HEAD 的 .gitleaks.toml：按值 allowlist 已覆盖历史提交中的 nonce → 不报 +# （review3g N5：扫描前移走开发仓库的 .gitleaksignore，否则 gitleaks 自动加载其中的指纹，T1 的通过不能归因于 allowlist） +# T1b 对照：同样条件下只用默认规则（无 allowlist）→ 必须检出 +# T2 CI 全量路径（push 默认分支且 before 全零 → 范围 = HEAD）：工作流按设计用默认规则 + 空 ignore， +# 即使 HEAD 的 .gitleaksignore 里有 commit:path:rule:line 指纹也不加载 → 仍报（已知限制，记录在 README-ops） +# T3 对照：同样的指纹若被加载（默认规则 + 该 ignore）则不报 —— 说明 T2 仍报是因为 CI 不加载，而不是指纹写错 +# T4 目标版本的 .gitleaksignore 没有有效条目，且注释如实写明 CI 全量路径不使用 allowlist/本文件 +# T5 目标版本的 .gitleaks.toml 不声称 RFC 6455 allowlist 可删除（review3g：历史中默认规则仍有 4 处命中） +# +# 用法: .github/tests/test-gitleaks-history-fullscan.sh [--rev ] （对修复前版本运行应报 FAIL） +# 依赖: git、gitleaks、python3。夹具为生成的最小 Go 文件，不读取 internal/ 下的真实文件。 +source "$(dirname "${BASH_SOURCE[0]}")/lib-gitleaks-workflow.sh" +gl_parse_args "$@" +for c in gitleaks python3; do command -v "$c" >/dev/null || { echo "缺少依赖: $c" >&2; exit 2; }; done + +LT=internal/middleware/login_test.go +NONCE="$(printf 'the sample nonce' | base64)" # RFC 6455 示例值，运行时计算 +ZERO=0000000000000000000000000000000000000000 + +gl_new_origin hist +mv "$GL_DEV/.gitleaks.toml" "$GL_TMP/target.gitleaks.toml" +# 第 1 个提交：只有夹具（模拟 allowlist 引入之前的历史提交） +mkdir -p "$GL_DEV/internal/middleware" +{ printf 'package middleware\n\n' + printf 'func a(h interface{ Set(string, string) }) {\n\th.Set("Sec-WebSocket-Key", "%s")\n}\n' "$NONCE"; } >"$GL_DEV/$LT" +git -C "$GL_DEV" add -A; git -C "$GL_DEV" -c core.hooksPath=/dev/null commit -qm "历史：含示例 nonce" +OLD="$(git -C "$GL_DEV" rev-parse HEAD)" +# 取该提交在默认规则下的精确指纹（commit:path:rule:line） +: >"$GL_TMP/empty.toml" +printf '[extend]\nuseDefault = true\n' >"$GL_TMP/default.toml" +gitleaks git --no-banner --redact --config "$GL_TMP/default.toml" --log-opts="$OLD" \ + --report-format json --report-path "$GL_TMP/fp.json" "$GL_DEV" >/dev/null 2>&1 || true +python3 -c 'import json,sys; [print(f["Fingerprint"]) for f in json.load(open(sys.argv[1])) or []]' "$GL_TMP/fp.json" >"$GL_TMP/fp.txt" +[ -s "$GL_TMP/fp.txt" ] || { echo "夹具在默认规则下未被检出，无法验证" >&2; exit 2; } +# 第 2 个提交：引入目标版本的 .gitleaks.toml，并在 .gitleaksignore 中加入上面的指纹（模拟“加指纹”方案） +cp "$GL_TMP/target.gitleaks.toml" "$GL_DEV/.gitleaks.toml" +{ if gl_has .gitleaksignore; then gl_src .gitleaksignore; fi; cat "$GL_TMP/fp.txt"; } >"$GL_DEV/.gitleaksignore" +git -C "$GL_DEV" add -A; git -C "$GL_DEV" -c core.hooksPath=/dev/null commit -qm "引入配置 + 模拟指纹" +git -C "$GL_DEV" push -q origin HEAD:main +HEAD_SHA="$(git -C "$GL_DEV" rev-parse HEAD)" + +# T1（review3g N5）：先把开发仓库中的 .gitleaksignore 移走——gitleaks 总会自动读取“源目录/.gitleaksignore”， +# 而上面第 2 个提交在其中写入了历史指纹，不移走时 T1 的通过可能来自指纹而不是 allowlist +mv "$GL_DEV/.gitleaksignore" "$GL_TMP/dev.gitleaksignore" +rc=0; gitleaks git --no-banner --redact --ignore-gitleaks-allow --config "$GL_DEV/.gitleaks.toml" \ + --gitleaks-ignore-path "$GL_TMP/empty.toml" --log-opts="HEAD" "$GL_DEV" >"$GL_TMP/t1.out" 2>&1 || rc=$? +if [ "$rc" -eq 0 ]; then echo "PASS [T1 全量历史 + HEAD 配置（无任何 .gitleaksignore）：按值 allowlist 覆盖历史提交中的 nonce]" +else echo "FAIL [T1 全量历史 + HEAD 配置仍报出（rc=${rc}）：allowlist 未覆盖历史 nonce]"; GL_FAILS=$((GL_FAILS+1)); fi +# T1b 对照：同样无 .gitleaksignore、只用默认规则（无 allowlist）→ 必须检出，证明 T1 的通过来自 allowlist +rc=0; gitleaks git --no-banner --redact --ignore-gitleaks-allow --config "$GL_TMP/default.toml" \ + --gitleaks-ignore-path "$GL_TMP/empty.toml" --log-opts="HEAD" --report-format json --report-path "$GL_TMP/t1b.json" \ + "$GL_DEV" >"$GL_TMP/t1b.out" 2>&1 || rc=$? +t1b="$(python3 -c 'import json,sys; print(sum(1 for f in json.load(open(sys.argv[1])) or [] if f["File"]==sys.argv[2] and f["RuleID"]=="generic-api-key"))' "$GL_TMP/t1b.json" "$LT" 2>/dev/null || echo 0)" +if [ "$rc" -ne 0 ] && [ "$t1b" -gt 0 ]; then echo "PASS [T1b 对照：无 allowlist、无 .gitleaksignore 时检出 ${t1b} 处（rc=${rc}）]" +else echo "FAIL [T1b 对照：无 allowlist 时应检出（rc=${rc}，命中 ${t1b}）]"; GL_FAILS=$((GL_FAILS+1)); fi +mv "$GL_TMP/dev.gitleaksignore" "$GL_DEV/.gitleaksignore" + +# T2：执行目标版本工作流的 Scan new commits（全量路径） +gl_checkout "$HEAD_SHA" +gl_step scan +gl_expect_block "T2 CI 全量路径（before 全零）：默认规则 + 空 ignore，HEAD 中的指纹不生效，仍报（已知限制）" \ + "leak:generic-api-key:$LT" EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA=$HEAD_SHA DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch +rng="$(grep -m1 'gitleaks scan range' "$GL_TMP/scan.out" | sed 's/.*range: *//' || true)" +case "$rng" in + *..*|"") echo "FAIL [T2 前提：before 全零时范围应为全量（单个提交号），实际: ${rng:-<无>}]"; GL_FAILS=$((GL_FAILS+1)) ;; + *) echo "PASS [T2 前提：范围为全量历史（${rng}）]" ;; +esac + +# T3 +rc=0; gitleaks git --no-banner --redact --ignore-gitleaks-allow --config "$GL_TMP/default.toml" \ + --gitleaks-ignore-path "$GL_DEV/.gitleaksignore" --log-opts="HEAD" "$GL_DEV" >"$GL_TMP/t3.out" 2>&1 || rc=$? +if [ "$rc" -eq 0 ]; then echo "PASS [T3 对照：指纹被加载时默认规则下不报（指纹本身正确）]" +else echo "FAIL [T3 对照：加载指纹后仍报（rc=${rc}）]"; GL_FAILS=$((GL_FAILS+1)); fi + +# T4 +gl_src .gitleaksignore >"$GL_TMP/target.ign" 2>/dev/null || : >"$GL_TMP/target.ign" +active="$(grep -v -E '^[[:space:]]*(#|$)' "$GL_TMP/target.ign" || true)" +if [ -z "$active" ]; then echo "PASS [T4 .gitleaksignore 无有效条目（未加历史指纹）]" +else echo "FAIL [T4 .gitleaksignore 有有效条目: $(printf '%s' "$active" | tr '\n' ' ')]"; GL_FAILS=$((GL_FAILS+1)); fi +if grep -F -q 'CI 全量' "$GL_TMP/target.ign" && grep -F -q '默认规则' "$GL_TMP/target.ign"; then + echo "PASS [T4 .gitleaksignore 注释写明 CI 全量路径用默认规则、不加载本文件]" +else echo "FAIL [T4 .gitleaksignore 注释未写明 CI 全量路径的限制（仍声称全量历史扫描不再报出）]"; GL_FAILS=$((GL_FAILS+1)); fi +# T5（review3g 注释修正）：.gitleaks.toml 不得再声称 RFC 6455 allowlist “不再命中任何内容/可删除”——历史仍含该 nonce（T1b 即证） +if grep -F -q '可在之后删除' "$GL_TMP/target.gitleaks.toml" || grep -F -q '不再命中任何内容' "$GL_TMP/target.gitleaks.toml"; then + echo "FAIL [T5 .gitleaks.toml 仍声称 RFC 6455 allowlist 可删除（历史中仍有命中）]"; GL_FAILS=$((GL_FAILS+1)) +elif grep -F -q '必须保留' "$GL_TMP/target.gitleaks.toml"; then echo "PASS [T5 .gitleaks.toml 注释写明历史仍命中、allowlist 必须保留]" +else echo "FAIL [T5 .gitleaks.toml 注释未写明 allowlist 必须保留]"; GL_FAILS=$((GL_FAILS+1)); fi +gl_finish diff --git a/.github/tests/test-gitleaks-lib-selfcheck.sh b/.github/tests/test-gitleaks-lib-selfcheck.sh new file mode 100644 index 00000000..bfb8fd18 --- /dev/null +++ b/.github/tests/test-gitleaks-lib-selfcheck.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# SEC-01A 第三轮审查 P3-3 自检：lib-gitleaks-workflow.sh 的 gl_expect_block 必须区分“检出密钥”与“步骤出错”， +# 并核对检出的 RuleID / File 与用例预期一致（否则 gitleaks 自身报错、范围脚本出错也会被当成“已拦下”）。 +# 用法: .github/tests/test-gitleaks-lib-selfcheck.sh [--rev ] （使用该版本的 lib；修复前版本应报 FAIL） +set -euo pipefail +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SELF_REV="" +while [ $# -gt 0 ]; do case "$1" in --rev) SELF_REV="$2"; shift 2;; *) echo "未知参数: $1" >&2; exit 2;; esac; done +SELF_LIB="$SELF_DIR/lib-gitleaks-workflow.sh" +if [ -n "$SELF_REV" ]; then + SELF_LIB="$(mktemp)"; git -C "$SELF_DIR" show "$SELF_REV:.github/tests/lib-gitleaks-workflow.sh" >"$SELF_LIB" +fi +# shellcheck source=/dev/null +source "$SELF_LIB" +[ -n "$SELF_REV" ] && rm -f "$SELF_LIB" + +# 用自定义步骤脚本替换“从工作流提取的步骤”，只检验判定逻辑本身 +SELF_STEP="$GL_TMP/selfcheck-step.sh" +gl_extract_run() { cp "$SELF_STEP" "$1"; } +GL_WS="$GL_TMP/selfcheck-ws"; mkdir -p "$GL_WS" +printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_WS/b.env" + +SELF_FAILS=0 +check() { # check <用例> <期望判定 PASS|FAIL> + local name="$1" want="$2" out verdict; shift 2 + out="$(GL_FAILS=0; gl_expect_block "$@" 2>&1 || true)" + verdict="$(printf '%s\n' "$out" | grep -m1 -oE '^(PASS|FAIL)' || echo '<无判定>')" + if [ "$verdict" = "$want" ]; then echo "PASS [${name}] gl_expect_block 判定为 ${verdict}（符合预期）" + else echo "FAIL [${name}] gl_expect_block 判定为 ${verdict}，应为 ${want} | ${out}"; SELF_FAILS=$((SELF_FAILS+1)); fi +} + +# S1 步骤因 gitleaks/脚本出错而失败（没有任何检出），用例期望检出 -> 必须判 FAIL +printf '%s\n' 'echo "ERR gitleaks: failed to load config" >&2; exit 1' >"$SELF_STEP" +check "S1 出错不能当成检出" FAIL "s1" leak:github-pat:b.env +# S2 检出了，但文件与预期不符 -> 必须判 FAIL +printf '%s\n' 'gitleaks dir --redact --no-banner --verbose --exit-code 1 .' >"$SELF_STEP" +check "S2 检出文件不符" FAIL "s2" leak:github-pat:a.env +# S3 检出了，但规则与预期不符 -> 必须判 FAIL +check "S3 检出规则不符" FAIL "s3" leak:aws-access-token:b.env +# S4 规则与文件都符合 -> PASS +check "S4 规则与文件都符合" PASS "s4" leak:github-pat:b.env +# S5 期望出错且出错信息匹配 -> PASS；期望出错但实际检出 -> FAIL +printf '%s\n' 'echo "[gitleaks-range] ERROR: 缺少 origin/main" >&2; exit 1' >"$SELF_STEP" +check "S5 期望的出错" PASS "s5" 'error:缺少 origin/main' +printf '%s\n' 'gitleaks dir --redact --no-banner --verbose --exit-code 1 .' >"$SELF_STEP" +check "S6 期望出错却检出" FAIL "s6" 'error:缺少 origin/main' +echo "目标 lib 版本: ${SELF_REV:-工作区}；失败 ${SELF_FAILS} 项" +[ "$SELF_FAILS" -eq 0 ] diff --git a/.github/tests/test-gitleaks-login-allowlist.sh b/.github/tests/test-gitleaks-login-allowlist.sh new file mode 100644 index 00000000..8c433cd4 --- /dev/null +++ b/.github/tests/test-gitleaks-login-allowlist.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# SEC-01A 第三轮审查 P2-1：login_test.go 中 RFC 6455 示例 nonce 的误报豁免必须“按值”放行。 +# 旧做法是 .gitleaksignore 里的 path:rule:line 指纹（:109 / :130）——同一行换成任何真实密钥都会被放行。 +# 新做法：.gitleaks.toml 中 [[allowlists]]（targetRules + paths + regexTarget=secret + regexes，condition=AND）。 +# +# 正例 原 nonce 出现在 login_test.go 第 109/130 行：pre-commit 与 CI（范围扫描 + 工作树扫描）均放行 +# 反例 同文件同一行换成合成密钥：pre-commit、CI 范围扫描、CI 工作树扫描都必须拦下（generic-api-key @ 该文件） +# 反例 同一个 nonce 值出现在其他文件：必须拦下（allowlist 只限该文件） +# 反例中的合成密钥先用默认规则探测、确认会被 generic-api-key 检出后才使用（review3h P3-h，避免偶发失败） +# +# 用法: .github/tests/test-gitleaks-login-allowlist.sh [--rev ] （对修复前版本运行应报 FAIL） +# 依赖: git、gofmt、gitleaks、python3。夹具为生成的最小 Go 文件（行号与原文件一致），不读取 internal/ 下的真实文件。 +source "$(dirname "${BASH_SOURCE[0]}")/lib-gitleaks-workflow.sh" +gl_parse_args "$@" +for c in gofmt gitleaks python3; do command -v "$c" >/dev/null || { echo "缺少依赖: $c" >&2; exit 2; }; done +export SKIP_VET=1 SKIP_BUILD=1 SKIP_GENERATE=1 GOTOOLCHAIN=local + +LT=internal/middleware/login_test.go +# RFC 6455 第 1.3 节示例值：base64("the sample nonce")，运行时计算，本脚本不含该字面量 +NONCE="$(printf 'the sample nonce' | base64)" + +# 生成与原文件行号一致的夹具：第 109 行与第 130 行为 Sec-WebSocket-Key 赋值，其余为注释/空行（gofmt 规范） +gen_fixture() { # gen_fixture <109 行的值> <130 行的值> + local i=3 + printf 'package middleware\n\n' + while [ "$i" -le 107 ]; do printf '// filler %s\n' "$i"; i=$((i+1)); done + printf 'func a(h interface{ Set(string, string) }) {\n\th.Set("Sec-WebSocket-Key", "%s")\n}\n\n' "$1" + i=112 + while [ "$i" -le 128 ]; do printf '// filler %s\n' "$i"; i=$((i+1)); done + printf 'func b(h interface{ Set(string, string) }) {\n\th.Set("Sec-WebSocket-Key", "%s")\n}\n' "$2" +} + +# review3h P3-h：随机 32 位样本偶尔会被 generic-api-key 的停用词/熵过滤掉（默认规则实测 300 个漏检 2 个，约 0.7%）， +# 反例因此偶发 FAIL。每个样本先用默认规则在中性路径、同样的行形状里探测，确认检出 generic-api-key 才使用； +# 连续 20 个都检不出视为环境问题（exit 2，调用处 || exit 2）。样本只经由标准输出传给调用方，不打印。 +gl_detectable_generic() { + local i=0 v rc d="$GL_TMP/probe" + mkdir -p "$d/src"; printf '[extend]\nuseDefault = true\n' >"$d/default.toml" + while [ "$i" -lt 20 ]; do + i=$((i+1)); v="$(gl_fake_generic)" + printf 'package probe\n\nfunc a(h interface{ Set(string, string) }) {\n\th.Set("Sec-WebSocket-Key", "%s")\n}\n' "$v" >"$d/src/probe.go" + rm -f "$d/r.json"; rc=0 + env -u GITLEAKS_CONFIG gitleaks dir --no-banner --config "$d/default.toml" --exit-code 3 \ + --report-format json --report-path "$d/r.json" "$d/src" >/dev/null 2>&1 || rc=$? + if [ "$rc" -eq 3 ] && grep -q '"RuleID": "generic-api-key"' "$d/r.json"; then printf '%s' "$v"; return 0; fi + [ "$rc" -eq 0 ] || [ "$rc" -eq 3 ] || { echo "gitleaks 探测失败（rc=${rc}）" >&2; exit 2; } + done + echo "连续 20 个合成样本都未被默认规则检出，无法构造反例" >&2; exit 2 +} + +# 开发者仓库：目标版本的 .gitleaks.toml / .gitleaksignore / .githooks/pre-commit + 夹具（原 nonce） +new_dev() { + gl_new_origin "$1" + if gl_has .gitleaksignore; then gl_src .gitleaksignore >"$GL_DEV/.gitleaksignore"; fi + mkdir -p "$GL_DEV/.githooks" "$GL_DEV/internal/middleware" + gl_src .githooks/pre-commit >"$GL_DEV/.githooks/pre-commit"; chmod +x "$GL_DEV/.githooks/pre-commit" + printf 'module example.com/t\n\ngo 1.21\n' >"$GL_DEV/go.mod" + gen_fixture "$NONCE" "$NONCE" >"$GL_DEV/$LT" + git -C "$GL_DEV" add -A; git -C "$GL_DEV" -c core.hooksPath=/dev/null commit -qm init + git -C "$GL_DEV" config core.hooksPath .githooks + [ -z "$(gofmt -l "$GL_DEV/$LT")" ] || { echo "夹具格式不规范" >&2; exit 2; } + [ "$(sed -n 109p "$GL_DEV/$LT" | grep -c "$NONCE")" = 1 ] && [ "$(sed -n 130p "$GL_DEV/$LT" | grep -c "$NONCE")" = 1 ] \ + || { echo "夹具行号不符" >&2; exit 2; } +} + +# 通过 pre-commit 钩子提交，判定是否被拦以及拦截原因 +hook_commit() { # hook_commit <名称> <期望 block|pass> <期望文件:行>（block 时） + local name="$1" want="$2" where="${3:-}" rc=0 before after + before="$(git -C "$GL_DEV" rev-parse HEAD)" + git -C "$GL_DEV" commit -qm "$name" >"$GL_TMP/hook.out" 2>&1 || rc=$? + after="$(git -C "$GL_DEV" rev-parse HEAD)" + if [ "$want" = pass ]; then + if [ "$rc" -eq 0 ] && [ "$after" != "$before" ]; then echo "PASS [$name] pre-commit 放行，提交成功" + else echo "FAIL [$name] 应放行但被拦（rc=${rc}）: $(grep -m1 -E 'RuleID|ERROR|发现' "$GL_TMP/hook.out")"; GL_FAILS=$((GL_FAILS+1)); fi + return 0 + fi + local file="${where%:*}" line="${where##*:}" + if [ "$rc" -ne 0 ] && [ "$after" = "$before" ] \ + && awk -v f="$file" -v l="$line" '$1=="RuleID:"{r=$2} $1=="File:"{p=$2} $1=="Line:"{ if (r=="generic-api-key" && p==f && $2==l) ok=1 } END{exit !ok}' "$GL_TMP/hook.out"; then + echo "PASS [$name] pre-commit 拦下（generic-api-key @ ${file}:${line}）" + elif [ "$rc" -ne 0 ] && [ "$after" = "$before" ]; then + echo "FAIL [$name] 提交被拒但不是因为检出 generic-api-key @ ${where}: $(grep -m1 -E 'ERROR|RuleID|gofmt' "$GL_TMP/hook.out")"; GL_FAILS=$((GL_FAILS+1)) + else + echo "FAIL [$name] pre-commit 放行（rc=${rc}），密钥进入提交"; GL_FAILS=$((GL_FAILS+1)) + fi +} + +# ---------- pre-commit ---------- +new_dev pc-pos +printf '\n// touch\n' >>"$GL_DEV/$LT"; git -C "$GL_DEV" add "$LT" +hook_commit "正例 pre-commit：原 nonce 在第 109/130 行" pass + +new_dev pc-neg109 +fk="$(gl_detectable_generic)" || exit 2 +gen_fixture "$fk" "$NONCE" >"$GL_DEV/$LT"; git -C "$GL_DEV" add "$LT" +hook_commit "反例 pre-commit：第 109 行换成合成密钥" block "$LT:109" + +new_dev pc-neg130 +fk="$(gl_detectable_generic)" || exit 2 +gen_fixture "$NONCE" "$fk" >"$GL_DEV/$LT"; git -C "$GL_DEV" add "$LT" +hook_commit "反例 pre-commit：第 130 行换成合成密钥" block "$LT:130" + +new_dev pc-other +mkdir -p "$GL_DEV/internal/other"; sed 's/^package middleware/package other/' "$GL_DEV/$LT" >"$GL_DEV/internal/other/ws_test.go" +git -C "$GL_DEV" add -A +hook_commit "反例 pre-commit：同一 nonce 出现在其他文件" block "internal/other/ws_test.go:109" + +# ---------- CI（执行目标版本工作流的 Scan new commits / Scan working tree 步骤） ---------- +ci_case() { # ci_case <名称前缀> <109 行的值> <期望 pass|block> + new_dev "ci$RANDOM" + git -C "$GL_DEV" push -q origin HEAD:main; local before; before="$(git -C "$GL_DEV" rev-parse HEAD)" + gen_fixture "$2" "$NONCE" >"$GL_DEV/$LT"; printf '\n// change\n' >>"$GL_DEV/$LT" + git -C "$GL_DEV" add -A; git -C "$GL_DEV" -c core.hooksPath=/dev/null commit -qm change + git -C "$GL_DEV" push -q origin HEAD:main; local head; head="$(git -C "$GL_DEV" rev-parse HEAD)" + gl_checkout "$head" + local args="EVENT_NAME=push PUSH_BEFORE=$before HEAD_SHA=$head DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch" + if [ "$3" = pass ]; then + gl_step scan; gl_expect_pass "$1 CI 范围扫描" $args + gl_step tree; gl_expect_pass "$1 CI 工作树扫描" $args + else + gl_step scan; gl_expect_block "$1 CI 范围扫描" "leak:generic-api-key:$LT" $args + gl_step tree; gl_expect_block "$1 CI 工作树扫描" "leak:generic-api-key:$LT" $args + fi + gl_step scan +} +ci_case "正例：原 nonce 保持不变" "$NONCE" pass +fk="$(gl_detectable_generic)" || exit 2 +ci_case "反例：第 109 行换成合成密钥" "$fk" block +gl_finish diff --git a/.github/tests/test-gitleaks-merge-diff.sh b/.github/tests/test-gitleaks-merge-diff.sh new file mode 100755 index 00000000..7ae1b356 --- /dev/null +++ b/.github/tests/test-gitleaks-merge-diff.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# SEC-01A 审查 P2-1：merge commit 自身引入的密钥（冲突解决 / evil merge）必须被 CI 扫描检出。 +# 用法: .github/tests/test-gitleaks-merge-diff.sh [--rev ] （对修复前版本运行应报 FAIL） +source "$(dirname "${BASH_SOURCE[0]}")/lib-gitleaks-workflow.sh" +gl_parse_args "$@" + +# 冲突合并：main 与 feat 改同一行，密钥只出现在 merge 提交的冲突解决结果里 +gl_new_origin merge +gl_commit app.conf 'mode = base' init +git -C "$GL_DEV" push -q origin HEAD:main +BEFORE="$(git -C "$GL_DEV" rev-parse HEAD)" +git -C "$GL_DEV" checkout -qb feat; gl_commit app.conf 'mode = feat' feat +git -C "$GL_DEV" checkout -q main; gl_commit app.conf 'mode = main' main +git -C "$GL_DEV" merge -q feat >/dev/null 2>&1 || true +printf 'mode = merged\ngithub_token = "%s"\n' "$(gl_fake_ghp)" >"$GL_DEV/app.conf" +git -C "$GL_DEV" add app.conf; git -C "$GL_DEV" commit -qm "merge feat" +git -C "$GL_DEV" push -q origin main feat +HEAD_SHA="$(git -C "$GL_DEV" rev-parse HEAD)" +gl_checkout "$HEAD_SHA" +gl_expect_block "push 到 main，密钥只在冲突合并提交中" leak:github-pat:app.conf EVENT_NAME=push PUSH_BEFORE="$BEFORE" HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch + +# 无冲突的 evil merge：合并时额外加入一个含 AWS key 的文件 +gl_new_origin evil +gl_commit a.txt a init; git -C "$GL_DEV" push -q origin HEAD:main +BEFORE="$(git -C "$GL_DEV" rev-parse HEAD)" +git -C "$GL_DEV" checkout -qb feat; gl_commit b.txt b feat +git -C "$GL_DEV" checkout -q main +git -C "$GL_DEV" merge -q --no-ff --no-commit feat >/dev/null 2>&1 +printf 'aws_access_key_id = %s\n' "$(gl_fake_aws)" >"$GL_DEV/creds.ini" +git -C "$GL_DEV" add creds.ini; git -C "$GL_DEV" commit -qm "merge feat (evil)" +git -C "$GL_DEV" push -q origin main feat +HEAD_SHA="$(git -C "$GL_DEV" rev-parse HEAD)"; FEAT_SHA="$(git -C "$GL_DEV" rev-parse feat)" +gl_checkout "$HEAD_SHA" +gl_expect_block "push 到 main，无冲突 evil merge 加入密钥文件" leak:aws-access-token:creds.ini EVENT_NAME=push PUSH_BEFORE="$BEFORE" HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch +gl_expect_pass "对照：PR 事件 base..feat，无密钥" EVENT_NAME=pull_request PR_BASE_SHA="$BEFORE" HEAD_SHA="$FEAT_SHA" DEFAULT_BRANCH=main REF_NAME=1/merge +gl_finish diff --git a/.github/tests/test-gitleaks-min-version.sh b/.github/tests/test-gitleaks-min-version.sh new file mode 100755 index 00000000..b056eeb5 --- /dev/null +++ b/.github/tests/test-gitleaks-min-version.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# review3g N6：本地 gitleaks 最低版本 8.25.0（.gitleaks.toml 的多个 [[allowlists]] 自 8.25.0 起支持）， +# 版本过低或无法解析时 .githooks/pre-commit 与 scripts/setup-git-hooks.sh 都必须失败；钩子不再回退到 `gitleaks protect --staged`。 +# 用假的 gitleaks（只改写 `gitleaks version` 的输出，其余参数交给真实 gitleaks）模拟各版本： +# 钩子 H1 8.24.3（H1b 干净提交同样拒绝）/ H2 v8.18.4（旧版本走 protect 回退的路径）/ H3 无法解析（dev）/ H4 8.25（缺补丁号） +# H5 空输出 / H6 8.25.0-rc1 → 拒绝提交，合成密钥不进入提交 +# 对照 HC1 8.25.0 / HC2 v9.0.0 / HC3 真实版本 干净提交成功；HC4 8.25.0 暂存合成 PAT 仍被检出 +# 钩子 H8–H13（review3h P3-i）前导零 / 超长数字 / 四段 → 拒绝；对照 HC5 8.100.0 / HC6 v10.0.0 / HC7 8.25.999999 成功 +# 钩子 H7 不含 `gitleaks protect` 调用（回退已删除） +# setup（不依赖 husky 的仓库）S1 8.24.3 / S2 无法解析 / S4 前导零 / S5 超长 → 非 0 退出；S3 8.25.0 → 成功 +# 一致性 K1 钩子与 setup 中的 gitleaks_version_check 逐字一致 +# 用法: .github/tests/test-gitleaks-min-version.sh [--rev ] （对修复前版本运行应报 FAIL） +# 在 mktemp -d 的临时仓库中运行；合成 PAT 在运行时随机生成，不是真实凭据。兼容 macOS bash 3.2 / BSD 工具。 +set -euo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +REV="" +while [ $# -gt 0 ]; do + case "$1" in + --rev) REV="$2"; shift 2 ;; + *) echo "未知参数: $1" >&2; exit 2 ;; + esac +done +export PATH="$HOME/.local/bin:$PATH" +for c in git gitleaks; do command -v "$c" >/dev/null || { echo "缺少依赖: $c" >&2; exit 2; }; done +REAL_GL="$(command -v gitleaks)" +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +export GIT_CONFIG_GLOBAL="$TMP/gitconfig" GIT_CONFIG_NOSYSTEM=1 +export SKIP_VET=1 SKIP_BUILD=1 SKIP_GENERATE=1 +unset GITLEAKS_CONFIG GITLEAKS_CONFIG_TOML || true +git config --global user.name test; git config --global user.email test@example.invalid +git config --global init.defaultBranch main + +src() { if [ -n "$REV" ]; then git -C "$REPO" show "$REV:$1"; else cat "$REPO/$1"; fi; } +fake_token() { printf 'ghp_%s' "$(LC_ALL=C tr -dc 'A-Za-z0-9' ：返回该假 gitleaks 所在目录 + local d="$TMP/bin-$1"; mkdir -p "$d" + { printf '#!/bin/sh\n' + printf 'if [ "$1" = version ]; then printf "%%s\\n" '"'"'%s'"'"'; exit 0; fi\n' "$2" + printf 'exec "%s" "$@"\n' "$REAL_GL"; } >"$d/gitleaks" + chmod +x "$d/gitleaks"; echo "$d" +} +new_repo() { # new_repo ：初始提交（不经钩子） + local d="$TMP/$1"; mkdir -p "$d/.githooks" "$d/scripts"; cd "$d"; git init -q . + src .githooks/pre-commit >.githooks/pre-commit; chmod +x .githooks/pre-commit + src scripts/setup-git-hooks.sh >scripts/setup-git-hooks.sh; chmod +x scripts/setup-git-hooks.sh + printf 'hello\n' >README.md + git add -A; git -c core.hooksPath=/dev/null commit -qm init + git config core.hooksPath .githooks +} +FAILS=0 +fail() { echo "FAIL [$1] $2"; FAILS=$((FAILS+1)); } +TOK="" +hook_case() { # hook_case <名称> <版本输出|REAL> <期望 refuse|ok> [] + local name="$1" ver="$2" kind="$3" want="$4" ere="${5:-}" bin rc + CASE="h$RANDOM$RANDOM"; new_repo "$CASE" + if [ "$ver" = REAL ]; then bin="$(dirname "$REAL_GL")"; else bin="$(fake_gl "$CASE" "$ver")"; fi + if [ "$kind" = secret ]; then TOK="$(fake_token)"; printf 'GITHUB_TOKEN=%s\n' "$TOK" >k.env; git add k.env + else TOK=""; printf 'ok\n' >a.txt; git add a.txt; fi + set +e; PATH="$bin:$PATH" git commit -qm "$CASE" >"$TMP/$CASE.out" 2>&1; rc=$?; set -e + git log --all -p >"$TMP/log.txt" # 不用管道：pipefail + grep -q 会因 SIGPIPE 误判 + if [ -n "$TOK" ] && grep -qF "$TOK" "$TMP/log.txt"; then fail "$name" "合成密钥进入了提交（rc=${rc}）"; return; fi + if [ "$want" = ok ]; then + git log -1 --format=%s >"$TMP/subj.txt" + if [ "$rc" -eq 0 ] && grep -qx "$CASE" "$TMP/subj.txt"; then echo "PASS [$name] 提交成功" + else fail "$name" "提交未成功（rc=${rc}）: $(tail -1 "$TMP/$CASE.out")"; fi + elif [ "$rc" -eq 0 ]; then fail "$name" "提交成功（应被拒绝）" + elif grep -qE "$ere" "$TMP/$CASE.out"; then echo "PASS [$name] 拒绝提交（rc=${rc}）: $(grep -m1 -E "$ere" "$TMP/$CASE.out")" + else fail "$name" "提交被拒但原因不符（期望 /$ere/）: $(tail -1 "$TMP/$CASE.out")"; fi +} +LOW='低于 8\.25\.0'; BAD='无法解析 gitleaks 版本' +hook_case 'H1 gitleaks 8.24.3' '8.24.3' secret refuse "$LOW" +hook_case 'H1b gitleaks 8.24.3 干净提交也拒绝' '8.24.3' clean refuse "$LOW" +hook_case 'H2 gitleaks v8.18.4（旧回退路径）' 'v8.18.4' secret refuse "$LOW" +hook_case 'H3 版本输出无法解析（dev）' 'dev' secret refuse "$BAD" +hook_case 'H4 版本缺补丁号（8.25）' '8.25' secret refuse "$BAD" +hook_case 'H5 版本输出为空' '' secret refuse "$BAD" +hook_case 'H6 预发布版本（8.25.0-rc1）' '8.25.0-rc1' secret refuse "$BAD" +hook_case '对照 HC1 8.25.0 干净提交' '8.25.0' clean ok +hook_case '对照 HC2 v9.0.0 干净提交' 'v9.0.0' clean ok +hook_case '对照 HC3 真实 gitleaks 干净提交' REAL clean ok +hook_case '对照 HC4 8.25.0 暂存合成 PAT 仍被检出' '8.25.0' secret refuse 'gitleaks 在暂存内容中发现' +hook_case 'H8 主版本前导零（08.25.0）' '08.25.0' clean refuse "$BAD" +hook_case 'H9 次版本前导零（8.025.0）' '8.025.0' clean refuse "$BAD" +hook_case 'H10 补丁号前导零（8.25.00）' '8.25.00' clean refuse "$BAD" +hook_case 'H11 超长主版本（20 位数字）' '99999999999999999999.0.0' secret refuse "$BAD" +hook_case 'H12 超长补丁号（7 位数字）' '8.25.1234567' clean refuse "$BAD" +hook_case 'H13 四段版本（8.25.0.1）' '8.25.0.1' clean refuse "$BAD" +hook_case '对照 HC5 8.100.0 干净提交' '8.100.0' clean ok +hook_case '对照 HC6 v10.0.0 干净提交' 'v10.0.0' clean ok +hook_case '对照 HC7 8.25.999999（6 位补丁号）' '8.25.999999' clean ok + +src .githooks/pre-commit >"$TMP/hook.txt" +if grep -E '^[[:space:]]*gitleaks[[:space:]]+protect' "$TMP/hook.txt" >/dev/null; then fail 'H7 钩子不再回退到 gitleaks protect' "仍有: $(grep -m1 -E '^[[:space:]]*gitleaks[[:space:]]+protect' "$TMP/hook.txt")" +else echo "PASS [H7 钩子不再回退到 gitleaks protect]"; fi + +# setup：依赖 husky 的仓库（web）由 scripts/tests/security/test-setup-git-hooks.sh 覆盖 +src scripts/setup-git-hooks.sh >"$TMP/setup.txt" +if grep -qF 'node_modules/.bin/husky' "$TMP/setup.txt"; then echo "SKIP [S1–S3 setup 依赖 husky，见 scripts/tests/security/test-setup-git-hooks.sh 情况5]" +else + setup_case() { # setup_case <名称> <版本输出> <期望 fail|ok> + local d rc; d="s$RANDOM$RANDOM"; new_repo "$d"; git config --unset core.hooksPath + set +e; PATH="$(fake_gl "$d" "$2"):$PATH" bash scripts/setup-git-hooks.sh >"$TMP/$d.out" 2>&1; rc=$?; set -e + if { [ "$3" = fail ] && [ "$rc" -ne 0 ]; } || { [ "$3" = ok ] && [ "$rc" -eq 0 ]; }; then echo "PASS [$1] rc=${rc}" + else fail "$1" "rc=${rc}（期望 $3）: $(tail -1 "$TMP/$d.out")"; fi + } + setup_case 'S1 setup：gitleaks 8.24.3' '8.24.3' fail + setup_case 'S2 setup：版本无法解析' 'dev' fail + setup_case 'S4 setup：前导零（08.25.0）' '08.25.0' fail + setup_case 'S5 setup：超长版本号' '99999999999999999999.0.0' fail + setup_case '对照 S3 setup：gitleaks 8.25.0' '8.25.0' ok +fi + +fn_of() { awk '/^gitleaks_version_check\(\) \{/{p=1} p{print} p&&/^}/{exit}' "$1"; } +fn_of "$TMP/hook.txt" >"$TMP/fn-hook.txt"; fn_of "$TMP/setup.txt" >"$TMP/fn-setup.txt" +if [ -s "$TMP/fn-hook.txt" ] && cmp -s "$TMP/fn-hook.txt" "$TMP/fn-setup.txt"; then echo "PASS [K1 钩子与 setup 的 gitleaks_version_check 一致]" +else fail 'K1 钩子与 setup 的 gitleaks_version_check 一致' "缺失或不一致"; fi + +echo "目标版本: ${REV:-工作区}；失败 $FAILS 项" +[ "$FAILS" -eq 0 ] diff --git a/.github/tests/test-gitleaks-policy-source.sh b/.github/tests/test-gitleaks-policy-source.sh new file mode 100755 index 00000000..c7b3a64d --- /dev/null +++ b/.github/tests/test-gitleaks-policy-source.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# review3g N2：扫描起点 ≠ 策略来源。push 到非默认分支 / tag、PR 进非默认分支时，gitleaks 配置与忽略列表 +# 一律取自 origin/<默认分支>，不能取自 push 的 before（同一分支上一次推送，攻击者可控）。 +# +# N2-1 feature 分支：先推全部放行的 .gitleaks.toml（无密钥，通过），再推合成 PAT（before=上一次推送）→ 必须检出 +# （范围扫描 + 工作树扫描各判一次） +# N2-2 同上，但第一次推送的是带该文件指纹的 .gitleaksignore → 必须检出 +# N2-3 PR 目标是非默认分支（其 tip 已改成全部放行），PR 中加入合成 PAT → 必须检出 +# N2-4 push tag（before 全零），tag 指向“全部放行配置 + 合成 PAT”的提交 → 必须检出 +# 对照 C1 origin/main 的配置按值放行某合成值 → feature 推送该值放行（证明策略确实取自 origin/main） +# 对照 C2 push 到 main：before 上的按值 allowlist 生效（默认分支仍用范围起点） +# E1 push 到 feature 但本地没有 origin/main → 失败（不回退到 before） +# E2 缺少 EVENT_NAME → 失败 +# +# 用法: .github/tests/test-gitleaks-policy-source.sh [--rev ] （对修复前版本运行，N2-1/N2-2/N2-3/E1/E2 应报 FAIL） +# 合成密钥均在运行时随机生成；所有仓库都在临时目录中创建。 +source "$(dirname "${BASH_SOURCE[0]}")/lib-gitleaks-workflow.sh" +gl_parse_args "$@" +ZERO=0000000000000000000000000000000000000000 +ALLOW_ALL="$(printf '[extend]\nuseDefault = true\n\n[[allowlists]]\ndescription = "allow everything"\npaths = [%s]\n' "'''.*'''")" + +main_base() { # main_base ：origin/main 上有目标版本的 .gitleaks.toml，本地切到 feature + gl_new_origin "$1"; printf 'hello\n' >"$GL_DEV/README.md" + git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm base; git -C "$GL_DEV" push -q origin HEAD:main + MAIN_SHA="$(git -C "$GL_DEV" rev-parse HEAD)"; git -C "$GL_DEV" checkout -q -b feature +} +push_feature() { # push_feature ：提交工作区全部改动并推到 origin/feature，设置 H + git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm "$1"; git -C "$GL_DEV" push -q origin HEAD:feature + H="$(git -C "$GL_DEV" rev-parse HEAD)" +} + +# N2-1：先推全部放行的配置，再推密钥 +main_base n21; printf '%s\n' "$ALLOW_ALL" >"$GL_DEV/.gitleaks.toml"; push_feature allow-all; H1="$H" +gl_checkout "$H1"; gl_step scan +gl_expect_pass "N2-1 前提：第一次推送只改配置、无密钥" EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA=$H1 DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch +printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env"; push_feature secret; H2="$H" +ARGS="EVENT_NAME=push PUSH_BEFORE=$H1 HEAD_SHA=$H2 DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch" +gl_checkout "$H2" +gl_step scan; gl_expect_block "N2-1 feature 先推全部放行配置再推合成 PAT（范围扫描）" leak:github-pat:k.env $ARGS +gl_step tree; gl_expect_block "N2-1 feature 先推全部放行配置再推合成 PAT（工作树扫描）" leak:github-pat:k.env $ARGS +gl_step scan + +# N2-2：先推指纹忽略列表，再推密钥 +main_base n22; printf 'k.env:github-pat:1\n' >"$GL_DEV/.gitleaksignore"; push_feature ignore; H1="$H" +printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env"; push_feature secret; H2="$H" +gl_checkout "$H2" +gl_expect_block "N2-2 feature 先推 .gitleaksignore 指纹再推合成 PAT" leak:github-pat:k.env \ + EVENT_NAME=push PUSH_BEFORE=$H1 HEAD_SHA=$H2 DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch + +# N2-3：PR 目标为非默认分支 feature（tip 已全部放行） +main_base n23; printf '%s\n' "$ALLOW_ALL" >"$GL_DEV/.gitleaks.toml"; push_feature allow-all; H1="$H" +git -C "$GL_DEV" checkout -q -b topic +printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env"; git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm pr +git -C "$GL_DEV" push -q origin HEAD:topic; H2="$(git -C "$GL_DEV" rev-parse HEAD)" +gl_checkout "$H2" +gl_expect_block "N2-3 PR 进非默认分支（其 tip 全部放行）" leak:github-pat:k.env \ + EVENT_NAME=pull_request PR_BASE_SHA=$H1 PR_BASE_REF=feature HEAD_SHA=$H2 DEFAULT_BRANCH=main REF_NAME=2/merge REF_TYPE=branch + +# N2-4：tag 推送 +main_base n24; printf '%s\n' "$ALLOW_ALL" >"$GL_DEV/.gitleaks.toml"; printf 'GITHUB_TOKEN=%s\n' "$(gl_fake_ghp)" >"$GL_DEV/k.env" +push_feature tagged; gl_checkout "$H" +gl_expect_block "N2-4 tag 推送（全部放行配置 + 合成 PAT）" leak:github-pat:k.env \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA=$H DEFAULT_BRANCH=main REF_NAME=v9.9.9 REF_TYPE=tag + +# 对照 C1：origin/main 的按值 allowlist 对 feature 推送生效 +tok="$(gl_fake_ghp)" +gl_new_origin cc1 +printf '\n[[allowlists]]\ndescription = "test control"\nregexTarget = "secret"\nregexes = [%s]\n' "'''^${tok}\$'''" >>"$GL_DEV/.gitleaks.toml" +git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm base; git -C "$GL_DEV" push -q origin HEAD:main; git -C "$GL_DEV" checkout -q -b feature +printf 'hello\n' >"$GL_DEV/README.md"; push_feature one; H1="$H" +printf 'GITHUB_TOKEN=%s\n' "$tok" >"$GL_DEV/k.env"; push_feature two; H2="$H" +gl_checkout "$H2" +gl_expect_pass "对照 C1：origin/main 配置中的 allowlist 对 feature 推送生效" \ + EVENT_NAME=push PUSH_BEFORE=$H1 HEAD_SHA=$H2 DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch + +# 对照 C2：push 到 main，before 上的按值 allowlist 生效 +gl_new_origin cc2 +printf '\n[[allowlists]]\ndescription = "test control"\nregexTarget = "secret"\nregexes = [%s]\n' "'''^${tok}\$'''" >>"$GL_DEV/.gitleaks.toml" +git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm base; git -C "$GL_DEV" push -q origin HEAD:main; H1="$(git -C "$GL_DEV" rev-parse HEAD)" +printf 'GITHUB_TOKEN=%s\n' "$tok" >"$GL_DEV/k.env"; git -C "$GL_DEV" add -A; git -C "$GL_DEV" commit -qm two +git -C "$GL_DEV" push -q origin HEAD:main; H2="$(git -C "$GL_DEV" rev-parse HEAD)" +gl_checkout "$H2" +gl_expect_pass "对照 C2：push 到 main 用 before 上的配置" EVENT_NAME=push PUSH_BEFORE=$H1 HEAD_SHA=$H2 DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch + +# E1：需要 origin/main 但不存在 +main_base e1; printf 'x\n' >"$GL_DEV/a.txt"; push_feature one; H1="$H" +printf 'y\n' >"$GL_DEV/a.txt"; push_feature two; H2="$H" +gl_checkout "$H2"; git -C "$GL_WS" update-ref -d refs/remotes/origin/main +gl_expect_block "E1 push 到 feature、缺少 origin/main 时失败（不回退到 before）" 'error:策略须取自' \ + EVENT_NAME=push PUSH_BEFORE=$H1 HEAD_SHA=$H2 DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch + +# E2：缺少 EVENT_NAME +gl_checkout "$H2" +gl_expect_block "E2 缺少 EVENT_NAME 时失败" 'error:缺少 EVENT_NAME' \ + EVENT_NAME= PUSH_BEFORE=$H1 HEAD_SHA=$H2 DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch +gl_finish diff --git a/.github/tests/test-gitleaks-range.sh b/.github/tests/test-gitleaks-range.sh new file mode 100755 index 00000000..a11a9817 --- /dev/null +++ b/.github/tests/test-gitleaks-range.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# SEC-01A 审查 P2-3：首推 / force-push 时扫描范围必须覆盖所有新提交（不能退化成只扫 HEAD 一个提交）， +# 无法确定完整范围时必须失败。执行目标版本工作流中 "Scan new commits" 步骤本身（含其调用的 .github/scripts/）。 +# 用法: .github/tests/test-gitleaks-range.sh [--rev ] （对修复前版本运行应报 FAIL） +# 每个拦截用例都声明期望的失败原因：leak::（检出）或 error:（无法确定范围而出错）。 +# R6/R7（review3h P3-i）：REF_TYPE 只接受 branch / tag（gitleaks-range.sh 与 gitleaks-base-config.sh）。 +source "$(dirname "${BASH_SOURCE[0]}")/lib-gitleaks-workflow.sh" +gl_parse_args "$@" +ZERO=0000000000000000000000000000000000000000 + +# 远端已有 main（2 个干净提交）；feature 分支 3 个新提交，密钥只在第 1 个（非 HEAD）提交中 +setup_feature() { + gl_new_origin "$1" + gl_commit README.md 'hello' init; gl_commit README.md 'hello 2' main-2 + git -C "$GL_DEV" push -q origin HEAD:main + git -C "$GL_DEV" checkout -qb feature + gl_commit deploy.env "GITHUB_TOKEN=$(gl_fake_ghp)" "feat-1 (secret)" + gl_commit deploy.env 'GITHUB_TOKEN=' "feat-2 remove secret" + gl_commit notes.txt 'done' "feat-3" + git -C "$GL_DEV" push -q origin feature + HEAD_SHA="$(git -C "$GL_DEV" rev-parse HEAD)" +} + +# R1 新分支首推（before 全零） +setup_feature r1; gl_checkout "$HEAD_SHA" +gl_expect_block "R1 首推 feature（3 个提交，密钥在非 HEAD 提交）" leak:github-pat:deploy.env \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch + +# R2 force-push（before 是被覆盖的旧提交，checkout 后不在历史中） +setup_feature r2; gl_checkout "$HEAD_SHA" +gl_expect_block "R2 force-push feature（before 不在历史中）" leak:github-pat:deploy.env \ + EVENT_NAME=push PUSH_BEFORE="$(gl_rand 'a-f0-9' 40)" HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch + +# R3 无法确定范围：缺少默认分支信息 / 没有 origin/main / 与默认分支无共同祖先 +setup_feature r3; gl_checkout "$HEAD_SHA" +gl_expect_block "R3a 首推但 DEFAULT_BRANCH 为空" 'error:DEFAULT_BRANCH 为空' \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH= REF_NAME=feature REF_TYPE=branch +git -C "$GL_WS" update-ref -d refs/remotes/origin/main +gl_expect_block "R3b 首推但缺少 origin/main" 'error:缺少 origin/main' \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch +gl_new_origin r3c +gl_commit a.txt a init; git -C "$GL_DEV" push -q origin HEAD:main +git -C "$GL_DEV" checkout -q --orphan orphan; git -C "$GL_DEV" rm -rqf . +gl_commit .gitleaks.toml '' o-1; gl_commit k.env "AWS_KEY=$(gl_fake_aws)" "o-2 (secret)"; gl_commit k.env 'AWS_KEY=' o-3 +git -C "$GL_DEV" push -q origin orphan; HEAD_SHA="$(git -C "$GL_DEV" rev-parse HEAD)"; gl_checkout "$HEAD_SHA" +gl_expect_block "R3c 首推与 main 无共同祖先的分支" 'error:没有共同祖先' \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=orphan REF_TYPE=branch + +# R4 新仓库首推默认分支本身：密钥在非 HEAD 提交 -> 全量扫描 +gl_new_origin r4 +gl_commit a.txt a init; gl_commit k.env "GITHUB_TOKEN=$(gl_fake_ghp)" "secret"; gl_commit k.env 'GITHUB_TOKEN=' clean +git -C "$GL_DEV" push -q origin HEAD:main; HEAD_SHA="$(git -C "$GL_DEV" rev-parse HEAD)"; gl_checkout "$HEAD_SHA" +gl_expect_block "R4 首推默认分支 main 本身（全量扫描）" leak:github-pat:k.env \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=branch + +# R5 PR 事件缺少/无效 base sha +setup_feature r5; gl_checkout "$HEAD_SHA" +gl_expect_block "R5 pull_request 的 base sha 不在历史中" 'error:PR base sha' \ + EVENT_NAME=pull_request PR_BASE_SHA="$(gl_rand 'a-f0-9' 40)" PUSH_BEFORE= HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=1/merge + +# 对照：正常 push（before 有效）仍检出；干净的首推 / 首推默认分支不误报 +setup_feature c1; BEFORE="$(git -C "$GL_DEV" rev-parse main)"; gl_checkout "$HEAD_SHA" +gl_expect_block "对照 正常 push before..HEAD 含密钥" leak:github-pat:deploy.env \ + EVENT_NAME=push PUSH_BEFORE="$BEFORE" HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=feature REF_TYPE=branch +gl_new_origin c2 +gl_commit README.md 'hello' init; git -C "$GL_DEV" push -q origin HEAD:main +git -C "$GL_DEV" checkout -qb clean; gl_commit a.txt a c-1; gl_commit b.txt b c-2 +git -C "$GL_DEV" push -q origin clean; HEAD_SHA="$(git -C "$GL_DEV" rev-parse HEAD)"; gl_checkout "$HEAD_SHA" +gl_expect_pass "对照 干净 feature 首推" \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=clean REF_TYPE=branch +gl_expect_pass "对照 HEAD 已在 main 上的新分支（空范围）" \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA="$(git -C "$GL_DEV" rev-parse main)" DEFAULT_BRANCH=main REF_NAME=copy REF_TYPE=branch + +# R6（review3h P3-i）：REF_TYPE 只接受 branch / tag，空值（GitHub 未提供）和大小写/拼写变体都必须失败， +# 不能再默认当作 branch（否则 REF_TYPE=Tag 的默认分支同名 tag 会被当成“推送默认分支本身”） +setup_feature r6; BEFORE="$(git -C "$GL_DEV" rev-parse main)"; gl_checkout "$HEAD_SHA" +gl_expect_block "R6a push 缺少 REF_TYPE（空值）" 'error:REF_TYPE=<空> 无效' \ + EVENT_NAME=push PUSH_BEFORE="$BEFORE" HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=feature +for rt in Tag BRANCH tags 'branch '; do + gl_expect_block "R6 push REF_TYPE='${rt}'" "error:REF_TYPE=${rt%% } *无效" \ + EVENT_NAME=push PUSH_BEFORE="$BEFORE" HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=feature "REF_TYPE=$rt" +done +gl_expect_block "R6 首推 REF_TYPE=Tag 且 REF_NAME 与默认分支同名" 'error:REF_TYPE=Tag 无效' \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=main REF_TYPE=Tag +# R7（review3h P3-i）：gitleaks-base-config.sh 单独调用时同样只接受 branch / tag +if gl_has .github/scripts/gitleaks-base-config.sh; then + gl_src .github/scripts/gitleaks-base-config.sh >"$GL_TMP/bc.sh" + rm -f "$GL_TMP/gitleaks-config-check.py" + if gl_has .github/scripts/gitleaks-config-check.py; then gl_src .github/scripts/gitleaks-config-check.py >"$GL_TMP/gitleaks-config-check.py"; fi + for rt in '' Tag BRANCH tag branch; do + rc=0; ( cd "$GL_WS" && env -u REF_TYPE EVENT_NAME=push DEFAULT_BRANCH=main REF_NAME=feature ${rt:+REF_TYPE=$rt} \ + bash "$GL_TMP/bc.sh" "$BEFORE..$HEAD_SHA" "$GL_TMP/r7-out-$rt" ) >"$GL_TMP/r7.log" 2>&1 || rc=$? + case "$rt" in + tag|branch) + if [ "$rc" -eq 0 ] && [ -s "$GL_TMP/r7-out-$rt/gitleaks.toml" ]; then echo "PASS [对照 R7 base-config REF_TYPE=${rt} 正常生成配置]" + else echo "FAIL [对照 R7 base-config REF_TYPE=${rt} 应正常生成配置] rc=${rc}: $(tail -1 "$GL_TMP/r7.log")"; GL_FAILS=$((GL_FAILS+1)); fi ;; + *) + if [ "$rc" -ne 0 ] && grep -F -q "REF_TYPE=${rt:-<空>} 无效" "$GL_TMP/r7.log"; then echo "PASS [R7 base-config REF_TYPE='${rt}' 拒绝]" + else echo "FAIL [R7 base-config REF_TYPE='${rt}' 应以 REF_TYPE 无效失败] rc=${rc}: $(tail -1 "$GL_TMP/r7.log")"; GL_FAILS=$((GL_FAILS+1)); fi ;; + esac + done +else + echo "FAIL [R7] 目标版本没有 .github/scripts/gitleaks-base-config.sh"; GL_FAILS=$((GL_FAILS+1)) +fi +# 对照：REF_TYPE=tag 的首推（非默认分支）按 merge-base 确定范围，照常检出 +gl_expect_block "对照 tag 首推（REF_TYPE=tag）" leak:github-pat:deploy.env \ + EVENT_NAME=push PUSH_BEFORE=$ZERO HEAD_SHA="$HEAD_SHA" DEFAULT_BRANCH=main REF_NAME=v1.0.0 REF_TYPE=tag +gl_finish diff --git a/.github/tests/test-hook-policy-files.sh b/.github/tests/test-hook-policy-files.sh new file mode 100755 index 00000000..d68fc302 --- /dev/null +++ b/.github/tests/test-hook-policy-files.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# review3g P3-1：.githooks/pre-commit 的 gitleaks 策略（.gitleaks.toml / .gitleaksignore）只取暂存区版本。 +# 修复前：钩子用工作区的 .gitleaks.toml，gitleaks 还会自动读取工作区的 .gitleaksignore——未跟踪或未暂存的 +# 放行配置/指纹在提交时生效，却不进入提交。 +# +# A1 未跟踪的 .gitleaks.toml（全部放行）+ 暂存合成 PAT → 拒绝提交（未跟踪），密钥不进提交 +# A2 已跟踪 .gitleaks.toml，工作区改成全部放行但未暂存 + 暂存 PAT → 拒绝提交（未暂存的修改） +# A3 未跟踪的 .gitleaksignore（含该 PAT 的指纹）+ 暂存 PAT → 拒绝提交（未跟踪） +# A4 已跟踪的空 .gitleaksignore，工作区加入指纹但未暂存 + 暂存 PAT → 拒绝提交（未暂存的修改） +# A5 已跟踪 .gitleaks.toml 在工作区被删除（未暂存） → 拒绝提交 +# review3h P3-a（git diff 不比较 skip-worktree / assume-unchanged 文件，旧钩子被绕过）： +# A6 .gitleaks.toml 标记 skip-worktree，工作区改成全部放行 + 暂存 PAT → 拒绝提交（索引标记） +# A7 .gitleaks.toml 标记 assume-unchanged，工作区改成全部放行 + 暂存 PAT → 拒绝提交 +# A8 .gitleaksignore 标记 skip-worktree，工作区加入指纹 + 暂存 PAT → 拒绝提交 +# A9 .gitleaksignore 标记 assume-unchanged，工作区加入指纹 + 暂存 PAT → 拒绝提交 +# A10 .gitleaksignore 标记 skip-worktree 但工作区与暂存区一致 → 仍拒绝（发现标记即拒绝） +# A11 已跟踪 .gitleaksignore 在工作区换成指向放行指纹的符号链接 → 拒绝提交 +# A12 暂存区 .gitleaks.toml 用 [extend] path 引用未跟踪的全部放行配置 → 拒绝（gitleaks-config-check.py） +# A13 同 A12，且工作区的 gitleaks-config-check.py 被改成直接 exit 0（未暂存）→ 仍拒绝（检查脚本取暂存区版本） +# A14 同 A12，且 [Extend] 大小写变体 → 拒绝 +# 对照 C1 暂存区 .gitleaks.toml 按值放行该 PAT（工作区一致） → 提交成功（证明用的是暂存区配置） +# 对照 C2 暂存 .gitleaksignore 指纹（工作区一致） → 提交成功 +# 对照 C3 无策略文件，暂存 PAT → gitleaks 检出，拒绝 +# 对照 C4 干净提交 → 成功 +# 对照 C5 暂存区 .gitleaks.toml 只用 [extend] useDefault（无 path），已跟踪文件无特殊标记 → 提交成功 +# 对照 C6 清除 skip-worktree 标记并还原工作区后（A6 的仓库） → 提交成功 +# +# 用法: .github/tests/test-hook-policy-files.sh [--rev ] （对 review3g 版本运行，A6–A14 应报 FAIL） +# 在 mktemp -d 的临时仓库中运行；合成 PAT 在运行时随机生成，不是真实凭据。 +set -euo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +REV="" +while [ $# -gt 0 ]; do + case "$1" in + --rev) REV="$2"; shift 2 ;; + *) echo "未知参数: $1" >&2; exit 2 ;; + esac +done +export PATH="$HOME/.local/bin:$PATH" +for c in git gitleaks; do command -v "$c" >/dev/null || { echo "缺少依赖: $c" >&2; exit 2; }; done +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +export GIT_CONFIG_GLOBAL="$TMP/gitconfig" GIT_CONFIG_NOSYSTEM=1 +export SKIP_VET=1 SKIP_BUILD=1 SKIP_GENERATE=1 +unset GITLEAKS_CONFIG GITLEAKS_CONFIG_TOML || true +git config --global user.name test; git config --global user.email test@example.invalid +git config --global init.defaultBranch main + +hook_src() { if [ -n "$REV" ]; then git -C "$REPO" show "$REV:.githooks/pre-commit"; else cat "$REPO/.githooks/pre-commit"; fi; } +CHECK_PY=.github/scripts/gitleaks-config-check.py +check_src() { if [ -n "$REV" ]; then git -C "$REPO" show "$REV:$CHECK_PY"; else cat "$REPO/$CHECK_PY"; fi; } +fake_token() { printf 'ghp_%s' "$(LC_ALL=C tr -dc 'A-Za-z0-9' [ ]...：初始提交（不经钩子） + local d="$TMP/$1"; shift; mkdir -p "$d/.githooks"; cd "$d"; git init -q . + hook_src >.githooks/pre-commit; chmod +x .githooks/pre-commit + mkdir -p .github/scripts; check_src >"$CHECK_PY" 2>/dev/null || rm -f "$CHECK_PY" # 目标版本的检查脚本（与仓库一致，随初始提交进入索引） + printf 'hello\n' >README.md + while [ $# -ge 2 ]; do printf '%s\n' "$2" >"$1"; shift 2; done + git add -A; git -c core.hooksPath=/dev/null commit -qm init + git config core.hooksPath .githooks +} +stage_secret() { TOK="$(fake_token)"; printf 'GITHUB_TOKEN=%s\n' "$TOK" >k.env; git add k.env; } +try_commit() { set +e; git commit -qm "$1" >"$TMP/$1.out" 2>&1; RC=$?; set -e; } +leaked() { git log --all -p >"$TMP/log.txt"; grep -qF "$TOK" "$TMP/log.txt"; } # 不用管道：pipefail + grep -q 会因 SIGPIPE 误判 + +FAILS=0 +fail() { echo "FAIL [$1] $2"; FAILS=$((FAILS+1)); } +expect_refuse() { # expect_refuse <名称> + if leaked; then fail "$1" "密钥进入了提交（commit rc=${RC}）-> 漏" + elif [ "$RC" -eq 0 ]; then fail "$1" "提交成功（应被拒绝）" + elif grep -qE "$2" "$TMP/$CASE.out"; then echo "PASS [$1] 拒绝提交（rc=${RC}）: $(grep -m1 -E "$2" "$TMP/$CASE.out")" + else fail "$1" "提交被拒但原因不符（期望 /$2/）: $(tail -1 "$TMP/$CASE.out")"; fi +} +expect_ok() { # expect_ok <名称> + if [ "$RC" -eq 0 ] && git log -1 --format=%s | grep -qx "$CASE"; then echo "PASS [$1] 提交成功" + else fail "$1" "提交未成功（rc=${RC}）: $(tail -2 "$TMP/$CASE.out" | tr '\n' ' ')"; fi +} + +CASE=a1; new_repo a1; printf '%s\n' "$ALLOW_ALL" >.gitleaks.toml; stage_secret; try_commit a1 +expect_refuse 'A1 未跟踪的全部放行 .gitleaks.toml' '\.gitleaks\.toml 未跟踪' +CASE=a2; new_repo a2 .gitleaks.toml "$DEFAULT_CFG"; printf '%s\n' "$ALLOW_ALL" >.gitleaks.toml; stage_secret; try_commit a2 +expect_refuse 'A2 .gitleaks.toml 未暂存的全部放行修改' '\.gitleaks\.toml 有未暂存的修改' +CASE=a3; new_repo a3; printf 'k.env:github-pat:1\n' >.gitleaksignore; stage_secret; try_commit a3 +expect_refuse 'A3 未跟踪的 .gitleaksignore 指纹' '\.gitleaksignore 未跟踪' +CASE=a4; new_repo a4 .gitleaksignore ''; printf 'k.env:github-pat:1\n' >.gitleaksignore; stage_secret; try_commit a4 +expect_refuse 'A4 .gitleaksignore 未暂存的指纹' '\.gitleaksignore 有未暂存的修改' +CASE=a5; new_repo a5 .gitleaks.toml "$DEFAULT_CFG"; rm .gitleaks.toml; TOK=none-a5; printf 'x\n' >a.txt; git add a.txt; try_commit a5 +expect_refuse 'A5 .gitleaks.toml 在工作区被删除（未暂存）' '\.gitleaks\.toml 有未暂存的修改' + +# review3h P3-a：skip-worktree / assume-unchanged 绕过 +FP='k.env:github-pat:1' +CASE=a6; new_repo a6 .gitleaks.toml "$DEFAULT_CFG"; git update-index --skip-worktree .gitleaks.toml +printf '%s\n' "$ALLOW_ALL" >.gitleaks.toml; stage_secret; try_commit a6 +expect_refuse 'A6 .gitleaks.toml skip-worktree + 工作区全部放行' '\.gitleaks\.toml 在索引中带有 skip-worktree.*: S' +CASE=a7; new_repo a7 .gitleaks.toml "$DEFAULT_CFG"; git update-index --assume-unchanged .gitleaks.toml +printf '%s\n' "$ALLOW_ALL" >.gitleaks.toml; stage_secret; try_commit a7 +expect_refuse 'A7 .gitleaks.toml assume-unchanged + 工作区全部放行' '\.gitleaks\.toml 在索引中带有 .*assume-unchanged.*: h' +CASE=a8; new_repo a8 .gitleaksignore ''; git update-index --skip-worktree .gitleaksignore +printf '%s\n' "$FP" >.gitleaksignore; stage_secret; try_commit a8 +expect_refuse 'A8 .gitleaksignore skip-worktree + 工作区指纹' '\.gitleaksignore 在索引中带有 skip-worktree.*: S' +CASE=a9; new_repo a9 .gitleaksignore ''; git update-index --assume-unchanged .gitleaksignore +printf '%s\n' "$FP" >.gitleaksignore; stage_secret; try_commit a9 +expect_refuse 'A9 .gitleaksignore assume-unchanged + 工作区指纹' '\.gitleaksignore 在索引中带有 .*assume-unchanged.*: h' +CASE=a10; new_repo a10 .gitleaksignore ''; git update-index --skip-worktree .gitleaksignore; TOK=none-a10 +printf 'x\n' >a.txt; git add a.txt; try_commit a10 +expect_refuse 'A10 .gitleaksignore skip-worktree（工作区一致）也拒绝' '\.gitleaksignore 在索引中带有 skip-worktree' +CASE=a11; new_repo a11 .gitleaksignore '' fp.txt "$FP"; rm .gitleaksignore; ln -s fp.txt .gitleaksignore; stage_secret; try_commit a11 +expect_refuse 'A11 .gitleaksignore 换成符号链接' '\.gitleaksignore 有未暂存的修改' +EXT_PATH="$(printf '[extend]\npath = "evil.toml"\n')" +CASE=a12; new_repo a12; printf '%s\n' "$ALLOW_ALL" >evil.toml; printf '%s\n' "$EXT_PATH" >.gitleaks.toml; git add .gitleaks.toml; stage_secret; try_commit a12 +expect_refuse 'A12 暂存区配置 [extend] path 引用未跟踪的放行配置' 'gitleaks-config-check\] ERROR: .*path' +CASE=a13; new_repo a13; printf '%s\n' "$ALLOW_ALL" >evil.toml; printf '%s\n' "$EXT_PATH" >.gitleaks.toml; git add .gitleaks.toml +if [ -f "$CHECK_PY" ]; then printf 'import sys\nsys.exit(0)\n' >"$CHECK_PY"; fi; stage_secret; try_commit a13 +expect_refuse 'A13 工作区检查脚本被改成 exit 0（未暂存）仍拒绝' 'gitleaks-config-check\] ERROR: .*path' +CASE=a14; new_repo a14; printf '%s\n' "$ALLOW_ALL" >evil.toml; printf '[Extend]\nPath = "evil.toml"\n' >.gitleaks.toml; git add .gitleaks.toml; stage_secret; try_commit a14 +expect_refuse 'A14 [Extend] Path 大小写变体' 'gitleaks-config-check\] ERROR: .*path' + +CASE=c1; new_repo c1; stage_secret +printf '%s\n\n[[allowlists]]\ndescription = "test control"\nregexTarget = "secret"\nregexes = [%s]\n' "$DEFAULT_CFG" "'''^${TOK}\$'''" >.gitleaks.toml +git add .gitleaks.toml; try_commit c1; expect_ok '对照 C1 暂存区配置按值放行（工作区一致）' +CASE=c2; new_repo c2; stage_secret; printf 'k.env:github-pat:1\n' >.gitleaksignore; git add .gitleaksignore; try_commit c2 +expect_ok '对照 C2 暂存的 .gitleaksignore 指纹（工作区一致）' +CASE=c3; new_repo c3; stage_secret; try_commit c3 +expect_refuse '对照 C3 无策略文件时暂存 PAT 被检出' 'gitleaks 在暂存内容中发现' +CASE=c4; new_repo c4; printf 'ok\n' >a.txt; git add a.txt; try_commit c4; expect_ok '对照 C4 干净提交' +CASE=c5; new_repo c5; printf '%s\n' "$DEFAULT_CFG" >.gitleaks.toml; git add .gitleaks.toml; printf 'ok\n' >a.txt; git add a.txt; try_commit c5 +expect_ok '对照 C5 暂存区配置只有 [extend] useDefault，通过检查脚本' +CASE=c6; cd "$TMP/a6"; git update-index --no-skip-worktree .gitleaks.toml; git checkout -q -- .gitleaks.toml; git rm -q --cached k.env; rm -f k.env +printf 'ok\n' >a.txt; git add a.txt; try_commit c6; expect_ok '对照 C6 清除 skip-worktree 并还原后（A6 的仓库）' + +echo "目标版本: ${REV:-工作区}；失败 $FAILS 项" +[ "$FAILS" -eq 0 ] diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml new file mode 100644 index 00000000..7b718a72 --- /dev/null +++ b/.github/workflows/gitleaks.yml @@ -0,0 +1,99 @@ +name: gitleaks + +on: + push: + pull_request: + +permissions: + contents: read + +# PR 的新推送可以取消旧的扫描（PR 最终以最新 head 为准）；push 不取消，否则被取消的 push 中的提交永远不会被补扫。 +# push 的 group 带 run_id 使每次运行独立：GitHub 会取消同组中“排队中”的旧运行（与 cancel-in-progress 无关）。 +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && 'pr' || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + # 官方发行版二进制（免费，不需要 gitleaks-action 的组织 license） + GITLEAKS_VERSION: 8.30.1 + GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb + # 同时扫描整个工作树。config/ 与 internal/ 下的硬编码默认密钥已在 sched/w1-mantou（SEC-22）中移除； + # 与该分支合并后的工作树经 gitleaks dir 扫描为 0 条。单独的 sched/w1-shaomai 未包含该修复，此步骤会失败。 + GITLEAKS_SCAN_TREE: "true" + +jobs: + gitleaks: + runs-on: ubuntu-latest + steps: + # 固定到完整 commit SHA（v4.4.0，git ls-remote https://github.com/actions/checkout refs/tags/v4.4.0 核对） + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 # 范围计算需要完整历史与 origin/<默认分支> + persist-credentials: false # 不把 GITHUB_TOKEN 写入 .git/config（本 job 不需要推送） + + # 固定到完整 commit SHA（v7.1.0，git ls-remote https://github.com/actions/setup-python refs/tags/v7.1.0 核对） + # gitleaks-config-check.py 需要 Python 3.11+ 的 tomllib，不依赖 runner 镜像自带的 python3 版本 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.12" + + - name: Install gitleaks (pinned + sha256) + run: | + set -euo pipefail + tarball="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -sSfL -o "/tmp/${tarball}" "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${tarball}" + echo "${GITLEAKS_SHA256} /tmp/${tarball}" | sha256sum -c - + mkdir -p "$HOME/.local/bin" + tar -xzf "/tmp/${tarball}" -C "$HOME/.local/bin" gitleaks + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Scan new commits + env: + EVENT_NAME: ${{ github.event_name }} + PUSH_BEFORE: ${{ github.event.before }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_BASE_REF: ${{ github.base_ref }} + HEAD_SHA: ${{ github.sha }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + REF_NAME: ${{ github.ref_name }} + REF_TYPE: ${{ github.ref_type }} + run: | + set -euo pipefail + git config --global --add safe.directory "$PWD" + # 历史中已存在的密钥由 SEC-01B（轮换/是否重写历史）处理，CI 只扫描本次新增的提交， + # 避免存量历史阻塞所有流水线；新提交中出现任何密钥都会使 job 失败。 + # 范围计算见 .github/scripts/gitleaks-range.sh：首推 / force-push 时按默认分支的 merge-base 确定范围， + # 推送默认分支本身且无可用 before 时全量扫描；无法确定完整范围则失败（不再退化成只扫 HEAD）。 + range="$(bash .github/scripts/gitleaks-range.sh)" + # gitleaks 8.30.1 在 git log 报错（如无效范围）时仍以 0 退出，这里先校验范围可解析。 + git rev-list ${range} >/dev/null + echo "gitleaks scan range: ${range}" + # --diff-merges=first-parent：merge 提交按相对第一父提交的差异输出， + # 冲突解决 / evil merge 中引入的内容也会被扫描（默认 git log -p 不输出 merge 提交的差异）。 + # 配置与指纹忽略列表只取自默认分支上的提交，不信任本次提交中的版本：push 默认分支 / PR 进默认分支用范围起点， + # push 其他分支 / tag、PR 进其他分支一律用 origin/<默认分支>（扫描起点 before 可能是攻击者上一次推送的、 + # 已经把 .gitleaks.toml 改成全部放行的提交，review3g N2）。策略提交上没有则用默认规则 / 空列表。 + # 因此对这两个文件的修改要等合入默认分支后才对后续扫描生效（见 .github/scripts/gitleaks-base-config.sh）。 + bash .github/scripts/gitleaks-base-config.sh "${range}" "${RUNNER_TEMP}/gitleaks-base" + # --ignore-gitleaks-allow：源码中的 "gitleaks:allow" 行内注释不豁免任何发现。 + gitleaks git --redact --no-banner --verbose --exit-code 1 --ignore-gitleaks-allow \ + --config "${RUNNER_TEMP}/gitleaks-base/gitleaks.toml" \ + --gitleaks-ignore-path "${RUNNER_TEMP}/gitleaks-base/gitleaksignore" \ + --report-format json --report-path "${RUNNER_TEMP}/gitleaks-commits.json" \ + --log-opts="--diff-merges=first-parent ${range}" . + + - name: Scan working tree (HEAD) + if: env.GITLEAKS_SCAN_TREE == 'true' + # 与上一步使用同一份基准配置（上一步失败时本步骤不会运行） + run: | + set -euo pipefail + cfg="${RUNNER_TEMP}/gitleaks-base/gitleaks.toml" + ign="${RUNNER_TEMP}/gitleaks-base/gitleaksignore" + if [ ! -f "$cfg" ] || [ ! -f "$ign" ]; then + echo "ERROR: 缺少基准 gitleaks 配置（应由 Scan new commits 步骤生成）" >&2 + exit 1 + fi + if [ -e .gitleaksignore ]; then mv .gitleaksignore "${RUNNER_TEMP}/gitleaks-base/head.gitleaksignore"; fi + gitleaks dir --redact --no-banner --verbose --exit-code 1 --ignore-gitleaks-allow \ + --config "$cfg" --gitleaks-ignore-path "$ign" \ + --report-format json --report-path "${RUNNER_TEMP}/gitleaks-tree.json" . diff --git a/.gitignore b/.gitignore index d2b82b55..ae225246 100644 --- a/.gitignore +++ b/.gitignore @@ -119,4 +119,12 @@ internal/model/kubeedge_types.go # AI-CloudOps executable AI-CloudOps -.cursor \ No newline at end of file +.cursor +# SEC-01A: 本地集群凭据与环境变量文件（含备份如 .env.bak-*），禁止提交 +/deploy/kubernetes/config +.env.* +# 模板保持跟踪（必须放在上面的忽略规则之后）。只逐个放行被跟踪的模板（SEC-01A 审查 P3）： +# 不用 "!*.example"，以免把 .env.prod.example 之类被 .env.* 忽略的文件重新放出来。新增模板需在此显式登记。 +!/env.example +!/deploy/release/env.example +!/deploy/kubernetes/config.example diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 00000000..08b1f559 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,18 @@ +# gitleaks 指纹忽略列表（SEC-01A）。 +# +# 当前无条目，请保持为空。 +# +# 2026-10-24（第三轮审查 P2-1）：原先 internal/middleware/login_test.go 中 RFC 6455 示例 nonce 的 4 条指纹 +# （工作树 path:rule:109/130；历史 24b8315:...:106/127）已删除。指纹按“文件+规则+行号”放行， +# 同一行换成真实密钥也会被放行；现改为 .gitleaks.toml 中按值精确匹配的 [[allowlists]]。 +# 实测：删除指纹 + 加 allowlist 后，工作树扫描、以 HEAD 配置做的全量历史扫描均不再报出这两处（证据见 review3/p2-1-measure.log）。 +# +# 已知限制（review3b）：CI 全量路径（push 默认分支且 before 全零/不可达，范围 = 全部历史）没有可信基准， +# 工作流按设计使用 gitleaks 默认规则 + 空忽略列表，既不用 .gitleaks.toml 的 allowlist，也不加载本文件。 +# 因此历史提交 24b8315（login_test.go:106/127）、125bc27（:109/130，在 sched/w1-mantou）中的示例 nonce 在该路径下仍会报出， +# 在这里加 commit:path:rule:line 指纹也不会生效（.github/tests/test-gitleaks-history-fullscan.sh 实测），故不加。 +# 增量路径（PR / 普通 push）用 base 上的配置，且这两个提交合入 main 后不会再落在增量范围内。 +# +# 如确需新增条目：只允许 commit:path:rule:line 形式的单条历史指纹，写明原因并经评审； +# 不要用 path:rule:line（会放行该行今后的任何值），也不要按整个文件忽略。 +# 注意：CI 使用 base 分支上的本文件（PR 中对本文件的修改在合入前不生效）。 diff --git a/Dockerfile b/Dockerfile index a6bb3ba0..5479c92d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,8 @@ RUN --mount=type=cache,target=/go/pkg/mod \ FROM alpine:latest ENV TZ=Asia/Shanghai +# SEC-27：镜像默认按生产启动（ENV 未设置会拒绝启动）；本地/开发运行请在 compose 或 docker run -e 显式覆盖为 development +ENV ENV=production RUN apk --no-cache add ca-certificates tzdata wget curl bash python3 nodejs && \ ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ echo $TZ > /etc/timezone diff --git a/README.en.md b/README.en.md index 26d7e9cd..f1806cd0 100644 --- a/README.en.md +++ b/README.en.md @@ -108,26 +108,28 @@ AI-CloudOps/ - Node.js 22 and pnpm 9 (for the [web console](https://github.com/GoSimplicity/AI-CloudOps-web)) - Python 3.11 (optional, for the [AIOps service](https://github.com/GoSimplicity/AI-CloudOps-aiops)) -### 1. Start MySQL, Redis and Prometheus +### 1. Create `.env`, then start MySQL, Redis and Prometheus -The root `docker-compose.yaml` starts the development dependencies (the application services in it are commented out): +The root `docker-compose.yaml` starts the development dependencies (the application services in it are commented out). It reads `MYSQL_ROOT_PASSWORD` and `REDIS_PASSWORD` from `.env` and refuses to start if either is unset or empty, so create and fill `.env` **before** running Compose: ```bash git clone https://github.com/GoSimplicity/AI-CloudOps.git cd AI-CloudOps +cp env.example .env +# Edit .env and fill in at least (generate secrets with e.g. `openssl rand -hex 24`; never commit .env): +# MYSQL_ROOT_PASSWORD= # root password of the Compose MySQL +# REDIS_PASSWORD= # Compose Redis runs with --requirepass +# REDIS_ADDR=localhost:36379 # Compose Redis is published on 36379 +# MYSQL_ADDR=root:@tcp(localhost:3306)/cloudops?charset=utf8mb4&parseTime=True&loc=Local +# BOOTSTRAP_ADMIN_PASSWORD=<>= 12 chars> # creates the `admin` user on first start docker compose up -d ``` -This exposes MySQL on `3306`, Redis on `36379` and Prometheus on `9090`. The credentials in this file are development defaults only. +This exposes MySQL on `3306`, Redis on `36379` and Prometheus on `9090`, using the passwords from your `.env`. -### 2. Configure and run the backend +### 2. Run the backend ```bash -cp env.example .env -# Edit .env. With the dev Compose file, Redis listens on localhost:36379: -# REDIS_ADDR=localhost:36379 -# Set BOOTSTRAP_ADMIN_PASSWORD (>= 12 chars) to create the `admin` user on first start. -# Replace JWT_KEY1 and JWT_KEY2 with two freshly generated keys (see below). go run main.go ``` @@ -144,7 +146,7 @@ If you prefer hex, use `openssl rand -hex 32` (once per key). The API listens on `http://localhost:8889` (`SERVER_PORT`). `GET /health` is a liveness probe only; it does not mean MySQL, Redis or external services are ready. -Configuration precedence is **environment variables > config file > defaults**. `ENV=production` selects `config/config.production.yaml`; anything else uses `config/config.development.yaml`. You can also pass `--config `. +Configuration precedence is **environment variables > config file > defaults**. `ENV` is required: `ENV=production` (aliases `prod`, `prd`) selects `config/config.production.yaml`; `ENV=development` (alias `dev`) uses `config/config.development.yaml`. Values are trimmed and case-insensitive. If `ENV` is unset or unrecognised the backend refuses to start; set `ENV=development` in `.env` for local development. You can also pass `--config ` (ENV is still required). ### 3. Run the web console @@ -199,19 +201,20 @@ All settings can be provided as environment variables (see [`env.example`](env.e | Variable | Description | | --- | --- | -| `ENV` | `development` (default) or `production`. Production turns on the strict checks below. The value is trimmed and compared case-insensitively, so ` Production ` and `PRODUCTION` also count as production. | +| `ENV` | Required: `development`/`dev` or `production`/`prod`/`prd`, trimmed and case-insensitive. Unset, empty and unknown values, including `test`/`testing`, are rejected. | | `SERVER_PORT` | HTTP port, default `8889`. | | `SERVER_ALLOWED_ORIGINS` | Comma-separated list of exact browser origins allowed for CORS and WebSocket. Empty means same-origin only. | | `MYSQL_ADDR`, `REDIS_ADDR`, `REDIS_PASSWORD` | Database and Redis connection. | -| `JWT_KEY1`, `JWT_KEY2`, `JWT_EXPIRATION` | JWT signing keys and settings. `JWT_KEY1` and `JWT_KEY2` are required in every environment: at least 32 bytes each, distinct, and not a repository sample value (generate with `openssl rand -base64 48`, or `openssl rand -hex 32` for hex). `JWT_EXPIRATION` is the access-token lifetime in **minutes** (not seconds); the default and the release configuration are `30`. A value such as `3600` means 60 hours. The upper limit is 1440 minutes (24 hours): above it the backend refuses to start with `ENV=production` and logs a warning but keeps starting in other environments. A value of 0 or below logs a warning and falls back to the default of 30. Expired access tokens are renewed automatically by the frontend with the refresh token (valid for 7 days). | +| `JWT_KEY1`, `JWT_KEY2`, `JWT_EXPIRATION` | JWT signing keys and settings. `JWT_KEY1` and `JWT_KEY2` are required in every environment: at least 32 bytes each, distinct, and not a repository sample value (generate with `openssl rand -base64 48`, or `openssl rand -hex 32` for hex). `JWT_EXPIRATION` is the access-token lifetime in **minutes** (not seconds); the default and the release configuration are `30`. A value such as `3600` means 60 hours. The upper limit is 1440 minutes (24 hours): above it production modes (`production`/`prod`/`prd`) refuse to start, while development modes (`development`/`dev`) log a warning and continue. Other `ENV` values are rejected before configuration loading. A value of 0 or below logs a warning and falls back to the default of 30. Expired access tokens are renewed automatically by the frontend with the refresh token (valid for 7 days). | | `CRON_ALLOW_LOCAL_EXEC` | `cron.allow_local_exec`, default `false`. Allows command (type 2) and script (type 4) jobs to run on the backend host. Read at startup; restart the backend after changing it. See the notes below. | -| `JWT_ISSUER` | Optional. The token issuer is configured as `jwt.issuer` in `config/*.yaml`; set this variable only to override it. The value must not be empty (the backend fails to start otherwise), and changing it invalidates all tokens already issued. After upgrading to the release that requires an issuer (`iss`), all older refresh tokens without `iss` are invalid and every user has to log in again once. | +| `JWT_ISSUER` | Optional. The token issuer is configured as `jwt.issuer` in `config/*.yaml`; the release Compose file supplies the same default, and other deployments without a config file must provide this variable. The value must not be empty (the backend fails to start otherwise), and changing it invalidates all tokens already issued. After upgrading to the release that requires an issuer (`iss`), all older refresh tokens without `iss` are invalid and every user has to log in again once. | | `TREE_PASSWORD_ENCRYPTION_KEY` | Key used to encrypt stored host passwords. Keep it safe: changing it breaks decryption of existing credentials. | -| `BOOTSTRAP_ADMIN_PASSWORD` | Creates the `admin` account on the first start with an empty database (min. 12 characters). Remove it after the first successful start. Existing users are never reset. | +| `BOOTSTRAP_ADMIN_PASSWORD` | Creates the `admin` account on the first start with an empty database (min. 12 characters). Remove it after the first successful start. Existing users are never reset. Too short: startup fails in production; outside production a `[SECURITY]` warning is logged and creation is skipped. The variable is removed from the process environment right after it is read. | | `SSH_KNOWN_HOSTS` | Path to an operator-verified OpenSSH `known_hosts` file used for web terminal and SSH connections. | | `AIOPS_BASE_URL` | URL of the AIOps service, e.g. `http://aiops:8080`. Enables the `/api/v1/*` gateway. | | `AIOPS_API_KEY` | Shared service key sent to the AIOps service as `X-API-Key`. Must match the Python service. Never put it in frontend config. | -| `MOCK_ENABLED` | Seed mock data for development. Must be `false` in production. | +| `MOCK_ENABLED` | Seed mock data for development. Defaults to `true` in development and `false` in production; enabling it in production is rejected at startup. | +| `MOCK_ADMIN_PASSWORD` | Password for the mock `admin` account (min. 12 characters), only used when mock is enabled and the user table is empty. If unset, no mock admin is created. There is no built-in password. Too short: error in production, warning and skip elsewhere. Removed from the process environment right after it is read. | | `LLM_API_KEY`, `LLM_BASE_URL`, `ALIYUN_ACCESS_KEY_ID`, `ALIYUN_ACCESS_KEY_SECRET`, `TAVILY_API_KEY` | External service credentials (environment variables only). | | `NOTIFICATION_EMAIL_*`, `NOTIFICATION_FEISHU_*` | Email (SMTP) and Feishu notification channels. | | `WEBHOOK_*` | Settings for the alert webhook service in `cmd/webhook`. `WEBHOOK_PORT` defaults to `8889` (previously `8888`), matching `config/webhook.yaml`, and must be a number between 1 and 65535. Integer fields accept decimal integers only (no `10.5`, `010`, `0x10`, `+3`) and are bounded: `WEBHOOK_FIXED_WORKERS` 1–1024, `WEBHOOK_DEFAULT_UPGRADE_MINUTES` 1–10080, `WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS` 1–86400, `WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS` 1–600 (0, i.e. no timeout, is rejected); leading/trailing ASCII spaces, tabs and CR/LF (only) are trimmed from integer variables and the port. `WEBHOOK_IM_FEISHU_GROUP_MESSAGE_API` defaults to `https://open.feishu.cn/open-apis/bot/v2/hook` (trailing slashes are ignored). URL settings (`front_domain`, `backend_domain`, `alert_manager_api`, `im_feishu.group_message_api`, `im_feishu.tenant_access_token_api`) are trimmed of surrounding whitespace; all but the token API also drop trailing `/`. The config file must be YAML (`.yaml`/`.yml`; `.json` is parsed as YAML); other formats fail startup. Setting `WEBHOOK` or `WEBHOOK_IM_FEISHU` to a non-empty value fails startup. Scalar settings written as a map/list, `im_feishu` written as a scalar/list, alias keys, quoted or tagged `<<` (yaml.v3 treats quoted `"<<"` as a plain key but `!!merge "<<"` as a merge, so neither is supported), merge chains or nesting deeper than 64, keys tagged with anything other than `!!str` (e.g. `!!binary`), NUL characters, multiple YAML documents, dotted key names, case-duplicate keys and leaf tags other than `!!str`/`!!int`/`!!null` fail startup; parse errors only report a line number and a fixed reason; unknown keys under `webhook:` are ignored with a warning. In Kubernetes, a Service named `webhook` injects `WEBHOOK_PORT=tcp://…` and makes startup fail; set `enableServiceLinks: false` or rename the Service. Environment variables override `config/webhook.yaml`, which overrides defaults; `WEBHOOK_MYSQL_ADDR` (full DSN) is required. See [docs/release/w1-webhook-env.md](docs/release/w1-webhook-env.md). | @@ -226,6 +229,7 @@ With `ENV=production` the backend additionally refuses to start unless: - `TREE_PASSWORD_ENCRYPTION_KEY`, `JWT_ISSUER` and `AIOPS_API_KEY` (when set) have no leading/trailing whitespace, control characters or invisible format characters (they are not trimmed automatically). - `MOCK_ENABLED=false`. - `AIOPS_API_KEY` is at least 32 characters whenever `AIOPS_BASE_URL` is set. +- No `admin` account still uses the built-in mock password shipped before SEC-27 (outside production this is logged as a `[SECURITY]` warning instead). See [docs/security-baseline.md](docs/security-baseline.md) for remediation. Other security behaviour to be aware of: diff --git a/README.md b/README.md index d18e6dbc..0e2c86a1 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,8 @@ GOTOOLCHAIN=auto go test ./... GOTOOLCHAIN=auto go build . ``` +提交前密钥扫描：运行 `bash scripts/setup-git-hooks.sh` 把 `core.hooksPath` 设为 `.githooks`，并检查本机依赖。需要 **gitleaks >= 8.25.0**（从 [官方发布页](https://github.com/gitleaks/gitleaks/releases) 下载并校验 sha256；版本过低或无法解析时脚本与 pre-commit 钩子都会失败），以及 **Python 3.11+**（提交 `.gitleaks.toml` 时钩子用 `tomllib` 检查配置，缺少时拒绝提交）。本地钩子只是第一道防线，CI 的 gitleaks 工作流是最终兜底。 + 前端安装、开发端口、类型检查与构建以 [前端 README](https://github.com/GoSimplicity/AI-CloudOps-web#readme) 为准；AIOps 的环境与验证方式见 [生产配置说明](https://github.com/GoSimplicity/AI-CloudOps-aiops/blob/main/docs/production.md)。开发进程连接容器依赖时，可使用部署指南中的只绑定回环地址的 `local-ports.yaml`，不要直接公开数据库端口。 ## 生产部署 @@ -295,16 +297,18 @@ AI-CloudOps-aiops/ 生产环境设置 `ENV=production`、`MOCK_ENABLED=false`，并分别注入独立的 `JWT_KEY1`、`JWT_KEY2`（各至少32字节）、`TREE_PASSWORD_ENCRYPTION_KEY`（恰好32字节），均不能使用仓库示例值。 -`ENV` 去首尾空白、忽略大小写后等于 `production` 即视为生产环境（如 ` Production `、`PRODUCTION`）。 +`ENV` 必须显式设置；去首尾空白、忽略大小写后，`production`/`prod`/`prd` 为生产环境，`development`/`dev` 为开发环境。未设置、空值及其他取值（包括 `test`/`testing`）均拒绝启动。 生产环境下 `TREE_PASSWORD_ENCRYPTION_KEY`、`JWT_ISSUER`、`AIOPS_API_KEY` 也不能有首尾空白、控制字符或不可见格式字符（不会自动去除）。 `JWT_EXPIRATION` 小于等于 0 时记告警日志并使用默认值 30。 其中 JWT 两把密钥的长度、熵（不同字符至少10个）、首尾空白/控制字符与不可见格式字符、互不相同和示例值（尽力而为，只匹配前缀片段和已登记哈希）检查在所有环境都执行，不仅限生产环境。 首次空数据库启动使用 `BOOTSTRAP_ADMIN_PASSWORD`（至少12字符）创建管理员，成功后移除该变量；已有用户不会被重置。 +生产环境 `MOCK_ENABLED` 默认关闭，显式开启会拒绝启动；开发环境默认开启。mock 管理员口令只从 `MOCK_ADMIN_PASSWORD`（至少12字符）读取，未设置时不创建 mock 管理员。bootstrap 和 mock 口令过短时，生产环境拒绝启动，开发环境告警并跳过创建；读取后立即清除对应环境变量。 +启动时若检测到 `admin` 仍在使用历史 mock 内置口令，生产环境拒绝启动，开发环境告警，处置方式见 [安全基线](docs/security-baseline.md)。 `SERVER_ALLOWED_ORIGINS` 是逗号分隔的精确 Origin 白名单，留空仅支持同源浏览器访问。 `JWT_EXPIRATION` 是访问令牌有效期，单位是**分钟**（不是秒）；默认值和发布配置均为 `30`（30 分钟）。 -请勿按秒填写 `3600` 之类的值，否则令牌会有效 60 小时。上限为 1440 分钟（24 小时）：超过时 `ENV=production` 拒绝启动，其他环境记告警日志后继续启动。 +请勿按秒填写 `3600` 之类的值，否则令牌会有效 60 小时。上限为 1440 分钟（24 小时）：超过时生产模式（`production`/`prod`/`prd`）拒绝启动，开发模式（`development`/`dev`）记告警日志后继续启动；其他 `ENV` 取值在环境校验阶段即被拒绝。 访问令牌过期后，前端会用刷新令牌（有效期 7 天）自动续期。 -`JWT_ISSUER` 可选：签发者在 `config/*.yaml` 的 `jwt.issuer` 中配置，只有需要覆盖时才设置该变量。该值不能为空，否则后端启动失败； +`JWT_ISSUER` 可选：签发者在 `config/*.yaml` 的 `jwt.issuer` 中配置，发布 compose 会注入与 YAML 一致的默认值，其他无配置文件的部署必须注入该变量。该值不能为空，否则后端启动失败； **修改 issuer 会让所有已签发的令牌（访问令牌和刷新令牌）失效**，所有用户需要重新登录。 **升级提示**：升级到要求签发者（iss）的版本后，之前签发的、不带 iss 的旧刷新令牌全部失效，所有用户需要重新登录一次；之后正常续期。 diff --git a/deploy/kubernetes/config b/deploy/kubernetes/config deleted file mode 100644 index cf105197..00000000 --- a/deploy/kubernetes/config +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJSVVvMDZCMEplSFl3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRBNU16QXdPREEyTWpkYUZ3MHpOREE1TWpnd09ERXhNamRhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUREOW02ZWZQQytMazZ2clkwVEUyMVJieU9WZGMvVmR3c0FMZ011ci9zMlR0MzVYV2I4aFczT3lrQTEKNEVPYWQ5VmNrNExnK1VnMXdDV1ozMTNPbnFmRUFEQ25OWjFiRHJGbCs0Smhya2c4M0pUanlPZStqZmdNOXFWTQptczVoSjhnT3didmQ2WmdaOFl2bHowbHZGU3hEZGdHNXhIZGdqemZYU0FMRlhIS1hweVpWbHpqaWNWT1FNRWlrClM3clBoQUEyTnNLdDljeFVHUkI1OUMzN3poNks5MjdFclNJbUlKalZ0Ynl4MXJaVnBFSW1STGRPcFI2NWZlYnoKU2h6UlVwejVxTGMxZjl5TWlnWVRFY2tRZHFlYXFJYi9aQ1RTWmw3d09ZcHJIT3NianJaR21WN243QkVGU3lsUgplYzcvalRQWTFya3JLaEw2ZWJCeVBEWm1MckJQQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJReUp2SWYremlBQ0xlYzZlUkxvbU5IdG5SUkNqQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ1o5c0NvT28wOQpVWnRpc2w2TTVLcmh4UFhKcFhmbXdSZ0wxd3doSTVFZWgvNy9pTnBkZlEzVW11Q2RiYWJ2YUUyQnB0bnBzN0dEClM2bytZNGo5dGRpSS9jUEZ0ekhXMVNCWU95SFEwYWptMTlTbkJtempUZzZKRDNTTVJiVVEzT1Q3VUtJVWpTQ0UKSnBRbWJoU2N6aHJKVjN1QnFLR1hzRk1xTDZRcXVaUlI5SVFJRmhzcnhidWp5cnNiZXZJbkhoQUFyWkp3ZnlVQgpTbmVjVkVsMm5NWFF3MjlRMDZIRVJkMWxJcW9KMFpoNzN2MWhmUDAyYmxma0xaL2tIaVE2ZGhLWDQrc0hVams0CnNlaVg2N3JnT2ZrRytBUFNGMEhaVEZqeUpHd3h3ZDFlem9DakdZbTRZc2oxNjUrSXQxZGYzWjN3QStoRFlCWmMKQjVPdDljejh2K0JHCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - server: https://192.168.0.104:6443 - name: kubernetes - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJVWt4L2ZkQnhLbU13RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRFeE1EWXdNakEzTlRWYUZ3MHpOREV4TURRd01qRXlOVFZhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUURZaDA4RFF3S2lhTWtMUHBNbElHQXArS21KVERLYnBLa2Y3NTNxS1VOeDdmRGVDU3lZWlpKK1VSRUoKUEsvKy9BMTFtdUw1cnZTSVplZDE4MC9xSGhUZGxYL3dQSlVMeG1pN1hNWHU2TG01SW95a1ZzTUJlTXhNUUVnSQovU01vdnZhdTZZY3Rhd2JCSi9ZME11MDkrNS9rTVAvTXdDVG9seGlETkJnTzZLZVJidEYyTWJLc0VxWTBrc3A5CkpnNm0xdUxUZEk2YTlTZ00yVUc1YUJZSzR5MTJ0WWFtY2pmVVZiSGxkR3lQNU9YeDY4RDNFNnI5NHhVUFVjR2kKc3BFekxMZkZQWnFmZGg3bDJjUEtnY3BiV1FPRklva0dKaGVJT1lPZ3pFOFZpbG0wS1dBbmhwWkFkQXZIbnVwRApNaUpHaSt1ejRTejJ3N2dJRzZPeXA1MkVyODVmQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJSdGpIa0trL0I2bVRsNUc4ZFJMZEFReVN3QlRqQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQ2R4WDE4VWhiTgpBcnE2UVg2K1B6WEFTS2FWWFBsWVNxV3VTS1dYUm1ZTjhvTnJZenpMZGZ0TGxmbWExWUZqaFpKaDY5Z2dTUlZ5CjlQTGU0OHRETDhFSDVMNFFOTDAwL2tZcThyaEtGei9lVC9TY1dITUMyUGZKaHM2Q3B5QzVtS3h5b0JGTE9rZlQKQ2hVOTlIalhkU0NmUmw1TXpGQWdSRVpQWWhhVHRRQWR5clZvcTdSUXZ2aFNWcVBBTmNobWJtMU5aVGIrVjhYSgo3SzFQUngrVmoraEFCRXhuSGR6d1BsYUdUVTJ0cjJUSTNGRjhkejZPdTVuV3E3aEdiNUl5Vk9RZ09QdHh0MW9UCko4ZG5aU0RDNWxjVnZtRzVQdElHcFFtYVkwMWtzMC8xejQ5ZXhEbllwb3dYMlhXZWk2eW5MU08wOEpjUHJKSHYKTGY2MlpQQUNoOU1WCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - server: https://127.0.0.1:59623 - name: kind-kind - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkakNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTkRBMU5EQTNORFV3SGhjTk1qVXdNakkyTURNek1qSTFXaGNOTXpVd01qSTBNRE16TWpJMQpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTkRBMU5EQTNORFV3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFURlZmUE9BQXFZU2ZlcUJKQ1B2U3p0TFRvcFkrNS9oeWZkcTFLVEZEcmwKZUMwVTUrM1VRc0hpbjNlUE11S0R6dE9oYjNyM0licmRjL01zcUhObFpuU2FvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVTNkWHBkNGRpVHFucTN4bEZyOTFsClQvZFhESEF3Q2dZSUtvWkl6ajBFQXdJRFJ3QXdSQUlnT1hucmhtaHFnQTlWc28rb3krbmR4QS9YdG1reFVsREkKMWxRNERROFZkemdDSURER2JEdUxjdUVJSXMySkhKQXpycWpCVjVEczNQc0VmYTRoTHBkUkZjcTgKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - server: https://127.0.0.1:26443 - name: orbstack -contexts: - - context: - cluster: kubernetes - user: kubernetes-admin - name: kubernetes-admin@kubernetes - - context: - cluster: kind-kind - user: kind-kind - name: kind-kind - - context: - cluster: orbstack - user: orbstack - name: orbstack -current-context: orbstack -kind: Config -preferences: {} -users: - - name: kubernetes-admin - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURJVENDQWdtZ0F3SUJBZ0lJR3VmQ1haOEJYNzR3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRBNU16QXdPREEyTWpkYUZ3MHlOVEE1TXpBd09ERXhNekZhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXlFdmFXMURaTkpHUE01ZkMKb1E4Wnplb0I2Nm5DQ3J3VndpM3I4blJwZG5WY0xBNGdNWVlXZnVtWm5IMEgxWEplUmdtanhGYUVDUDM1aEtQegphLzZMSDkvTnkrK2NJZXFGc1Q3NVZ2bGc3NWZVQ0Fvd0VoU3JQSHVZSDllTUVKRlV1d2xkcmZUdGxQaVFoM0MwCnhxd2lORkVSbXRmTnIrWjJTeXFLczAydmFtOWpuZW5Jdk90MzJMdWFmVWxwUHdGaGR0bkwrdTI3WXJVcTVSZDEKazhxSTBuS0NJdTVQeGxORUlyQnpEN29SbldRZEt1U0tyZFQyemI0dHdjRWlDckpLRzMwQVRZbjFOcGF5RW03egpZVkdTbjRhOFRCV1AzSUxpWDZhYjkwbEVWTGU3b3J2dmVrbTFlbzA0eDZoRWJETHlkaTg4amFsZnRjcjBqK1hlCkw1RS9hUUlEQVFBQm8xWXdWREFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JReUp2SWYremlBQ0xlYzZlUkxvbU5IdG5SUgpDakFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBdkZBcElheW9TT0I4Y0oyMWp2TE1pMldmY20zRUtteG1GajFZCjYwV045TXlHTlRwM2dEbVpUWmlKRGN3TXYvNlNZUG5KalMwTVpMb3dmQkZoOXhGM3BnUEJKMEF2UkNkcGl0aFoKelVYYlZZQlkzK1p0bHBwenU3aU9WVllkbkg0ZW9tTTJJS2VDUHpTamNScloxbk52V3QrYTk3ejU2T0hOcDdVcgpkZEFXN1pRQ21LT1ZGTzVvWlk3QkdmVTNpTzJvN1o4OURvT01RSzZjMElKSHBRNkxqQTNrdHA4YVBFKytsZ2IwCjNEcjEzRmVGUHQyNmYvL1Zwd1laMjlGN0JIeUFUK2F1SWphTy9MTGdLVUt4STdrQkFBRXZuempQeXp2QnlHbzkKcE9HU0ZuRE5yTVlOWmdpWWZnbTBrc3dLaWpIdmxQaXNFanhYaUUrWFpnQzRFOXA0WkE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBeUV2YVcxRFpOSkdQTTVmQ29ROFp6ZW9CNjZuQ0Nyd1Z3aTNyOG5ScGRuVmNMQTRnCk1ZWVdmdW1abkgwSDFYSmVSZ21qeEZhRUNQMzVoS1B6YS82TEg5L055KytjSWVxRnNUNzVWdmxnNzVmVUNBb3cKRWhTclBIdVlIOWVNRUpGVXV3bGRyZlR0bFBpUWgzQzB4cXdpTkZFUm10Zk5yK1oyU3lxS3MwMnZhbTlqbmVuSQp2T3QzMkx1YWZVbHBQd0ZoZHRuTCt1MjdZclVxNVJkMWs4cUkwbktDSXU1UHhsTkVJckJ6RDdvUm5XUWRLdVNLCnJkVDJ6YjR0d2NFaUNySktHMzBBVFluMU5wYXlFbTd6WVZHU240YThUQldQM0lMaVg2YWI5MGxFVkxlN29ydnYKZWttMWVvMDR4NmhFYkRMeWRpODhqYWxmdGNyMGorWGVMNUUvYVFJREFRQUJBb0lCQVFDcm9ETnVZNHg2ZXU5VgpxZ2hmc1d6UEFHQzg2aTBXdmF0M1E1b1ZtcUp6bW9Sc1MzNVNjUzc5ZUhUam5rOEVHb2VsUThWTUMwWC8zbi9iCnBCQ0V6UXV4T0RoRE13RjZIbGFJVmdtWStQNlN6bW9rcVhZZlNBNmlPTlZWRTRFMUFSSzFZWVVmOWV0TjV0OFEKN3dZMzVtODRuTzZVMjYybnQ3Wk5HaHJYSVEzYUNDb3pVNktwRkNjUGJtMWZaY0ZseDBzMkFpTCttVHhCQ3BJdQo5R2pJUS9sLy84QUNPcWx5Wi91VUFEa2tPQ1FtQTNvNEVVeEdaaGx5WXpJYkRpOEhXYWtLY3Jkc0p4T05MMk9GCkVYbTlrdXB4aUR1bmVLMHdLU0oxY1U5QWhjNGxqakRjWDcwN1lJaTVtNFhEc3FxYmt6MGoxVEl4TG1maDVhVzgKQTNxRlhZQnhBb0dCQVBwVlA5RjFMNStPU094N01rQjZLMHluTjNvUUhPaDM0a05BVU1LUTJ4YWlXZGs0TXdGago4WmMvSWdra004ZHA0YWhLM1JsS3lFL1J3YnJNYW5BSkR4QlZFeVgydi95UUFGNDhsbEM3UjU5T296aG10ZU0zCnR3Y0hvSHRWaUhUR29ZeXdGOE9pbk9udVJFUm1semJYT3pENDBoYWFKaCtNNnJCTFJBMDhhSHd6QW9HQkFNelUKb0hUY1h6cW0zNzAxTUUwMFVrWlpjb281SzR3L0lSVGIrc3d3Mk8razYvZE13QnA3bkxZQnBSTVZWTlhIbU5rVAorNGl4U04xYnpzU3gra0RIWDdmZWQybEFUK2g1cEpvcVV2WHAzcWtpWUtOSGxlSkFKYkEwM2pudS9kQUlORkNQCnNpWnVST1JmeW1nNnFYTXg0NXBjbnh6cUFuUGFzaXVrdE5MY1ZEbnpBb0dCQU8yOVNINkQ3Rlo3cW9YcitpMkIKMk4xVGNUeGJVUmoxd2N4Y3FGWWZlL0ppL1RGdVRnSmtDR3k3YUhlR0NpYTRSN2FzWW81Q2x6bzIydVdzZk9rcApzVVN4aHgzbTJTM2pGSFpxMDlhWUJjMGx3WjB1N2s1Nyt6YVI1N2M1NC80REppbVdrdnNZMUN6V083ODZMeUhHCkJsRGIvYW01ZTdzNitTZTBVMHkrc2Z4QkFvR0JBSU16czNBSGRLeENGZERCa0NYejNMdVpNZ2dkNUtvYUNkdXQKcUxGQW5NU3NORVdkRVBRbHQ5VFJxdVpWWkpqbkdCMzhjY00ySkFFK2ZHeDd3RnZjR1pEU1hGUzcwRE9PTDRSYwpsZlZWRDczdytrdThYK0tqeWtCYkxQbVkvMVZRM0FtNmNaZXlURWlvbnlNeWFEWVVmOER4a1MzWkt5Y0FyOTNLCnk5VEJNdVpIQW9HQVdYc1VGK25QbWdHc1l5bHpUSzl6NStxNmloelI1enhvNUVCd2ZHTlpBRXBSMVphSHdVcXIKWGVzOFBSWWp0c3FORTBsdWJNOGs1QS9hM2Q4QzhBdU5IdjBXSFRTQUVlRUlqeWkxeTVoakJyaklhWXdPNGZmTgpOK0I3Rm9OOVJQbDdUWHRkckNmck1hbklYU3JLOXdKMlpXRm1LaWNJT1lVUzBZYlNaOHh2emhBPQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= - - name: kind-kind - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURLVENDQWhHZ0F3SUJBZ0lJRHNFcTVnRyt1VkV3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRFeE1EWXdNakEzTlRWYUZ3MHlOVEV4TURZd01qRXlOVFZhTUR3eApIekFkQmdOVkJBb1RGbXQxWW1WaFpHMDZZMngxYzNSbGNpMWhaRzFwYm5NeEdUQVhCZ05WQkFNVEVHdDFZbVZ5CmJtVjBaWE10WVdSdGFXNHdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFERDdMQTIKdUVDcE5meDBDdGE3RmFHWTF1YjF4cTdoRGdGTVhvTHk2NkxKbXBKT3lYUllHUFU1LzJHUlRId1UvRnVzSWYvQgphN2RzRjhWL0MvZStlKzZUR3BLOEZBR3ZsTmp4dkpEaENOVlRSTTR0bXpSR2lTTXo5cTJtMURWcW5GMDNDTjBzCjhXZXJIWmdjNVIyQ3A2WkYyMUdXTDIxVWU0UENvN2k5alZkNXJQNERHTWRCTkFYSVlGaXg1VWE2akd5NkpncTgKWXNDTms1Wko4Z0tIOFFIa0VtckZzWmV1NEI4MmxGUkdpWHN6Sjc2VVdvSE1TNWlJWGtSNWlrUmNJVEFvUFVxRgoyZEtiSnZxODN5RFdQVzVrVDNoMmh1Ri9ZWkppQ0xwVDJlYmRQSUxjR1lSczd0dlhlbXpBdXpVaU8xcUlBeitsClAvaSsyQmNNRDFQY0FQaUpBZ01CQUFHalZqQlVNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUsKQmdnckJnRUZCUWNEQWpBTUJnTlZIUk1CQWY4RUFqQUFNQjhHQTFVZEl3UVlNQmFBRkcyTWVRcVQ4SHFaT1hrYgp4MUV0MEJESkxBRk9NQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUURHWHJuUWduN2I4elJlYlZ4clpGTndpWktPCmFYaDZKTVNIbm0wNm9iYThkMDR2YVVCbzZwT01lOE9PNTcvclRNcWR4cXhFNTAxMlY0eXZBNng4RzgzY01KdDgKaGNkQ2ZlcjFQbE9YZ1RaVlBnV2hiVHJJSjc2M3hUQm1IOEJYT3hsQ1p4ZXAzS2xZWk9nSGxpeFJtRnYwZklMQgpwdERCdGE4R3BUcnRqaFZ5eWxHMjA4ajJwK3oxOE5Sc3A2bHdmRVZnenRFbTBFeFM2MFA5VjY1ZzhoR2M1UVpoCm5RSVhTSDVBZGxVYTYrajRuSndPOG01Rmk4NGFMbHFzR3czalJCeHZnUkxYUk81clBXWHpJL1NrL3ZpcytlTDEKbTVnMFVmQXJLQU4wc096bTFoTGR4YVZ5QUZUVU5XS204aTJCd0U3SXk2bXJ6VnZMVWJyVjNpR2s3aEZ6Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBdyt5d05yaEFxVFg4ZEFyV3V4V2htTmJtOWNhdTRRNEJURjZDOHV1aXlacVNUc2wwCldCajFPZjloa1V4OEZQeGJyQ0gvd1d1M2JCZkZmd3Yzdm52dWt4cVN2QlFCcjVUWThieVE0UWpWVTBUT0xaczAKUm9rak0vYXRwdFExYXB4ZE53amRMUEZucXgyWUhPVWRncWVtUmR0UmxpOXRWSHVEd3FPNHZZMVhlYXorQXhqSApRVFFGeUdCWXNlVkd1b3hzdWlZS3ZHTEFqWk9XU2ZJQ2gvRUI1QkpxeGJHWHJ1QWZOcFJVUm9sN015ZStsRnFCCnpFdVlpRjVFZVlwRVhDRXdLRDFLaGRuU215YjZ2TjhnMWoxdVpFOTRkb2JoZjJHU1lnaTZVOW5tM1R5QzNCbUUKYk83YjEzcHN3THMxSWp0YWlBTS9wVC80dnRnWERBOVQzQUQ0aVFJREFRQUJBb0lCQVFDYzdxTWUwV3NKbm5KKwpTSWhEQmtxUDcrTERqc2RaQVN6TkRRNzZvUCtkV0RCRTUxeEhqSVl3Vkh6RU0yMVlLZU1MOTVleVNDTjljM1VBCkZJZjJqYkpGSmczT2xIL2RNZTZyZ296Ums0Kzd5T3NVNExKNHBUUUxWVlUyd2RlZmMydSt2MXpadU90K3hvK20KNVdaRDF5RjU1dmhzd2NSaTNTUm03VmoyaTVZN29JeGhvVjl6S2dXc2VBbllkeXgzU3Z5WTBXaGZ6eENuSTUxQgovVUVTd1hDak5LVUdXUDZSNElDTDYwTGw1Z1RoOWtrMTh1S1BrWWJLSDlmRGw2MjNBTC9tSkZpZ3g1NUxJL25NCkhORUpBMjdYcnhUTCtVK2M3TXdvd0tKM2o0V210ZkZOdDFZeHhjS3l2OXB0R1FPRUVTY25oRG9zNDNkMDdaMGIKaWovQ1RIN2hBb0dCQU9UWE9EUWlrYUsrcXV3NVdQZTU2djZXWk1QeGdtQzFCbThOOGJNQm9aT2xVS3pUbS9nSgpuSFVaMWYyQ1V4UmYxTysycUJwa0RmRmw4MmZVVTBzNk0zZXc4WHNyZXdFK01MYmUrcjc3UWFaVVAvQjAyYW1UCkRwazQrakl6K0lWSGZIbXVTWkIvd08zZTNHQ1ZPeUIxb2p6SnBJYXRTdWdXU243M3ljVDFXOW10QW9HQkFOc3QKWStaMTdrRldVdVJ5T1F4WVJMYkxYbWpIQ1V3OWlhdFk3L1ZtL1VLcDB3MmxlSDJHQ2NRRCtIYzliS3FTeDJWSQpoRWIwZ1dPZldNeUpES0w2QVZtVzR2SkdXR0tSeVJ5Unl0ZU5PZk56V2dReFVtcFkrZDZzRFZYN0dacXJ5UWRNClZtODVpaTBENk1OcCtXUkpadHczWlBkZHNwOWhIMzhWeVdGUGVXM05Bb0dBR0gzSU1CdzdCZlh1Q1JZaUpYRXEKYTFEaE8rOVBDdGFVOTdIQVdtNGtRczhBa1Y2Y1pMRnlvejIrbjBFaGJ4N0toVlZCTElIazFCOGJLOU9Yam9lTwpGcE5EWlBGRVd0K3pDdjlXU3JaTlVtWFY2Z0EzZzJTUHZXcFJyS25QUVVSaldBcUZLUWZqT0JJUDkrNUF3N3FUClFIbzhONFc0YkpwbUlxeVdWWlFFM29rQ2dZQTNlTzUrNXJ3dGh6YWxvUTgxUTZYb2llSlVMSVA2NnR4TUpNOWUKMGZrcGhTZm9uVWU0cFZNVmJGZlhmaEZodnBKKzNQSzFycTZNMDBpN1E3aVNDeXFLVFRrVlRwNlNIQW5GbEZTOQpaMzRTVXRDbW5RRVo3M2tXVlg5dWtvWHhjcWNIbE5lUGdRV3F6UUY5YS9YMTN1b010R3gyZXgxNVh6Q0VqclFRClQvZ1F4UUtCZ1FES1FkSzhFalFKc3lWTUJ1ZmlJdGdaalVZK0h3YUFoMC9NeHNvZmxHYWJiUkNnckdFaUpwRmkKYVc0Y1M2dUV1SVlDcStZRVo5b3FJOHd2WUlMOGU0bGwwV1BCTDQwL2pqZFkrSmVzYmNmbmIxMW5vVCtOUXdqSQpYdWh1VER4WDBuVjJXNGE4Ni9vaUtQQnh1ZmFwSEhrWGRTckJjYmcyODFQRTJZd1VLYVQzWVE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= - - name: orbstack - user: - client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRlZ0F3SUJBZ0lJWThOM1lVM25JU0l3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOelF3TlRRd056UTFNQjRYRFRJMU1ESXlOakF6TXpJeU5Wb1hEVEkyTURJeQpOakF6TXpJeU5Wb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJJamYxYmR1Qkswa0ZST1QKbDQvVmFRR2h0Mnd5RFE3Q2lib1RwUWZiZTdObWdoMVR1a0F2eG8rcU9QOVkwMW1TWFgyRnBxQlRJM0VYT2t5cwp0WmRheFk2alNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCVHhxcS9TV3l4V0N3TEZxZ2hFT0RSTU80a0RGVEFLQmdncWhrak9QUVFEQWdOSkFEQkcKQWlFQXpFdmxGTWNwQ1owUkRHVTlpbzJyUDZLclNHRzlKWW55ajBNZCtoZ29OTVVDSVFDeG9tRGc0SHkwYmROMwpOZXhaa1FFckdXYmFOWitjWUZpOURMUWxia0E4SGc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZHpDQ0FSMmdBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwClpXNTBMV05oUURFM05EQTFOREEzTkRVd0hoY05NalV3TWpJMk1ETXpNakkxV2hjTk16VXdNakkwTURNek1qSTEKV2pBak1TRXdId1lEVlFRRERCaHJNM010WTJ4cFpXNTBMV05oUURFM05EQTFOREEzTkRVd1dUQVRCZ2NxaGtqTwpQUUlCQmdncWhrak9QUU1CQndOQ0FBU3VWQzFQcnI5eXV1dENBRndtc0pUMHJVa0Y1MkpJNDg3ZGQ4RUo4ZFZmCmN4ZkEya2NGaERpTWlDcnhYT1c3MlhRSVpoWEFQdkM4dzJkeVdEQWJkQnJxbzBJd1FEQU9CZ05WSFE4QkFmOEUKQkFNQ0FxUXdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVU4YXF2MGxzc1Znc0N4YW9JUkRnMApURHVKQXhVd0NnWUlLb1pJemowRUF3SURTQUF3UlFJZ0RYSTJUOVdMMTZhUFZyMnhyT2tYd1BoNjRQTVdBL1JpClhQQmw5SEhkV0xvQ0lRRG5WQVZQR0dCcy9EU2xnQ1ZJR3B4UVArVStrdnpoMkRvN1lJMDBmSkdIWUE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== - client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUlVVUhsTG10NG9mamk1dkt0c0svVWJreFhhcEl2eGkyRUtlRzAzWjlKcENvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFaU4vVnQyNEVyU1FWRTVPWGo5VnBBYUczYkRJTkRzS0p1aE9sQjl0N3MyYUNIVk82UUMvRwpqNm80LzFqVFdaSmRmWVdtb0ZNamNSYzZUS3kxbDFyRmpnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/deploy/mysql/init.sql b/deploy/mysql/init.sql index 95feffdd..10229182 100644 --- a/deploy/mysql/init.sql +++ b/deploy/mysql/init.sql @@ -8,7 +8,7 @@ USE cloudops; SET time_zone = '+08:00'; -- 创建用户（如果需要） --- CREATE USER IF NOT EXISTS 'cloudops'@'%' IDENTIFIED BY 'cloudops123'; +-- CREATE USER IF NOT EXISTS 'cloudops'@'%' IDENTIFIED BY '<请替换为强口令，勿提交真实值>'; -- GRANT ALL PRIVILEGES ON cloudops.* TO 'cloudops'@'%'; -- FLUSH PRIVILEGES; diff --git a/deploy/release/alertmanager.yml b/deploy/release/alertmanager.yml new file mode 100644 index 00000000..01b6bb5c --- /dev/null +++ b/deploy/release/alertmanager.yml @@ -0,0 +1,13 @@ +# 可选 Alertmanager 配置（compose profile: alerting）。只把告警转发给内网 webhook 服务，不含任何凭据。 +# 需要 Prometheus 推送告警时，另在 prometheus.yml 增加 alerting.alertmanagers 指向 alertmanager:9093。 +route: + receiver: cloudops-webhook + group_by: [alertname] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h +receivers: + - name: cloudops-webhook + webhook_configs: + - url: http://webhook:8889/api/v1/alerts/receive + send_resolved: true diff --git a/deploy/release/compose.yaml b/deploy/release/compose.yaml index cd7f54b0..ef38ecd5 100644 --- a/deploy/release/compose.yaml +++ b/deploy/release/compose.yaml @@ -64,6 +64,14 @@ services: ENV: production SERVER_PORT: "8889" PROMETHEUS_HTTP_SD_API: ${PROMETHEUS_HTTP_SD_API:-http://backend:8889/api/not_auth/getTreeNodeBindIps} + # 告警推送地址：backend 按它生成 Alertmanager 的 webhook 接收地址，目标是 webhook 服务（不是 backend 自己）， + # 与 alertmanager.yml 的 http://webhook:8889/api/v1/alerts/receive 一致。本模板用 Compose 默认网络，服务名 webhook 可解析； + # 端口 8889 来自 webhook 镜像内 config/webhook.yaml（compose 不给 webhook 传 WEBHOOK_PORT），两边改端口要同步。 + # 不设时代码默认 http://localhost:8889/...：Alertmanager 按它推送，localhost 指向 Alertmanager 所在的容器或主机。完整 URL 和 host:port 两种写法代码都接受； + # 完整 URL 的 scheme 建议写小写 http:// 或 https://，不区分大小写（合入 w1 后 HTTP:// 也按完整 URL 处理；合入前的旧代码区分大小写）。 + # Alertmanager 在 Compose 网络外（解析不了服务名 webhook）时，在 .env 或 shell 环境里设 PROMETHEUS_ALERT_WEBHOOK_ADDR + # 覆盖为它能访问到的 webhook 地址（webhook 默认不发布端口，需自行暴露）；未设或为空时用下面的默认值。 + PROMETHEUS_ALERT_WEBHOOK_ADDR: ${PROMETHEUS_ALERT_WEBHOOK_ADDR:-http://webhook:8889/api/v1/alerts/receive} MYSQL_ADDR: cloudops:${MYSQL_PASSWORD:?Generate .env first}@tcp(mysql:3306)/cloudops?charset=utf8mb4&parseTime=True&loc=Local MYSQL_MAX_OPEN_CONNS: ${MYSQL_MAX_OPEN_CONNS:-20} MYSQL_MAX_IDLE_CONNS: ${MYSQL_MAX_IDLE_CONNS:-5} @@ -75,6 +83,7 @@ services: REDIS_PASSWORD: ${REDIS_PASSWORD:?Generate .env first} JWT_KEY1: ${JWT_KEY1:?Generate .env first} JWT_KEY2: ${JWT_KEY2:?Generate .env first} + # Match config/config.production.yaml; the runtime image contains no config files. JWT_ISSUER: ${JWT_ISSUER:-K5mBPBYNQeNWEBvCTE5msog3KSGTdhmx} JWT_EXPIRATION: "30" TREE_PASSWORD_ENCRYPTION_KEY: ${TREE_PASSWORD_ENCRYPTION_KEY:?Generate .env first} @@ -145,6 +154,53 @@ services: retries: 5 start_period: 30s + webhook: + <<: *runtime + # Prometheus Alertmanager 告警接收服务（cmd/webhook）；只在 Compose 内网监听 8889，不发布端口。 + # 监听端口来自镜像内 config/webhook.yaml 的 webhook.port（8889）。WEBHOOK_PORT：合入 w1 前不起作用，w1 之后由配置生效 + # （优先级 WEBHOOK_PORT > 配置文件 > 默认 8889）；compose 没有传这个变量，所以这里合入前后都是 8889。 + # 改端口：w1 前挂载配置并加 --config，w1 后也可在 environment 里传 WEBHOOK_PORT；都要同步 alertmanager.yml 与 backend 的告警推送地址 + image: cloudops-webhook:${RELEASE_VERSION:-local} + build: + context: ../.. + dockerfile: deploy/release/webhook.Dockerfile + args: + GOPROXY: ${GO_MODULE_PROXY:-https://proxy.golang.org,direct} + environment: + # 完整 DSN（含口令），由 scripts/release-env.py 依据 MYSQL_PASSWORD 生成；webhook 不读取 backend 的 MYSQL_ADDR + WEBHOOK_MYSQL_ADDR: ${WEBHOOK_MYSQL_ADDR:?Run python3 scripts/release-env.py (or --add-missing for an existing .env)} + # 仅在 --profile alerting 启动 alertmanager 时可达；未启用时静默/取消静默接口返回失败，不影响告警接收 + WEBHOOK_ALERT_MANAGER_API: http://alertmanager:9093 + depends_on: + mysql: + condition: service_healthy + healthcheck: + # 代码中没有专用健康端点：GET /api/v1/alerts/silence 缺 fingerprint 时不访问数据库/Alertmanager，直接返回 HTTP 200。 + # 进程启动时 gorm.Open 会 Ping MySQL，连不上即退出，因此 HTTP 可响应即表示已连上 MySQL。 + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8889/api/v1/alerts/silence] + interval: 15s + timeout: 5s + retries: 10 + start_period: 30s + + alertmanager: + <<: *runtime + # 可选：默认不启动，`docker compose --profile alerting up -d` 时才启动 + profiles: [alerting] + image: prom/alertmanager:v0.28.1 + command: [--config.file=/etc/alertmanager/alertmanager.yml, --storage.path=/alertmanager] + volumes: + - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager:/alertmanager + depends_on: + webhook: + condition: service_healthy + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:9093/-/ready] + interval: 15s + timeout: 5s + retries: 5 + web: <<: *runtime image: cloudops-web:${RELEASE_VERSION:-local} @@ -184,3 +240,4 @@ volumes: redis_data: prometheus: aiops: + alertmanager: diff --git a/deploy/release/env.example b/deploy/release/env.example new file mode 100644 index 00000000..9bea048c --- /dev/null +++ b/deploy/release/env.example @@ -0,0 +1,82 @@ +# AI-CloudOps 发布栈环境变量模板（deploy/release/compose.yaml 读取 deploy/release/.env） +# 用法：优先运行 python3 scripts/release-env.py 自动生成 .env（0600，不打印密钥）， +# 或复制本文件为 deploy/release/.env 并逐项填写，然后 chmod 600 deploy/release/.env。 +# 本文件只允许出现变量名和注释，所有值必须为空；禁止提交真实值（.env 已被 .gitignore 忽略）。 +# 已与后端 W1 必需变量清单对账：后端必填 JWT_KEY1/JWT_KEY2/TREE_PASSWORD_ENCRYPTION_KEY 在下方；MYSQL_ADDR 由 compose 用 MYSQL_PASSWORD 拼接。 +# JWT_ISSUER 由 compose 注入与 config yaml 一致的默认值（运行镜像不包含配置文件）。通常不要覆盖；修改它会使现有 token 全部失效。 +# WEBHOOK_MYSQL_ADDR 供 compose 中的 webhook 服务（告警接收）使用，compose 为 :? 强制；CRON_ALLOW_LOCAL_EXEC 当前未接入发布栈 compose，写在 .env 中不会生效。 +# 可选 Alertmanager 用 compose profile 启用：docker compose --profile alerting up -d（默认不启动，无需额外变量）。 +# 运行环境 ENV 不在本文件设置：compose.yaml 已为 backend/aiops/mcp 显式写 ENV: production，后端镜像默认也是 production。 +# 后端 ENV 可选值（大小写不敏感，首尾空白忽略）：production/prod/prd（生产）、development/dev（开发）。 +# ENV 未设置、为空（含只有空白）或为其他取值（包括 test/testing）时后端拒绝启动（SEC-27）；APP_ENV 不生效。 + +# ---------- 数据库 / 缓存（必填，compose 中为 :? 强制）---------- +# MySQL cloudops 业务账号口令（生成: openssl rand -hex 24；仅用 hex，避免 DSN 分隔符） +MYSQL_PASSWORD= +# MySQL root 口令（生成: openssl rand -hex 24） +MYSQL_ROOT_PASSWORD= +# Redis requirepass，backend/aiops/mcp 共用（生成: openssl rand -hex 24） +REDIS_PASSWORD= + +# webhook 进程（cmd/webhook）的完整 MySQL DSN（含口令，必填）。release-env.py 用 MYSQL_PASSWORD 自动拼出： +# cloudops:@tcp(mysql:3306)/cloudops?charset=utf8mb4&parseTime=True&loc=Local +# 已有 .env 升级时运行 python3 scripts/release-env.py --add-missing 只追加此项，不改动其他值。webhook 不读取 backend 的 MYSQL_ADDR。 +WEBHOOK_MYSQL_ADDR= + +# ---------- 后端鉴权与加密（必填）---------- +# JWT 签名密钥 1/2（生成: openssl rand -hex 24） +JWT_KEY1= +JWT_KEY2= +# 主机密码 AES 加密密钥，必须恰好 32 字节（生成: openssl rand -hex 16）；升级时保持不变，并与数据库一同备份 +TREE_PASSWORD_ENCRYPTION_KEY= +# 首次空库启动时的 admin 初始口令（至少 12 位） +BOOTSTRAP_ADMIN_PASSWORD= +# backend 与 aiops/mcp 之间的内部服务 API Key（生成: openssl rand -hex 24），勿暴露给浏览器 +AIOPS_API_KEY= + +# ---------- 访问入口 ---------- +# 浏览器访问的精确 Origin（用作 SERVER_ALLOWED_ORIGINS），如 http://localhost:18080 +PUBLIC_ORIGIN= +# gateway 对外端口（compose 默认 18080） +HTTP_PORT= +# gateway 绑定地址（compose 默认 127.0.0.1；仅在受控网络中改为 0.0.0.0） +BIND_ADDRESS= +# 镜像 tag（compose 默认 local） +RELEASE_VERSION= + +# ---------- AIOps LLM（aiops 服务）---------- +# LLM 服务 API Key（OpenAI 兼容接口） +LLM_API_KEY= +# OpenAI 兼容接口 base URL（compose 默认 https://api.siliconflow.cn/v1） +LLM_BASE_URL= +# 主对话模型（compose 默认 Qwen/Qwen2.5-32B-Instruct） +LLM_MODEL= +# 任务/工具调用模型（compose 默认 Qwen/Qwen2.5-14B-Instruct） +LLM_TASK_MODEL= +# 向量化模型（compose 默认 BAAI/bge-m3） +LLM_EMBEDDING_MODEL= +# RAG 使用的 OpenAI 兼容 embedding 模型（compose 默认 BAAI/bge-m3） +RAG_OPENAI_EMBEDDING_MODEL= + +# ---------- 通知（aiops）---------- +# 是否启用通知（compose 默认 true） +NOTIFICATION_ENABLED= +# 飞书自定义机器人 webhook（含 token，属凭据；可留空） +FEISHU_WEBHOOK_URL= + +# ---------- 可选：构建/调优（不设置则使用 compose 默认值）---------- +# GO_MODULE_PROXY= +# NPM_REGISTRY= +# PROMETHEUS_HTTP_SD_API= +# backend 告警推送地址，compose 默认 http://webhook:8889/api/v1/alerts/receive（Compose 内的 webhook 服务）； +# 仅当 Alertmanager 在 Compose 网络外时覆盖为它能访问的 webhook 地址（webhook 默认不发布端口，需自行暴露）； +# scheme 建议写小写 http:// 或 https://，不区分大小写（合入 w1 后 HTTP:// 也按完整 URL 处理；合入前的旧代码区分大小写），也可写 host:port +# PROMETHEUS_ALERT_WEBHOOK_ADDR= +# MYSQL_MAX_OPEN_CONNS= +# MYSQL_MAX_IDLE_CONNS= +# MYSQL_CONN_MAX_LIFETIME= +# MYSQL_CONN_MAX_IDLE_TIME= +# MYSQL_CONNECT_TIMEOUT= +# MYSQL_STARTUP_TIMEOUT= +# 仅在叠加 ssh.yaml 时必填：已核验的 known_hosts 文件路径 +# SSH_KNOWN_HOSTS_FILE= diff --git a/deploy/release/webhook.Dockerfile b/deploy/release/webhook.Dockerfile new file mode 100644 index 00000000..26070ebb --- /dev/null +++ b/deploy/release/webhook.Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1 +# 告警 webhook 服务（cmd/webhook）镜像；构建上下文为后端仓库根目录（compose.yaml 中 context: ../..） +FROM golang:1.26.10-alpine AS builder +ARG GOPROXY=https://proxy.golang.org,direct +ENV GOPROXY=${GOPROXY} +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /app +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY . . +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -trimpath -o webhook ./cmd/webhook + +FROM alpine:latest +ENV TZ=Asia/Shanghai +RUN apk --no-cache add ca-certificates tzdata && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ + addgroup -S cloudops && adduser -S -G cloudops -H cloudops +WORKDIR /app +COPY --from=builder /app/webhook ./webhook +# 默认读取 ./config/webhook.yaml（--config 可改）；DSN 不在文件里，由环境变量 WEBHOOK_MYSQL_ADDR 注入 +COPY --from=builder /app/config/webhook.yaml ./config/webhook.yaml +USER cloudops +EXPOSE 8889 +CMD ["./webhook"] diff --git a/docker-compose.yaml b/docker-compose.yaml index c1884220..78ed42c4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -55,6 +55,8 @@ services: # env_file: # - .env # environment: + # # 本地开发栈显式 development；不依赖 .env 是否写了 ENV（未设置会拒绝启动，镜像默认是 production） + # - ENV=development # - GOPROXY=https://goproxy.cn,direct # - TZ=Asia/Shanghai # networks: @@ -107,7 +109,7 @@ services: - ./data/mysql/data:/var/lib/mysql:rw - ./deploy/mysql/init.sql:/docker-entrypoint-initdb.d/init.sql:ro environment: - - MYSQL_ROOT_PASSWORD=root + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:?set MYSQL_ROOT_PASSWORD in .env (see env.example)} - MYSQL_DATABASE=cloudops - MYSQL_CHARSET=utf8mb4 - MYSQL_COLLATION=utf8mb4_general_ci @@ -123,7 +125,7 @@ services: - cloudops_net restart: always healthcheck: - test: [ "CMD-SHELL", "mysqladmin ping -h localhost -u root -proot || exit 1" ] + test: [ "CMD-SHELL", "MYSQL_PWD=\"$$MYSQL_ROOT_PASSWORD\" mysqladmin ping -h localhost -u root || exit 1" ] interval: 10s timeout: 5s retries: 5 @@ -161,7 +163,7 @@ services: command: - redis-server - --requirepass - - v6SxhWHyZC7S + - "${REDIS_PASSWORD:?set REDIS_PASSWORD in .env (see env.example)}" networks: - cloudops_net restart: always diff --git a/docs/api/user-permissions-v2.md b/docs/api/user-permissions-v2.md new file mode 100644 index 00000000..08c1e62c --- /dev/null +++ b/docs/api/user-permissions-v2.md @@ -0,0 +1,160 @@ +# 用户权限码 v2 契约（SEC-24 / SEC-07） + +面向前端（花卷）对齐。本契约描述新增的 `GET /api/user/permissions`，以及它与旧接口 `GET /api/user/codes` 的关系。后端鉴权以本文“匹配规则”为准，`permissions` 与后端 `CheckAuth` 使用同一份匹配代码生成，二者不会出现“前端显示可用、后端拒绝”的差异（超管码与下文列出的例外除外）。 + +## 1. 接口 + +| 项目 | 值 | +| --- | --- | +| 方法 / 路径 | `GET /api/user/permissions` | +| 认证 | 需要登录（Bearer JWT，与 `/api/user/codes` 相同） | +| 授权 | 任何已登录用户都可调用，只返回**调用者本人**的权限；不接受 `id` 等参数 | +| 旧接口 | `GET /api/user/codes` **保持不变**：仍返回角色授予的原始 `path` 字符串数组（不带方法、不展开通配、去重），前端可按自己的节奏迁移 | + +## 2. 响应 + +外层仍是统一的 `ApiResponse`（`code` / `data` / `message`），`data` 结构： + +| 字段 | 类型 | 说明 | +| --- | --- | --- | +| `version` | int | 固定为 `2`。后续若格式变化会递增 | +| `super_admin` | bool | 调用者是否持有启用中的内置超级管理员角色（`code=super_admin` 且 `is_system=1`） | +| `permissions` | string[] | 权限码列表，**已排序、已去重**，永不为 `null`（没有权限时是 `[]`） | + +### 2.1 权限码格式 + +``` +<空格><路由模板> +``` + +- `METHOD` 只会是 `GET` / `POST` / `PUT` / `DELETE`（大写）。 +- `路由模板` 是 gin 注册时的模板，参数保持 `:name` 原样，例如 `PUT /api/tree/cloud/account/:id/status`。 +- 前端匹配方式：把“当前要调用的 方法 + 路由模板”拼成同样格式，用 `Set.has()` 精确比较即可。**不要**把真实 ID 代入后再比较，也**不要**把 `:param` 当作通配。 +- AIOps 代理（`/api/v1/*path`）下的接口按 AIOps 接口目录展开成具体模板，例如 `GET /api/v1/assistant/session/:session_id`。若角色被授予了整个代理入口，还会额外出现 ` /api/v1/*path` 本身。 + +### 2.2 超管码 + +超级管理员不展开列表，`permissions` **只包含一个元素** `"*"`（超管码），并且 `super_admin=true`。 + +- `"*"` 只会在 `super_admin=true` 时出现，且不会与其他权限码混在一起。 +- 前端判断建议：`super_admin === true || permissions.includes('*')` 即视为全部可用。 +- 选择超管码而不是全量列表的原因：超管在后端是“无条件放行”，全量列表会随路由增减而漂移，也比实际授权更窄（目录外的接口超管同样能访问）。 + +### 2.3 通配授予的展开 + +角色里保存的授予（`cl_system_apis.path` + `method`）按下列规则展开到**已注册路由目录**（见第 4 节），只保留方法相同且被覆盖的路由模板： + +| 授予写法 | 含义 | 展开结果 | +| --- | --- | --- | +| 普通模板，如 `/api/k8s/clusters/:cluster_id/events/:namespace/:name` | 只覆盖**完全相同**的模板 | 1 条（若该模板已注册） | +| `/*` | 覆盖除下述敏感接口外的模板 | 该方法下目录内非敏感路由 | +| 以 `*` 结尾，如 `/api/k8s/*` | 对**模板字符串**做前缀匹配（前缀为去掉末尾 `*` 的部分），下述敏感接口除外 | 该方法下以 `/api/k8s/` 开头的非敏感模板 | +| 单独的 `*` | 等同 `/*`（前缀为空），下述敏感接口除外 | 该方法下目录内非敏感路由 | +| `/api/v1/*` | 末尾 `*` 前缀，同时覆盖代理模板 `/api/v1/*path` 本身 | 等同授予整个 AIOps 代理（含目录外的接口） | +| gin 通配模板，如 `/api/v1/*path` | 与模板完全相同；对 AIOps 代理而言表示整个代理模块 | ` /api/v1/*path` 以及该方法下全部 AIOps 目录接口 | +| 其他含 `*` 的写法（开头 `*`、中间 `*`、多个 `*`） | **不再支持**，不覆盖任何模板 | 0 条 | +| 真实路径，如 `/api/tree/cloud/9/status` | 不是模板，不覆盖任何模板（改动前会精确命中该 ID） | 0 条 | +| 未注册的路径 / 已失效的授予 | 不覆盖任何模板 | 0 条 | + +普通角色访问以下敏感接口时，必须显式授予对应的 **HTTP 方法 + 完整路由模板**： + +- `GET /api/k8s/secret/:cluster_id/:namespace/:name/reveal` +- `GET /api/k8s/serviceaccount/:cluster_id/:namespace/:name/token` +- `POST /api/k8s/serviceaccount/:cluster_id/:namespace/:name/token` + +`*`、`/*`、`/api/k8s/*` 等通配授予不会覆盖这些接口，也不会将它们展开到 `permissions`。GET token 与 POST token 独立授权；参数名必须与模板逐字一致，不能使用真实 ID 或改名后的参数代替。启用中的内置超管角色仍按超管规则放行。 + +### 2.4 不出现在列表中的接口 + +- 登录后即可访问、不经角色授权的接口：`/api/user/login`、`/api/user/logout`、`/api/user/refresh_token`、`/api/user/profile`、`/api/user/codes`、`/api/user/permissions`、`/api/not_auth/getBindIps`、`/api/not_auth/getTreeNodeBindIps`。 +- 仅超管可用的接口，即使角色里授予了也不会出现（后端同样会拒绝非超管）：`POST /api/user/signup`、`/api/department` 下全部接口。 +- 非 `/api/` 前缀的路由（`/`、`/health`、`/ready` 等）。 + +## 3. 示例 + +### 3.1 普通用户 + +角色授予：`GET /api/k8s/clusters/:cluster_id/events/:namespace/:name`、`PUT /api/tree/cloud/account/:id/status`、`GET /api/v1/assistant/*`。 + +```json +{ + "code": 0, + "data": { + "version": 2, + "super_admin": false, + "permissions": [ + "GET /api/k8s/clusters/:cluster_id/events/:namespace/:name", + "GET /api/v1/assistant/config", + "GET /api/v1/assistant/health", + "GET /api/v1/assistant/info", + "GET /api/v1/assistant/ready", + "GET /api/v1/assistant/session/:session_id", + "GET /api/v1/assistant/stream", + "PUT /api/tree/cloud/account/:id/status" + ] + }, + "message": "请求成功" +} +``` + +注意：上例**不会**包含 `GET /api/k8s/clusters/:cluster_id/events/nodes/:node_name` 或 `PUT /api/tree/cloud/account/batch/status`，后端也会拒绝这两个请求（这是 SEC-24 修复的授权外溢）。 + +### 3.2 超级管理员（含 admin） + +```json +{ + "code": 0, + "data": { + "version": 2, + "super_admin": true, + "permissions": ["*"] + }, + "message": "请求成功" +} +``` + +### 3.3 没有任何角色（或角色全部禁用） + +```json +{ + "code": 0, + "data": { + "version": 2, + "super_admin": false, + "permissions": [] + }, + "message": "请求成功" +} +``` + +同一用户调用旧接口 `GET /api/user/codes` 的结果不变，例如 3.1 的用户仍得到： + +```json +{ + "code": 0, + "data": [ + "/api/k8s/clusters/:cluster_id/events/:namespace/:name", + "/api/tree/cloud/account/:id/status", + "/api/v1/assistant/*" + ], + "message": "请求成功" +} +``` + +（顺序由数据库决定；admin 的内置超管角色本身不绑定接口，所以 admin 的旧 `codes` 仍是 `[]`，与改动前一致。） + +## 4. 后端匹配规则（与 `permissions` 同源） + +1. 鉴权对象是 **HTTP 方法 + gin 路由模板**（`c.FullPath()`），不是真实请求路径。`:param` 不再作为通配符参与授权匹配。 +2. 角色授予的 `/*` 和末尾 `*` 只对**模板**做前缀匹配；第 2.3 节列出的 Secret reveal 与 ServiceAccount token 敏感接口只接受方法与模板均相同的显式授予，不接受通配授予。 +3. `c.FullPath()` 为空（请求未命中任何路由，即 404）时：非超管直接返回 `403 无权限访问该接口`，不暴露路由是否存在；超管放行，由 gin 返回 404。 +4. AIOps 代理入口 `/api/v1/*path` 的 `FullPath` 只有通配模板，后端用内置 AIOps 接口目录把真实路径解析为最具体的目录模板（逐段比较，静态段优先于 `:param`，`:param` 只匹配一个非空且不为 `.`/`..` 的段），再按第 1、2 条比较；解析不到目录模板时，只有覆盖 `/api/v1/*path` 本身的授予（如 `/api/v1/*path`、`/api/*`、`/*`）能放行。 +5. 只计入启用中（`status=1`）、未删除的角色及其未删除的接口。 +6. 超管（持有启用中的内置超管角色）对所有请求放行；`POST /api/user/signup` 与 `/api/department/*` 只允许超管。 +7. 目录来源：运行时 gin 已注册路由（`engine.Routes()`）+ 内置 AIOps 接口目录。之所以不用数据库 `cl_system_apis` 做展开：数据库里可能有已下线或手工录入的条目，展开后会给前端“幽灵权限”；而已注册路由正是 `FullPath` 的取值空间，和后端判定严格一致。 + +## 5. 前端迁移建议 + +- 读取 `permissions`，构造 `Set`；按钮/菜单声明自己依赖的 `METHOD 模板`。 +- `super_admin` 为真时跳过所有判断。 +- 旧 `codes` 中基于 `*` / `/*` 的前端通配逻辑可以在迁移完成后删除。 diff --git a/docs/kubeconfig-security.md b/docs/kubeconfig-security.md index b693eb46..c96ee8e7 100644 --- a/docs/kubeconfig-security.md +++ b/docs/kubeconfig-security.md @@ -23,3 +23,9 @@ kubectl config view --raw --flatten --minify > cloudops-kubeconfig.yaml 安全解析的错误信息不回显配置内容。已有不安全配置不会继续加载或运行其中程序；可以用相同集群传输身份的内嵌凭据替换。配额的集群身份比较只检查 API 地址、TLS server name 与 CA 数据，允许凭据轮换或 YAML 格式变化，不读取旧凭据指向的文件或执行插件。 依据：[Kubernetes kubeconfig 风险说明](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/)、项目使用的 client-go `clientcmd` 与认证 transport 实现。 + +## 接口返回与更新约定 + +- 集群列表（`GET /api/k8s/cluster/list`）与详情（`GET /api/k8s/cluster/:cluster_id/detail`）不返回 `kube_config_content`，改为返回 `has_kube_config`（bool，数据库中是否存有非空 kubeconfig）。该屏蔽在 `model.K8sCluster` 的 JSON 序列化层统一完成，任何序列化集群对象的接口都不会输出 kubeconfig；数据库读写与服务端内部连接仍使用原值。 +- 更新集群（`PUT /api/k8s/cluster/:cluster_id/update`）时 `kube_config_content` 选填：不传或空字符串保留原值；非空则整段替换，并执行与创建相同的安全解析校验。 +- 创建集群时 `kube_config_content` 仍为必填。 diff --git a/docs/release/README.md b/docs/release/README.md index 5e39fc1c..2565bae8 100644 --- a/docs/release/README.md +++ b/docs/release/README.md @@ -32,10 +32,20 @@ python3 scripts/release-smoke.py 本机运行 Go/Python 进程联调时，可叠加 `-f deploy/release/local-ports.yaml` 暴露回环地址上的 MySQL `13306` 与 Redis `16379`。对此环境后续所有 Compose 操作都应使用相同文件集合，否则 Compose 会重新创建服务并移除端口。测试直连 Go 使用 `release-smoke.py --api-only --url http://127.0.0.1:17889`。 -密钥生成器创建权限为 `0600` 的 `deploy/release/.env`，文件已被 Git 忽略；重复执行会拒绝覆盖。首次管理员为 `admin`，密码取该文件中的 `BOOTSTRAP_ADMIN_PASSWORD`。已有数据库的管理员不会被重置。检查日志时不要粘贴完整环境变量或展开后的 Compose 配置。 +后端镜像（根目录 `Dockerfile`）运行阶段默认 `ENV=production`，会启用严格的 SSH 主机密钥校验：按 `pkg/ssh/hostkey.go`，生产模式下未设置 `SSH_KNOWN_HOSTS` 时拒绝建立 SSH 连接，不会退回到跳过主机密钥校验（设置了 `SSH_KNOWN_HOSTS` 时任何 `ENV` 下都按该文件校验，不匹配即拒绝）；同时启动时按生产要求校验 JWT 与资产密码密钥（`internal/config/config.go`）。本地手动 `docker run` 该镜像做开发调试时要加 `-e ENV=development`。`ENV` 由 `pkg/envmode` 统一解析：忽略首尾空白和大小写，`production`/`prod`/`prd` 按生产处理，`development`/`dev` 按开发处理；未设置、空值和其他取值（包括 `test`/`testing`）拒绝启动。 + +密钥生成器创建权限为 `0600` 的 `deploy/release/.env`，文件已被 Git 忽略；重复执行会拒绝覆盖。生成器同时用 `MYSQL_PASSWORD` 拼出告警 webhook 服务所需的完整 DSN `WEBHOOK_MYSQL_ADDR`（格式同 backend 的 `MYSQL_ADDR`，主机为 Compose 服务 `mysql:3306`），值不打印。升级已有部署时执行 `python3 scripts/release-env.py --add-missing`，只在缺少该项时追加，不覆盖任何已有值；`--env-file`、`--mysql-host` 仅供隔离验证改目标文件与 MySQL 地址。首次管理员为 `admin`，密码取该文件中的 `BOOTSTRAP_ADMIN_PASSWORD`。已有数据库的管理员不会被重置。检查日志时不要粘贴完整环境变量或展开后的 Compose 配置。 默认访问地址为 `http://localhost:18080`，只绑定回环地址。对外服务应通过有 TLS 的反向代理，配置 `PUBLIC_ORIGIN` 为实际 HTTPS 源站。不要直接暴露 MySQL、Redis 或 AIOps。浏览器所有 `/api/v1/*` 请求经过 Go 的 JWT/RBAC 检查，再由 Go 注入内部密钥。 +## 告警 webhook 与可选 Alertmanager + +`webhook` 服务（`cmd/webhook`，镜像 `cloudops-webhook:${RELEASE_VERSION}`，由 `deploy/release/webhook.Dockerfile` 构建）默认随栈启动，只在 Compose 内网监听 `8889`（取自镜像内 `config/webhook.yaml` 的 `webhook.port`。`WEBHOOK_PORT` 的优先级是 `WEBHOOK_PORT > 配置文件 > 默认 8889`；compose 没有传这个变量，所以发布栈默认监听 `8889`），不发布端口；依赖 `mysql` 健康，且只读取 `WEBHOOK_MYSQL_ADDR`（缺失时 `compose config` 即报错）。代码没有专用健康端点，健康检查请求 `GET /api/v1/alerts/silence`：缺少 fingerprint 时不访问数据库和 Alertmanager，直接返回 HTTP 200；进程启动时会 Ping MySQL，连不上即退出，因此 HTTP 可响应即表示已连上 MySQL。 + +Alertmanager 是可选 profile：默认 `docker compose up` 不启动；`docker compose -f deploy/release/compose.yaml --profile alerting up -d` 才启动，告警转发到 `http://webhook:8889/api/v1/alerts/receive`（配置 `deploy/release/alertmanager.yml`）。未启用时 webhook 的静默/取消静默接口会返回失败，告警接收不受影响。Prometheus 默认不推送告警，需要时在 `prometheus.yml` 自行增加 `alerting.alertmanagers`。 + +backend 的告警推送地址 `PROMETHEUS_ALERT_WEBHOOK_ADDR`（backend 据此生成 Alertmanager 的 webhook 接收地址）在 `compose.yaml` 中写为 `${PROMETHEUS_ALERT_WEBHOOK_ADDR:-http://webhook:8889/api/v1/alerts/receive}`：未设置或设为空时指向 Compose 内的 `webhook` 服务。Alertmanager 部署在 Compose 网络外、解析不了服务名 `webhook` 时，在 `deploy/release/.env` 中设置 `PROMETHEUS_ALERT_WEBHOOK_ADDR=http(s)://:<端口>/api/v1/alerts/receive` 覆盖（scheme 建议写小写 `http://` 或 `https://`，不区分大小写，`HTTP://` 也按完整 URL 处理；也可只写 `host:port`，按 http 处理）。`webhook` 默认不发布端口，需要另用 override 文件或反向代理暴露，并自行限制来源。shell 中导出的同名变量优先于 `.env`（导出空串也一样：会压过 `.env` 里的值并回落到上面的默认值），发布前确认没有误带本机开发用的 `http://localhost:8889/...`（这个地址由 Alertmanager 用来推送告警，localhost 指向 Alertmanager 所在的容器或主机，到不了 `webhook` 服务）。核对时只取这一个字段：`docker compose -f deploy/release/compose.yaml config --format json | jq -r '.services.backend.environment.PROMETHEUS_ALERT_WEBHOOK_ADDR'`，不要输出完整配置。 + ## 真实依赖 - MySQL 保存用户、权限、资产、工单等业务数据。使用独立 `cloudops` 用户，初始化权限只作用于其数据库。 @@ -59,7 +69,7 @@ AIOps 飞书自定义机器人可在服务端私有 `.env` 设置 `FEISHU_WEBHOO ## 发布门槛 -发布工具回归使用 `python3 -m unittest discover -s scripts/tests -v`，无需运行容器，覆盖仓库路径、启动预检与私有密钥的防覆盖行为。CI 与标签发布均执行该检查。 +发布工具回归使用 `python3 -m unittest discover -s scripts/tests -v`，无需运行容器，覆盖仓库路径、启动预检与私有密钥的防覆盖行为。CI 与标签发布均执行该检查。其中发布 compose 用例需要 `docker compose` 与 `jq`（只做 `compose config`，不连守护进程）：本地缺少时跳过；`CI=true`（或 `1`）或 `RELEASE_TESTS_REQUIRE_COMPOSE=1` 时缺少即判失败，不允许跳过。 每次交付保存命令、退出码和日期，区分离线测试、隔离环境集成、真实外部依赖验收： diff --git a/docs/release/admin-reset-proposal.md b/docs/release/admin-reset-proposal.md new file mode 100644 index 00000000..d0f36a2a --- /dev/null +++ b/docs/release/admin-reset-proposal.md @@ -0,0 +1,121 @@ +# admin 口令重置途径方案（SEC-27 后续） + +> **状态：草稿，等用户审批。** 本文只比较方案，不包含实现代码；在审批通过前不新增任何命令、环境变量或代码路径。 + +## 1. 背景 + +SEC-27 之后，启动时会检测用户名为 `admin` 的所有行（表 `cl_system_users`）是否仍在使用 SEC-27 之前 mock 内置的旧弱口令 +（`internal/app/legacy_admin_password.go`）： + +- 生产环境（`ENV` 为 `production`/`prod`/`prd`）命中，或检查本身读库失败：`App.Run` 返回错误，进程退出（`internal/app/app.go:142-144`，`main.go:41-44`）。 +- 非生产环境（`development`/`dev`、`test`/`testing`）：只打 `[SECURITY]` Warn。 + +已有的 `BOOTSTRAP_ADMIN_PASSWORD` 只在**用户表为空**时创建 admin，不会重置已有账号（`internal/app/bootstrap.go`）。 +因此一旦生产库中的 admin 仍是旧口令，服务起不来，也就无法通过 Web 界面修改口令；目前唯一的处置方式是负责人手工生成 bcrypt 并直接改库， +这一步没有统一流程，也没有审计。本文比较两种受控的重置途径。 + +## 2. 共同约束（两种方案都必须满足） + +1. 口令**不得**出现在命令行参数（会进入 `ps`、shell history、容器 `docker inspect`/K8s 事件）和任何日志、审计记录、错误信息里。 +2. 新口令至少 12 字符（与 `MinBootstrapAdminPasswordLen` 一致），并且不得等于旧内置口令（复用 `legacyMockAdminCandidate` 做比对，用后清零）。 +3. 只写入 bcrypt 哈希（`bcrypt.DefaultCost`），明文在内存中用完即清零；读取环境变量后立即 `Unsetenv`（复用 `takeSecretEnv`）。 +4. 作用范围仅限 `username = 'admin'` 的行；同名多行时全部更新或拒绝执行（审批时二选一，草稿倾向「拒绝执行并提示人工处理」）。 +5. 重置后让 admin 已签发的会话失效（复用现有退出会话失效机制），避免旧口令期间签发的 token 继续有效。 +6. 审计记录**只记**：操作者、时间（UTC 与 Asia/Shanghai）、主机名/Pod 名、目标用户 id 与 username、动作（`admin_password_reset`）、结果（成功/失败原因分类）、触发途径（CLI / BOOTSTRAP 开关）、变更单号。**不记**口令、哈希、环境变量值。 + +## 3. 方案 A：一次性 CLI `reset-admin` + +### 3.1 触发 + +- 在与服务相同的镜像中提供子命令，例如 `AI-CloudOps reset-admin --operator <工号> --ticket <变更单号>`（参数只含身份与单号，**不含口令**）。 +- 运维在维护窗口用 `kubectl exec`/`docker run --rm` 以一次性任务运行，服务本体无需先启动；命令复用 `config.Load` 的 `ENV` 校验与数据库配置。 +- 执行前打印目标行（id、username、account_type、updated_at）并要求交互确认（`--yes` 仅允许在非交互的一次性 Job 中使用）。 + +### 3.2 口令传入 + +- 首选：交互式 TTY 关闭回显读取两次（`golang.org/x/term` `ReadPassword`），两次一致才继续。 +- 非交互（K8s Job）：只从**文件**读取，例如 `--password-file /run/secrets/admin_reset`（路径是参数，口令不是），文件来自一次性 Secret 挂载；读取后进程内清零，命令结束后由运维删除 Secret。 +- 不接受 `--password=...`，也不从普通环境变量读取（避免进入 `/proc//environ` 与容器元数据）。 + +### 3.3 审计 + +- 在同一事务内更新口令哈希并写入 `cl_system_audit_logs` 一条记录：`UserID`=目标 admin id，`OperationType`=`admin_password_reset`，`TargetType`=`user`，`TargetID`=admin id，`Endpoint`=`cli:reset-admin`，`HttpMethod`=`CLI`，`Description` 中写操作者与变更单号，`IPAddress` 写主机名/Pod 名；`RequestBody`/`ResponseBody` 置空。 +- 同时向 stderr 输出一行结构化日志（同样不含口令/哈希），便于平台日志留存。 +- 数据库不可写时整体失败，不留下「改了口令但没审计」的状态。 + +### 3.4 用完如何失效 + +- 命令本身是一次性进程，退出即结束，没有常驻入口；服务进程不暴露任何 HTTP 重置接口。 +- 一次性 Secret（口令文件）在执行后删除；K8s Job 设置 `ttlSecondsAfterFinished`。 +- 可选加强：命令只在「检测到旧弱口令」或带 `--force --ticket` 时允许执行，避免被当作日常改密工具。 + +### 3.5 风险 + +- 任何能 `exec` 进容器并拿到数据库凭据的人都能执行 → 依赖集群 RBAC；审计记录可追溯但无法阻止。 +- 新增二进制入口，需要单独测试与代码评审；误把口令写进 shell history（如 `echo pass > file`）属于操作风险，需在 Runbook 中给出安全写法。 +- 同名多个 admin 行时语义不清（见共同约束 4）。 + +### 3.6 回滚 + +- 执行前按 `docs/release/backup-restore.md` 备份 `cl_system_users`（至少导出 admin 行的 `id`、`password`、`updated_at`，备份文件按密钥材料保管）。 +- 代码回滚：删除子命令即可，不影响服务主流程与数据结构。 +- 数据回滚：用备份的旧哈希写回——但这会恢复旧弱口令，生产环境将再次拒绝启动，因此实际回滚应是「再执行一次 reset-admin 设置另一个新口令」，而不是写回旧哈希。 + +## 4. 方案 B：BOOTSTRAP 重置开关 + +### 4.1 触发 + +- 新增开关，例如 `BOOTSTRAP_ADMIN_RESET=<变更单号>` 配合已有的 `BOOTSTRAP_ADMIN_PASSWORD`：服务启动时若两者都存在，且 admin 行存在，则在 `checkLegacyAdminPassword` **之前**用新口令覆盖 admin 哈希，然后继续正常启动。 +- 通过修改部署（Secret/环境变量）并重启服务触发。 + +### 4.2 口令传入 + +- 复用 `BOOTSTRAP_ADMIN_PASSWORD`：读取后立即 `Unsetenv`（`takeSecretEnv` 已实现），并在启动 cron/asynq 等子进程前兜底清除（`scrubAdminPasswordEnv`）。 +- 风险点：环境变量在进程启动前已进入容器规格、`/proc/1/environ`（`Unsetenv` 只影响进程内副本，不改内核保存的初始环境）以及编排系统元数据；必须用 Secret 引用而非明文写入清单。 + +### 4.3 审计 + +- 启动流程在同一事务内更新哈希并写 `cl_system_audit_logs`：`Endpoint`=`bootstrap:admin-reset`，`Description` 写变更单号与 Pod 名；操作者只能从变更单号间接得到（进程内不知道是谁改了部署），需要依赖编排平台的变更审计补充「谁」。 +- 启动日志打一条 `[SECURITY] admin 口令已通过 BOOTSTRAP 重置` Warn，不含口令。 + +### 4.4 用完如何失效 + +- 关键难点：开关是**状态**而不是**动作**。只要变量未从部署中移除，每次重启都会再次重置口令（会覆盖 admin 之后自行修改的口令）。 +- 缓解：重置后在库中记录「该变更单号已使用」（需要新表或配置表一行），再次遇到同一单号时拒绝并在生产环境拒绝启动，提示移除变量；这引入了新的持久化状态。 +- 仍需运维在部署中手工删除 `BOOTSTRAP_ADMIN_RESET` 与 `BOOTSTRAP_ADMIN_PASSWORD`。 + +### 4.5 风险 + +- 常驻的服务进程获得了「改管理员口令」的能力，攻击面落在服务主流程上；配置漂移（变量忘删、被模板复制到其他环境）会导致重复或越界重置。 +- 多副本同时启动会并发执行重置，需要加锁或依赖幂等单号记录。 +- 「谁执行」无法在进程内得知，审计完整性依赖外部平台。 + +### 4.6 回滚 + +- 代码回滚：去掉开关逻辑；已写入的「单号已使用」记录可保留或清理。 +- 数据回滚：同方案 A，不建议写回旧哈希，而是用新的单号再重置一次；同时从部署中移除两个变量后重启。 + +## 5. 对比 + +| 维度 | A：一次性 CLI `reset-admin` | B：BOOTSTRAP 重置开关 | +| --- | --- | --- | +| 触发 | 维护窗口内显式执行一次 | 修改部署并重启 | +| 口令传入 | TTY 无回显或一次性 Secret 文件 | 环境变量（进程内可清除，但初始环境与编排元数据仍留存） | +| 审计「谁」 | 进程内可记录 `--operator` + 单号 | 只能记单号，「谁」依赖平台审计 | +| 用完失效 | 进程退出即失效，无常驻入口 | 需手工移除变量，否则每次重启重复执行；需额外持久化防重放 | +| 多副本 | 不涉及 | 需加锁/幂等 | +| 新增攻击面 | 镜像中的一次性子命令 | 服务启动主流程 | +| 实现量 | 子命令 + 事务 + 审计 + 测试 | 启动分支 + 防重放状态 + 审计 + 测试 | + +## 6. 推荐（草稿，等用户审批） + +**推荐方案 A：一次性 CLI `reset-admin`。** 理由： + +1. 天然一次性，用完即失效，不依赖「记得删变量」； +2. 口令可以完全不经过环境变量与命令行参数（TTY 无回显或一次性 Secret 文件）； +3. 审计可以在进程内直接记录操作者与变更单号； +4. 不给常驻服务进程增加改管理员口令的能力，也不涉及多副本并发。 + +方案 B 的唯一优势是运维流程只需改部署，若审批倾向 B，至少需要「单号防重放 + 生产环境发现变量未移除即拒绝启动」两项加强。 + +审批前不做任何实现；审批通过后另开单，代码、测试与 Runbook 单独提交评审。 diff --git a/docs/release/k8s-secret-redaction.md b/docs/release/k8s-secret-redaction.md new file mode 100644 index 00000000..57ef64c3 --- /dev/null +++ b/docs/release/k8s-secret-redaction.md @@ -0,0 +1,158 @@ +# K8s Secret 脱敏与 reveal 契约（SEC-25） + +## 结论 + +- `list`、`detail`、`detail/yaml` 三个接口**默认不返回任何 Secret 值**，只返回键名和值的字节数。 +- 明文只能通过新接口 `GET /api/k8s/secret/:cluster_id/:namespace/:name/reveal` 获取。每次调用都会同步写一条审计（`cl_system_audit_logs`），审计只含谁、何时、哪个集群/命名空间/Secret、哪些 key，**不含值**。审计写入失败时不返回明文。 +- `kubectl.kubernetes.io/last-applied-configuration` 注解可能含整份明文，list/detail 的 `annotations` 和 YAML 中都不再返回它。 +- 本次改动不涉及数据库表结构变化（`K8sSecret` 不参与 AutoMigrate，`cl_system_apis`、`cl_system_audit_logs` 的结构未改）。 + +## 1. list / detail：字段变化 + +| 字段 | 改造前 | 改造后 | +| --- | --- | --- | +| `data` | `{key: base64(值)}` | `{key: ""}`：键都保留，值一律为空字符串 | +| `string_data` | `{key: 明文}`（通常为空） | `{key: ""}`：键都保留，值一律为空字符串 | +| `data_keys` | 无 | **新增**。`[{key, size}]` 按 key 升序，`size` 为 base64 解码后的值字节数 | +| `string_data_keys` | 无 | **新增**。`[{key, size}]` 按 key 升序，`size` 为字符串的 UTF-8 字节数 | +| `redacted` | 无 | **新增**。固定为 `true`，表示值已被移除 | +| `annotations` | 原样返回 | 不再返回 `kubectl.kubernetes.io/last-applied-configuration`，其余注解照常返回 | +| `data_count` / `size` | 键数 / 总大小 | 不变 | + +前端兼容说明：`data` 仍是对象，键集合没变，已有的“按 key 列表展示”逻辑无需修改。如果前端原来会对值做 base64 解码，现在得到的是空字符串，请改为展示 `data_keys[].size` 并提供“查看”按钮调用 reveal。 + +示例（`GET /api/k8s/secret/1/list` 中的单个 item，或 `GET /api/k8s/secret/1/ops/app-secret/detail` 返回的 `data`）： + +```json +{ + "code": 0, + "data": { + "name": "app-secret", + "namespace": "ops", + "cluster_id": 1, + "uid": "00000000-0000-0000-0000-000000000000", + "type": "Opaque", + "data": { "db-user": "", "db-pass": "" }, + "string_data": { "note": "" }, + "data_keys": [ + { "key": "db-pass", "size": 16 }, + { "key": "db-user", "size": 5 } + ], + "string_data_keys": [ + { "key": "note", "size": 9 } + ], + "redacted": true, + "labels": { "app": "demo" }, + "annotations": { "owner": "team-a" }, + "immutable": false, + "data_count": 3, + "size": "30 B", + "age": "240h0m0s", + "created_at": "2026-10-01T00:00:00Z" + }, + "message": "请求成功" +} +``` + +## 2. detail/yaml：字段变化 + +- `data` 和 `stringData` 下每个键的值都替换为占位符 ``，键保留。这个占位符不是合法的 base64，不可能被当作真实值写回集群。 +- `metadata.annotations` 中的 `last-applied-configuration` 被移除（改造前已经如此），`resourceVersion`、`uid`、`managedFields` 等系统字段的清理方式不变。 + +```json +{ + "code": 0, + "data": { + "yaml": "data:\n db-pass: \n db-user: \nmetadata:\n annotations:\n owner: team-a\n labels:\n app: demo\n name: app-secret\n namespace: ops\nstringData:\n note: \ntype: Opaque\n" + }, + "message": "请求成功" +} +``` + +## 3. 新接口：reveal + +`GET /api/k8s/secret/:cluster_id/:namespace/:name/reveal[?keys=k1&keys=k2]` + +- `keys` 可选，可以重复传，也可以用逗号分隔。为空时返回全部键；指定的键不存在时报错，同样会写审计。同一个键同时出现在 `data` 和 `stringData` 中时，两处的值都会返回，`keys` 中该键只出现一次。 +- `data` 的编码与改造前 detail 一致（base64），前端原有的解码逻辑可以直接复用。 +- 响应头带 `Cache-Control: no-store`、`Pragma: no-cache`。 +- 同步审计写入失败时返回 **HTTP 500**（`code=1`，`message` 为“写入审计失败，已拒绝返回Secret明文”），不含任何值。其他失败（Secret 不存在、键不存在等）沿用项目惯例：HTTP 200、`code=1`。 +- 权限码：接口目录（`cl_system_apis`）中的名称为 `k8s:secret:reveal`，路径为 `/api/k8s/secret/:cluster_id/:namespace/:name/reveal`，方法为 GET。**只有显式授予这一条才生效**：授权记录的 `path` 必须与上面的路由模板逐字相等、方法为 GET。`/api/k8s/*`、`/*`、`/api/k8s/*/reveal` 这类通配符授权，`/api/k8s/secret/:cluster_id/:namespace/:name/:action`、`/api/k8s/secret/:a/:b/:c/reveal` 这类参数化授权（不含 `*`，参数名或段不同），具体请求路径（如 `/api/k8s/secret/1/ops/app/reveal`），以及 detail 权限，都不覆盖 reveal。用户名为 `admin` 的账号与其他接口一样直接放行。 + +```json +{ + "code": 0, + "data": { + "cluster_id": 1, + "namespace": "ops", + "name": "app-secret", + "type": "Opaque", + "data": { "db-pass": "", "db-user": "" }, + "string_data": null, + "keys": ["db-pass", "db-user"] + }, + "message": "请求成功" +} +``` + +### 审计记录 + +由 reveal 接口在 service 层同步写入，不依赖审计中间件（中间件对 GET 请求不记录，也从不记录请求体或响应体）。 + +| 列 | 值 | +| --- | --- | +| `user_id` | 操作者 ID | +| `ip_address` / `user_agent` / `trace_id` | 请求元数据（表结构要求 IP 非空） | +| `http_method` / `endpoint` | `GET` / 实际请求路径 | +| `status_code` | 本次 reveal 的实际结果：成功 `200`；指定的键不存在 `400`；读取 Secret 失败时取 K8s API 返回的状态码（如 `404`、`403`），无法识别时 `500` | +| `operation_type` | `REVEAL`（已加入 `GetAuditTypes`） | +| `target_type` / `target_id` | `k8s_secret` / `集群ID/命名空间/名称` | +| `request_body` | 见下方 JSON，只含元数据和键名 | +| `response_body` | 始终为空 | +| `error_msg` | 失败原因（不含值） | +| `created_at` | 写入时间 | + +```json +{ + "action": "k8s_secret_reveal", + "user_id": 7, + "username": "alice", + "cluster_id": 1, + "namespace": "ops", + "name": "app-secret", + "keys": ["db-pass", "db-user"], + "success": true, + "revealed_at": "2026-10-23T15:00:00Z" +} +``` + +## 4. 回写保护（保证前端改造前不丢数据） + +- `PUT .../update`：如果请求 `data` 中某个键的值为空，而集群中该键当前有非空值，就保留原值。这样前端把脱敏后的详情原样回填提交时，真实值不会被清空。要删除一个键，请在请求里去掉它。`string_data` 按同样规则处理：某个键的值为空、而集群中该键当前有非空值时，这个键不会作为 `stringData` 提交（否则 K8s 合并 `stringData` 时会把 `data` 中的值覆盖成空），而是在 `data` 中保留原值；如果请求 `data` 对同名键给了非空新值，以请求的新值为准。集群中不存在的键按原样写入。副作用是：通过表单无法把一个已有非空值的键改成空值（`data` 和 `string_data` 都不行），需要时请改用 YAML 更新，在 YAML 中写 `""`。请求没有携带 `last-applied-configuration` 注解时，保留集群中的原注解。 +- **SEC-29 同批次调整（SEC-25 P3）**： + - `data` 缺省与空对象的区别：请求**没有 `data` 字段**或为 `"data": null` 时，视为“不修改 data”，**保留集群现有的全部 `data` 键和值**（改造前会把 `data` 整体清空）；请求带 **`"data": {}`** 时表示**显式清空**，集群中的 `data` 键全部删除。 + - `string_data` 空串的恢复范围收窄：某个键在 `string_data` 中为空串、集群中有非空原值时，**只有请求 `data` 也包含该键**（或请求根本没带 `data`，即整体保留）才恢复原值；请求 `data` 不包含该键时，该键视为已被删除，不会被恢复，也不会被写成空值（从 `stringData` 中移除，更新后集群中不再有该键）。 + - 示例：集群 `data` 为 `{a, b}`，请求 `{"data": {"a": ""}, "string_data": {"a": "", "b": ""}}` → 更新后 `a` 保留原值、`b` 被删除；请求 `{"string_data": {"a": ""}}`（无 `data`）→ `a`、`b` 都保留；请求 `{"data": {}}` → `a`、`b` 都删除。 +- `PUT .../update/yaml`：值仍为 `` 的键保持集群中的原值，用户只需修改需要变更的键。占位符对应的键在集群中不存在时拒绝更新。 +- `POST .../create/yaml`：YAML 中含 `` 时拒绝创建。 +- **SEC-26 P2：YAML 解析与校验收紧**（`update/yaml` 与 `create/yaml` 都适用）： + - 占位符的识别和恢复改为在类型化的 Secret 对象上进行，字段名按 encoding/json 不区分大小写的规则匹配（apiserver/kubectl 是区分大小写的）：`Data`、`DATA`、`StringData`、`stringdata`、`STRINGDATA` 等写法与 `data`/`stringData` 一样处理（更新时恢复原值，创建时拒绝）。改造前这些写法会绕过检查，把 `` 原样写入集群。 + - 只接受**单个 YAML 文档**。含 `---` 分隔的第二个文档（包括末尾多出一个只有 `---` 的空文档）一律拒绝。开头的 `---` 和末尾的 `...` 仍可使用。 + - 拒绝**重复键**：任意层级完全相同的重复键，以及顶层字段、`metadata` 下只差大小写的字段（如同时写 `stringData` 和 `StringData`，或 `name` 和 `Name`）。使用合并键 `<<` 后又显式覆盖其中某个键的写法也会因为重复键被拒绝。同一映射中转成字符串后相同的键（如 `1` 与 `"1"`、`true` 与 `"true"`、`yes` 与 `"true"`、`1.5` 与 `"1.5"`）也按重复键拒绝，改造前以哪个为准取决于遍历顺序、结果不确定。 + - 下面这些写法会被视为仍含占位符：`data` 中 base64 解码后（含折行写法，比如 `|` 块里分两行）为 `` 或其首尾带空白、NUL（`\0`）、零宽字符（U+200B、U+200C、U+200D、U+FEFF）的变体；`stringData` 中首尾带上述字符的 ``（如 `" "`、`"\u200b"`）。这些字符只在首尾时去掉，夹在中间的不算占位符。更新时报错要求填写真实值，创建时拒绝。 + - YAML 库给出的解析错误按白名单改写，只保留错误类别和行号（如 `解析YAML失败: line 3: 映射键必须是字符串等标量`），不再带出任何键或值；本模块自己的校验报错（只差大小写的字段、非法 base64、仍为占位符）只包含字段名和键名。不回显 Secret 的值。 + - 入口大小上限 **2 MiB（2097152 字节）**：`update/yaml`、`create/yaml` 提交的 YAML 超过上限时在任何解析之前拒绝，报错为通用的 `YAML 超过 2 MiB 上限`，不回显内容。上限按整个 YAML 文本的字节数计算。取 2 MiB 而不是 1 MiB，是因为 `data` 中的值是 base64（膨胀约 4/3），1 MiB 的入口上限会挡住数据量接近 Kubernetes 自身 1 MiB 上限的 Secret；放宽到 2 MiB 后这类 Secret 也能提交，Secret 真正的大小限制交给 apiserver 判定，入口上限只用于防止超大输入进入解析。 + - 更新时如果发生了占位符恢复，提交给集群的 YAML 由解析后的对象重新生成，字段顺序和格式会规范化（未识别的顶层字段会被丢弃，与改造前写入集群的结果一致）。 + - **已知行为差异**：因为字段名按 encoding/json 不区分大小写匹配，单独写的 `StringData:`（没有标准的 `stringData:`）也会被当作 `stringData` 处理；而 apiserver/kubectl 区分大小写，会把它当作未知字段。按 TL 决定保留这一兼容行为，前提是任何情况下都不会把占位符写回集群：更新时其中的 `` 恢复为原值，创建时拒绝。 + - **升级风险**（TL 已接受）： + - 使用合并键 `<<` 后再显式覆盖其中某个键的 YAML，改造前能提交，现在按重复键拒绝，需要去掉覆盖或展开合并键。 + - 末尾多出一个只有 `---` 的空文档（常见于模板拼接或 `kubectl get -o yaml` 结果拼接）的 YAML，改造前能提交，现在按多文档拒绝，需要删掉末尾的 `---`。 + - 超过 2 MiB（按 YAML 文本字节数计）的 YAML，改造前直接交给 apiserver 判定，现在在入口按大小拒绝。由于 Secret 本身受 Kubernetes 1 MiB 上限约束，正常的 Secret 不会触及这一上限；只有带大量注释、空白等非数据内容的 YAML 可能受影响，需要精简后再提交。 + +## 5. 前端待办 + +“查看值”“复制”“编辑”三个入口需要明文时，改为先调用 reveal（可以按 `keys` 只取需要的键），不再读取 detail/yaml 中的值。 + +## 6. 回滚 + +本次没有数据库迁移，回退提交即可恢复旧行为。回退后，reveal 接口在 `cl_system_apis` 中登记的那一行会留下，但不会产生影响；如需清理，删除 `name = 'k8s:secret:reveal'` 的记录及其在 `cl_system_role_apis` 中的关联即可。 diff --git a/docs/release/k8s-serviceaccount-token.md b/docs/release/k8s-serviceaccount-token.md new file mode 100644 index 00000000..d0891dc8 --- /dev/null +++ b/docs/release/k8s-serviceaccount-token.md @@ -0,0 +1,235 @@ +# K8s ServiceAccount token 接口加固（SEC-29） + +## 结论 + +- `GET /api/k8s/serviceaccount/:cluster_id/:namespace/:name/token` 只返回**属于该 SA 的** token Secret：类型必须为 `kubernetes.io/service-account-token`，注解 `kubernetes.io/service-account.name` 必须等于 SA 名称，若有注解 `kubernetes.io/service-account.uid` 则必须等于 SA 的 UID，命名空间一致且 `token` 非空。仅凭 `sa.secrets` 引用挂进来的其他 SA 的 token Secret（借挂）一律不返回。 +- 创建/更新 SA 时请求里的 `secrets` 引用**被忽略并记 Warn 日志**（只记 Secret 名称），不再写入集群。 +- GET 与 POST（TokenRequest）`/token` 都改为**只认显式授权**：必须按完整路由模板 + 方法授权，`/api/k8s/*`、`/*`、带 `:param` 的模板、具体请求路径都不覆盖。admin 照常放行。 +- 两个接口每次调用都**同步**写一条审计，只含元数据、不含 token；审计写入失败时**不返回 token**（HTTP 500）。响应头带 `Cache-Control: no-store`、`Pragma: no-cache`。 +- 不涉及数据库表结构变化，没有 DDL。启动时只会向 `cl_system_apis` 补登/改名权限行（见第 3 节）。**该启动迁移需要用户批准后才能上线**（见 3.6）。 +- `PUT .../update/yaml`：YAML 带 `metadata.resourceVersion` 时按乐观锁更新，版本过期返回 **HTTP 409**；不带时保持冲突重试（见 1.4）。 + +## 1. 接口契约变化 + +### 1.1 GET `/token`（读取 token Secret） + +| 场景 | 改造前 | 改造后 | +| --- | --- | --- | +| SA 自己的 token Secret | 返回 token | 不变 | +| `sa.secrets` 引用了其他 SA 的 token Secret | 返回其中第一个 token（越权） | 跳过；只剩借挂的 Secret 时返回 `code=1`、`msg` 含“没有属于自己的token Secret” | +| 注解 name 相同但 uid 不同（同名 SA 删除重建后残留） | 返回旧 token | 跳过 | +| 无 uid 注解 | 返回 | 只校验 name，返回 | +| 审计写入失败 | 无审计 | HTTP 500，不返回 token | + +返回结构 `{token, expiration_seconds, created_at, expiration_time}` 不变。 + +注意：Kubernetes 1.24 及以上的集群（`LegacyServiceAccountTokenNoAutoGeneration` 默认开启）通常**不会**再为 SA 自动生成 token Secret，`sa.secrets` 一般为空，GET `/token` 基本拿不到结果（返回 `code=1`，提示没有属于自己的 token Secret）。除非手工创建了带 `kubernetes.io/service-account.name` 注解的 `kubernetes.io/service-account-token` Secret，否则**主要入口是 POST（TokenRequest）**签发短期 token。 + +### 1.2 POST `/token`（TokenRequest） + +- 请求体 `{"expiration_seconds": 600}`；`cluster_id`、`namespace`、`service_account_name` 以 URL 为准，请求体里的同名字段会被 URL 值覆盖。 +- `expiration_seconds`（秒）： + + | 请求值 | 结果 | + | --- | --- | + | 不传 / `null` / `0` | 按默认 **3600** 签发 | + | `600` ~ `3600`（含边界） | 按请求值签发 | + | `< 600`（含负数）或 `> 3600` | **HTTP 400**，`code=1`，`msg` 说明取值范围；**不调用 K8s、不签发 token**；同步写一条审计（`status_code=400`、`success=false`、`expiration_seconds` 为请求值） | + | 非整数（如字符串） | 绑定失败，HTTP 400（与其他接口的绑定错误一致，此时不进入 service、不写 token 审计） | + + 下限 600 是 K8s TokenRequest 允许的最小值；**上限 3600 是本平台自己的策略，不是 K8s 的限制**（K8s 本身允许更长的有效期，最大值由 apiserver 的 `--service-account-max-token-expiration` 决定）。超出范围返回 **HTTP 400**，这一行为已确认可以接受。校验在调用 K8s 之前完成。响应中的 `expiration_seconds` 为实际签发使用的值（不传时为 3600）。 +- 响应中的 `expiration_time` 固定为 **UTC**，格式 `2006-01-02T15:04:05Z`：K8s 返回的过期时间先 `.UTC()` 再格式化（改造前直接格式化，非 UTC 时区的时间会被错误地加上 `Z` 后缀）。 +- K8s 返回错误时审计记录真实状态码（403/404/…）。 +- **审计写入失败时不返回 token（HTTP 500）。但 token 已由 K8s 签发**，TokenRequest 签发的 token 无法单独吊销，会在 `expiration_seconds` 到期前保持有效（调用方拿不到它）。由于有效期上限为 3600，这类“签发了但没交出去”的 token **最多有效 1 小时**。如需立即失效，只能删除并重建该 SA。 + +### 1.3 创建 / 更新 SA 的 `secrets` 字段 + +| 接口 | 改造后行为 | +| --- | --- | +| `POST .../create`、`POST .../create/yaml` | 请求里的 `secrets` 被清空后再创建，`image_pull_secrets` 不受影响 | +| `PUT .../update` | 忽略请求里的 `secrets`，保留集群中该 SA 现有的 `secrets` 引用（原来 Update 是整体替换，不传会清空控制器维护的引用） | +| `PUT .../update/yaml` | 同上；另外 YAML 中的 `metadata.name` / `metadata.namespace` 被强制改为 URL 中的值，不能借 YAML 改到其他 SA | + +为什么是“忽略 + 日志”而不是“拒绝”：`secrets` 字段在现有 API 契约和前端类型（`k8s_serviceaccount.ts` 的 `secrets?: string[]`）中都存在，并且 detail YAML 里带有集群维护的 `secrets`，前端/用户把 YAML 原样回传时如果直接拒绝会把正常的编辑请求打失败。忽略后既堵住了“把别人的 token Secret 挂到自己 SA 上”的路径，又不破坏现有调用。日志关键字：`忽略请求中的ServiceAccount secrets引用`（仅含 Secret 名称）。 + +### 1.4 更新 SA：YAML 带版本走乐观锁，其余冲突重试（`PUT .../update`、`PUT .../update/yaml`） + +**YAML 带 `metadata.resourceVersion` 时（严格乐观锁）**： + +1. 按 URL 中的 `namespace` / `name` Get 当前对象作为基础（保留不可编辑字段和集群维护的 `secrets` 引用，`name` / `namespace` 以 URL 为准）； +2. 只应用 4 个可编辑字段（同下文第 2 步）； +3. 把 `resourceVersion` 设为 **YAML 中提交的值**，只 Update **一次**； +4. 版本已过期时 API Server 返回 Conflict，**不重试**，接口返回 **HTTP 409**、`code=1`，`msg` 形如 `ServiceAccount ops/app 已被修改（提交的 resourceVersion=4 已过期），请重新获取 YAML 后再提交`。 + +409 的映射方式沿用仓库已有的错误映射风格：service 返回带类型的业务错误 `base.BusinessError{Code: constants.ErrK8sResourceConflict}`（与有效期越界返回 `BusinessError{constants.ErrInvalidParam}` → HTTP 400 同一套机制），handler 通过 `respondBusinessError`（`internal/k8s/api/secret.go`，从原 `handleAuditedRequest` 中抽出）用 `errors.As` + `errors.Is` 识别，调用新增的 `base.ConflictError`（HTTP 409 + `code=1`，与 `base.BadRequestError` 同构）。其他错误仍是项目约定的 HTTP 200 + `code=1`。 + +**YAML 不带 `resourceVersion`，以及 JSON 更新（`PUT .../update`）**：保持冲突重试。两条路径都使用 `k8s.io/client-go/util/retry` 的 `retry.RetryOnConflict(retry.DefaultRetry, ...)`（`client-go` 原有依赖，`go.mod`/`go.sum` 未改动）： + +1. 每次尝试先按 URL 中的 `namespace` / `name` **Get 最新版本**（带最新 `resourceVersion`）； +2. 只把请求中允许修改的字段**整体覆盖**到最新版本上：`labels`、`annotations`、`automount_service_account_token`、`image_pull_secrets`； +3. `secrets` 引用保留集群最新版本中的值（请求/YAML 中的 `secrets` 忽略并记 Warn 日志），`name` / `namespace` 以 URL 为准； +4. Update；返回 409 Conflict 时回到第 1 步。 + +| 场景 | 改造前 | 改造后 | +| --- | --- | --- | +| 读取与写入之间 SA 被并发修改（如 token controller 追加了 `secrets` 引用） | 用旧快照整体替换，可能抹掉刚写入的引用 | 冲突后重新读取最新版本再应用字段，保留最新的 `secrets` 引用 | +| 连续冲突超过 `retry.DefaultRetry`（共 5 次尝试，间隔约 10ms） | — | 返回 `code=1`，`msg` 形如 `更新ServiceAccount失败: Operation cannot be fulfilled ...` | +| SA 不存在 | 报“ServiceAccount不存在” | 不变；不会调用 Update | +| YAML 中带 `metadata.resourceVersion` | 透传给 K8s，旧版本 YAML 直接失败（HTTP 200 + `code=1`） | **乐观锁**：版本一致则更新成功；版本过期只 Update 一次并返回 **HTTP 409**，不重试 | +| YAML 中不带 `metadata.resourceVersion` | 用读取时的快照整体替换 | 以最新版本为基础、冲突重试（对允许修改的字段是“后写覆盖”） | +| YAML 中上述 4 个字段以外的内容（`ownerReferences`、`finalizers`、`uid` 等） | 随整体替换写入 | **不生效**，保留集群现有值 | +| JSON 中未传 `labels` / `annotations` / `image_pull_secrets` / `automount_service_account_token` | 整体替换为空 | 不变（仍是整体覆盖为空），与原契约一致 | + +“JSON 更新时没传的字段会被清空”是**改造前就存在的行为**，本次不改，留给 SEC-30 处理。manager 中旧的 `UpdateServiceAccount`（整体替换）方法同样保留，留给 SEC-30。 + +日志关键字：失败时 `更新ServiceAccount失败` / `按resourceVersion更新ServiceAccount失败`（manager），忽略 secrets 时同 1.3。 + +## 2. 权限码与授权 + +| 方法 | 路由（`cl_system_apis.path`） | `method` | 权限码（`cl_system_apis.name`） | +| --- | --- | --- | --- | +| GET | `/api/k8s/serviceaccount/:cluster_id/:namespace/:name/token` | 1 | `k8s:serviceaccount:token` | +| POST | `/api/k8s/serviceaccount/:cluster_id/:namespace/:name/token` | 2 | `k8s:serviceaccount:token:create` | + +- `cl_system_apis.name` 有唯一索引，GET 与 POST 不能共用一个名字，所以 POST 用 `:create` 后缀。 +- 授权按 `path + method` 精确匹配：GET 授权不覆盖 POST，反之亦然。 +- 以下授权都**不再**放行 `/token`：`/api/k8s/*`、`/*`、`/api/k8s/serviceaccount/:cluster_id/:namespace/:name/:action`、换了参数名的模板、具体路径如 `/api/k8s/serviceaccount/1/ops/app/token`、中间通配 `/api/k8s/*/token`。 + +## 3. 启动时的权限目录迁移（无 DDL） + +**本节描述的启动迁移需要用户批准后才能上线。** 它在服务每次启动时由 `bootstrapPermissionCatalog`（`internal/app/permissions.go`）自动执行，没有单独的开关；批准前不要把包含本改动的版本部署到已有数据的环境。 + +### 3.1 涉及的表和行 + +只涉及 `cl_system_apis` 一张表（`model.Api`），不改表结构、不动其他表、不授予任何角色（不写 `cl_role_apis` / `cl_user_apis`）。可能被写入的只有下面三行（`method`：1=GET，2=POST）： + +| `path` | `method` | 目标 `name` | 目标 `description` | +| --- | --- | --- | --- | +| `/api/k8s/serviceaccount/:cluster_id/:namespace/:name/token` | 1 | `k8s:serviceaccount:token` | `读取 K8s ServiceAccount token Secret（SEC-29，每次调用写审计；仅显式授权生效，通配符授权不覆盖）` | +| `/api/k8s/serviceaccount/:cluster_id/:namespace/:name/token` | 2 | `k8s:serviceaccount:token:create` | `通过 TokenRequest 签发 K8s ServiceAccount token（SEC-29，每次调用写审计；仅显式授权生效，通配符授权不覆盖）` | +| `/api/k8s/secret/:cluster_id/:namespace/:name/reveal`（SEC-25） | 1 | `k8s:secret:reveal` | `查看 K8s Secret 明文（SEC-25，…）` | + +其他路由仍按原逻辑只补登缺失的行（自动名 + 说明 `" "`），本次没有改变。 + +### 3.2 两种操作与条件 + +**(a) 插入（行不存在时）**：按 `(path, method)` 判断该路由在表中没有未删除的行（查询走 GORM 软删除过滤 `deleted_at = 0`，已软删除的行不计入，与原逻辑一致）时，按上表的 `name` / `description` 插入，`version='v1'`、`category=2`、`is_public=0`。插入使用 `INSERT ... ON CONFLICT DO NOTHING`：若 `name` 已被其他行占用（唯一索引 `idx_name_del` = `name + deleted_at`），插入被静默跳过。插入后会核对这三个权限码，**缺行时记一条 Warn**：`敏感接口权限码未登记到接口目录（插入因唯一键冲突被跳过），请人工核对 cl_system_apis`，字段只有 `permission=<权限码>`，不影响启动。 + +**(b) 改名（行已存在时）**：已部署实例在 SEC-29 之前就登记过 `/token` 的 GET/POST 行，名称是自动生成的。只有同时满足以下条件才更新： + +- `id` = 按 `(path, method)` 查到的那一行； +- `name` 仍等于自动生成值 `" ") 前 4 字节 hex>"`，对 `/token` 两行分别为 `GET GetServiceAccountToken-fm af783601`、`POST CreateServiceAccountToken 8ff0d051`（由代码计算，上线前请与库中实际值核对）； +- `description` 仍等于自动生成值 `" "`，即 `GET /api/k8s/serviceaccount/:cluster_id/:namespace/:name/token` / `POST /api/k8s/serviceaccount/:cluster_id/:namespace/:name/token`。 + +等价 SQL（**仅说明语义，由程序通过 GORM 执行，请勿手工执行**）： + +```sql +UPDATE cl_system_apis + SET name = '<目标 name>', description = '<目标 description>', updated_at = <当前时间> + WHERE id = ? AND name = '<自动名>' AND description = ' ' AND deleted_at = 0; +``` + +只改 `name`、`description` 两个字段（GORM 会同时刷新 `updated_at`），`path`、`method`、`version`、`category`、`is_public` 和角色关联都不动。 + +- 管理员改过名称或说明（任一项）的行**整行不动**，记一条 Info 日志 `敏感接口权限行的说明已被管理员修改，保留原名称与说明`。这类行保留自动名，不影响授权（授权只看 `path + method`）。 +- 改名失败（例如目标权限码已被其他行占用触发唯一索引冲突）只记 Warn 日志，不影响启动。 + +### 3.3 幂等性 + +- 插入以 `(path, method)` 是否已有行为前提，再加 `ON CONFLICT DO NOTHING`，重复启动不会产生重复行。 +- 改名的 WHERE 带“名称、说明都仍是自动值”的条件，改过一次后条件不再成立，再次启动是空操作；多个实例同时启动时，同一行最多被改一次，其余实例的 UPDATE 影响 0 行。 +- 缺行 Warn 只针对本次实际尝试插入的权限码；已存在（包括被管理员改名）的行不会每次启动都告警。 + +### 3.4 回滚 SQL（草稿，只写在文档里，绝不执行） + +以下 SQL **只是草稿**，本次开发和验证过程中没有执行过，也不应由程序或流水线执行。如确需把改过名的两行恢复为自动名，由 DBA 在批准后先核对 `id` 和当前值再手工执行： + +```sql +-- 草稿：仅供审阅，未执行。执行前请先 SELECT 确认行与值。 +-- SELECT id, name, path, method, description FROM cl_system_apis +-- WHERE path = '/api/k8s/serviceaccount/:cluster_id/:namespace/:name/token' AND deleted_at = 0; + +UPDATE cl_system_apis + SET name = 'GET GetServiceAccountToken-fm af783601', + description = 'GET /api/k8s/serviceaccount/:cluster_id/:namespace/:name/token' + WHERE path = '/api/k8s/serviceaccount/:cluster_id/:namespace/:name/token' + AND method = 1 AND name = 'k8s:serviceaccount:token' AND deleted_at = 0; + +UPDATE cl_system_apis + SET name = 'POST CreateServiceAccountToken 8ff0d051', + description = 'POST /api/k8s/serviceaccount/:cluster_id/:namespace/:name/token' + WHERE path = '/api/k8s/serviceaccount/:cluster_id/:namespace/:name/token' + AND method = 2 AND name = 'k8s:serviceaccount:token:create' AND deleted_at = 0; +``` + +注意：新实例上这两行是 SEC-29 直接插入的（没有“原自动名”），回滚代码后保留即可；恢复名称不是回滚的必要步骤（见第 6 节）。不建议删除这些行：删除会让已经按这两行显式授权的角色失去授权关联。 + +### 3.5 不会做的事 + +- 不会自动授予任何角色； +- 不会修改管理员维护过的名称、说明； +- 不会删除任何行，也不会改 `path` / `method`。 + +### 3.6 上线审批 + +该迁移会在启动时写生产库的 `cl_system_apis`，**需要用户批准后才能上线**。审批时请确认：3.2 中的两个自动名与库中实际值一致；是否接受“管理员只改过说明的行保留自动名”；是否需要按 3.4 准备回滚。 + +## 4. 审计字段 + +写入 `cl_system_audit_logs`，同步写入（不经异步队列）： + +| 字段 | GET | POST | +| --- | --- | --- | +| `operation_type` | `TOKEN_REVEAL` | `TOKEN_CREATE` | +| `http_method` | `GET` | `POST` | +| `target_type` | `k8s_serviceaccount_token` | 同左 | +| `target_id` | `//` | 同左 | +| `status_code` | 200 / 404 / K8s 真实状态码 | 200 / 400（有效期越界）/ K8s 真实状态码 | +| `user_id`、`ip_address`、`user_agent`、`trace_id`、`endpoint`、`duration`、`error_msg` | 照常 | 照常 | +| `response_body` | 空 | 空 | + +`request_body` 为 JSON 元数据（示例值均为虚构）： + +```json +{ + "action": "k8s_serviceaccount_token_reveal", + "user_id": 7, + "username": "operator", + "cluster_id": 1, + "namespace": "ops", + "service_account": "app", + "service_account_uid": "00000000-0000-0000-0000-000000000000", + "secret_name": "app-token-xxxxx", + "success": true, + "issued_at": "2026-01-01T00:00:00Z" +} +``` + +POST 的 `action` 为 `k8s_serviceaccount_token_create`，不含 `secret_name`，含 `expiration_seconds`（生效值：不传时记 3600；越界被拒时记请求值）。**任何审计字段都不含 token。** 请求失败（SA 不存在、无合法 token Secret、有效期越界、K8s 拒绝）也会写审计，`success=false`。通用审计中间件对 GET 不记录；对 POST 另记一条 `CREATE` 元数据记录，不记录请求体和响应体。 + +## 5. 兼容性影响 + +- **只靠通配符授权（`/api/k8s/*`、`/*` 等）的非 admin 角色将失去 GET/POST `/token`**，返回 403。需要时按第 2 节为角色显式授权对应的 path + method。 +- 依赖“借挂 token Secret”拿 token 的用法失效（这正是要堵的越权路径）。 +- 创建/更新时传 `secrets` 不再生效，但请求不会失败。 +- 更新 SA 改为“读最新版本 + 只应用允许修改的字段”：YAML 中 4 个可编辑字段以外的内容不再生效；YAML 带 `resourceVersion` 时为乐观锁，版本过期返回 **HTTP 409**（原来是 HTTP 200 + `code=1`），不带时冲突重试（见 1.4）。调用方需要能处理 409（提示用户重新获取 YAML）。 +- JSON 更新时没传的字段会被清空：原有行为，本次未改，留给 SEC-30。 +- `expiration_time` 统一为 UTC 字符串。 +- `/token` 响应不可缓存（`no-store`）。 +- POST `/token` 的 `expiration_seconds` 超出 600~3600 时由原来的“透传给 K8s”改为 HTTP 400；不传时由“交给 K8s 默认”改为由平台显式传 3600（K8s API 默认值同为 3600，实际效果不变）。 +- 审计库不可用时 `/token` 返回 HTTP 500。 + +## 6. 回滚 + +- 回滚代码即可，无需数据库回滚（3.4 的 SQL 只是可选的名称恢复草稿，不是必要步骤）。 +- 回滚后 `cl_system_apis` 中残留的行：`k8s:serviceaccount:token`（GET）和 `k8s:serviceaccount:token:create`（POST），或在已部署实例上是被改名后的原行。旧代码启动时按 `(path, method)` 判断已存在，**不会改回自动名称，也不会新增重复行**，残留名称不影响授权。如需恢复原名称，可由 DBA 手动改 `name`；保留不动也无副作用。 +- 回滚后通配符授权重新覆盖 `/token`，借挂 token Secret 重新可被读出，审计和 no-store 一并消失。为 SEC-29 新增的显式授权行在旧代码下仍然有效（旧代码同样接受精确授权）。 +- 回滚期间已写入的 `TOKEN_REVEAL` / `TOKEN_CREATE` 审计记录保留，旧代码不读取也不受影响。 +- 回滚后 YAML 更新不再返回 409（恢复为改造前行为）。 + +## 7. 同批次的 SEC-25 语义变化（Secret 更新） + +详见 `docs/release/k8s-secret-redaction.md` 第 4 节，摘要： + +- `PUT /api/k8s/secret/.../update` 的 `string_data` 中值为空串的键，**只有当请求 `data` 也包含该键时**才恢复集群原值；请求 `data` 不包含该键时，该键被**删除**（不再因为 `string_data` 里残留的空串被恢复）。 +- 请求**没有 `data` 字段**（或 `"data": null`）时，**保留**集群现有的 `data`（原来会把 `data` 整体清空）；请求带 **`"data": {}`**（空对象）表示**显式删除全部 `data` 键**。 diff --git a/docs/release/rbac-super-admin.md b/docs/release/rbac-super-admin.md new file mode 100644 index 00000000..d82a1d0c --- /dev/null +++ b/docs/release/rbac-super-admin.md @@ -0,0 +1,89 @@ +# 内置超级管理员角色与精确鉴权（SEC-07 / SEC-24）迁移说明 + +## 变更摘要 + +- SEC-07：去掉所有按用户名 `admin` 判断特权的逻辑，改为判断用户是否持有**启用中的内置超级管理员角色**（`cl_system_roles.code='super_admin' AND is_system=1 AND status=1`）。 +- SEC-24：`CheckAuth` 改为“HTTP 方法 + gin 路由模板”精确比较；新增 `GET /api/user/permissions`，契约见 [docs/api/user-permissions-v2.md](../api/user-permissions-v2.md)。 + +## 表结构 + +**没有新表、新列、新索引，没有修改任何 GORM 模型 tag。** `pkg/di/init.go` 的 AutoMigrate 不会因本次改动产生新的 DDL。 + +## 种子数据（启动引导，幂等） + +每次启动在权限目录与 Mock 初始化之后执行 `bootstrapSuperAdmin`，在一个事务里： + +1. 查找 `code='super_admin' AND is_system=1` 且未删除的角色，按 `id` 取最小的一条；没有则插入： + + | 列 | 值 | + | --- | --- | + | `name` | `超级管理员` | + | `code` | `super_admin` | + | `description` | `内置超级管理员角色，由启动引导维护，不可修改或删除` | + | `status` | `1` | + | `is_system` | `1` | + + 该角色**不绑定任何接口**，超管在鉴权中直接放行。 +2. 查找所有未删除且用户名**严格等于** `admin` 的用户（Go 侧大小写敏感比较，避免 MySQL 排序规则把 `Admin` 也匹配进来），若尚未绑定该角色则在 `cl_system_user_roles` 插入一行。已绑定时不写库。 +3. 若已存在的内置超管角色 `status != 1`（只能是有人直接改库），引导**不会自动重新启用**，只打印告警；此时 admin 不再是超管，需人工处理（见“应急”）。 +4. 如果存在 `code='super_admin'` 但 `is_system=0` 的普通角色（历史上手工创建的同名角色），引导**不会提升**它，只告警并另建系统角色；持有那个普通角色的用户不会获得超管权限。 + +幂等性：重复启动不会产生新行。多副本**同时首次**启动时，因角色表没有唯一索引，极小概率会各插入一条系统超管角色；这不影响正确性（判断依据是 `code + is_system`，任一条都生效），下次启动会统一绑定 `id` 最小的那条，多余的一条可在确认无用户绑定后手工删除。 + +没有设置 `BOOTSTRAP_ADMIN_PASSWORD` 且未开启 Mock 的空库不会创建 admin，引导只建角色不绑定用户。 + +## 权限相关的其他行为变化 + +- 角色接口 `POST /api/role/assign_users`、`POST /api/role/revoke_users`：涉及内置超管角色时，调用者必须是超管，否则 `403`。防止“有分配角色权限的普通用户把自己提升为超管”。 +- `PUT /api/role/update/:id`、`DELETE /api/role/delete/:id` 原本就拒绝系统角色；`POST /api/role/create` 原本就强制 `is_system=0`，因此无法通过 API 伪造系统超管角色。 +- 由于引导每次启动都会把 `admin` 绑回超管角色，**撤销 admin 的超管角色只在下次重启前有效**。 +- 用户名 `admin` 仍是保留名，注册接口拒绝（大小写不敏感），以免被他人注册后在重启时自动获得超管。 + +## 上线步骤 + +1. 备份 MySQL（至少 `cl_system_roles`、`cl_system_user_roles`、`cl_system_role_apis`、`cl_system_apis`）。 +2. 上线前用只读 SQL 盘点会受精确鉴权影响的授予（在数据库副本或只读账号执行）： + + ```sql + -- 含 * 但不是“恰好一个且位于末尾”的授予：上线后不再生效 + SELECT id, path, method FROM cl_system_apis + WHERE deleted_at = 0 AND path LIKE '%*%' + AND NOT (path LIKE '%*' AND path NOT LIKE '%*%*') ; + -- 被角色引用、不含 * 也不含 : 的授予：若写的是真实路径（如 /api/tree/cloud/9/status） + -- 而不是模板，上线后不再生效；请与 gin 路由表人工核对 + SELECT DISTINCT a.id, a.path, a.method FROM cl_system_apis a + JOIN cl_system_role_apis ra ON ra.api_id = a.id + WHERE a.deleted_at = 0 AND a.path NOT LIKE '%*%' AND a.path NOT LIKE '%:%' ; + -- 已存在的同名角色（引导不会提升它们） + SELECT id, name, code, is_system, status FROM cl_system_roles + WHERE deleted_at = 0 AND (code = 'super_admin' OR name = '超级管理员'); + ``` + 第一条结果里形如 `/api/v1/*path` 的 gin 通配模板仍然有效（与模板完全相等），其余需要改成末尾 `*` 或具体模板。 + 第二条结果中，启动时 `bootstrapPermissionCatalog` 自动补录的条目都是路由模板，不受影响；需要关注的是手工录入的真实路径。上线后也可以用受影响用户调用 `GET /api/user/permissions`，列表里缺失的就是不再生效的授予。 +3. 部署新版本，启动日志确认无 `初始化超级管理员角色失败`。 +4. 验证：以 admin 登录，`GET /api/user/permissions` 返回 `super_admin=true`、`permissions=["*"]`；以普通测试账号验证契约第 3.1 节的拒绝用例。 + +## 回滚 + +应用回滚到旧镜像即可，**不需要删除数据**： + +- 旧版本按用户名判断 admin，不读取新角色；新增的系统角色和绑定行对旧版本无副作用（旧版本鉴权会遍历该角色的接口列表，而它是空的）。 +- 旧版本的“`:param` 匹配任意段”宽松鉴权会随回滚恢复，即 SEC-24 修复的外溢重新出现，回滚需安全负责人知情。 +- 如需彻底清理种子数据（可选，先确认没有其他用户绑定该角色）： + + ```sql + -- 先查 + SELECT r.id, ur.user_id FROM cl_system_roles r + LEFT JOIN cl_system_user_roles ur ON ur.role_id = r.id + WHERE r.code = 'super_admin' AND r.is_system = 1 AND r.deleted_at = 0; + -- 确认后删除（:role_id 为上面查到的 id） + DELETE FROM cl_system_user_roles WHERE role_id = :role_id; + DELETE FROM cl_system_role_apis WHERE role_id = :role_id; + DELETE FROM cl_system_roles WHERE id = :role_id AND is_system = 1; + ``` + 注意：只要新版本再次启动，引导会重新创建并绑定。 + +## 应急 + +- admin 被误撤销超管：重启服务即可由引导重新绑定；不能重启时，用有权限的 DBA 账号插入 `cl_system_user_roles(user_id, role_id)`。 +- 内置超管角色被改库禁用：`UPDATE cl_system_roles SET status = 1 WHERE code='super_admin' AND is_system=1;`，然后让 admin 重新登录（权限每个请求实时读库，无需重启）。 diff --git a/docs/release/sec35-admin-accounttype.md b/docs/release/sec35-admin-accounttype.md new file mode 100644 index 00000000..c4ff3836 --- /dev/null +++ b/docs/release/sec35-admin-accounttype.md @@ -0,0 +1,106 @@ +# SEC-35：存量 admin 行 account_type 由服务账号改为普通用户 + +> **草稿，不执行，等用户批准。** 本文中的 SQL 仅供评审，未在任何数据库上执行；批准前不得执行 UPDATE。 + +## 1. 背景 + +- SEC-27（00cc906）起，`bootstrapAdmin`（`internal/app/bootstrap.go:66`）与 mock 管理员（`mock/user_mock.go:44` `AdminAccountType`）创建的 admin 都使用 `constants.AccountTypeUser`。 +- SEC-27 之前的 mock（188e907:`mock/user_mock.go:39`）以 `AdminAccountType = 2` 创建 admin，代码改动不会修改库中已有行，因此曾开启过 mock 的库里 admin 行的 `account_type` 仍为 2。 +- 常量（`internal/constants/user.go:38-39`）：`AccountTypeUser = 1`（普通用户），`AccountTypeService = 2`（服务账号）。 +- 表名与列名以 model 为准（`internal/model/system_user.go`）：表 `cl_system_users`（`TableName()`），列 `id`、`username`、`account_type`、`updated_at`、`deleted_at`。`deleted_at` 为 `gorm.io/plugin/soft_delete` 的整型软删除列，未删除行为 `0`。 +- 生产库为 MySQL（`gorm.io/driver/mysql`），以下 SQL 按 MySQL 语法书写。 + +## 2. 影响范围 + +- 仅影响 `cl_system_users` 中 `username = 'admin' AND account_type = 2` 的行，预期为 0 或 1 行（用户名无唯一约束，可能多行，须以核对 SQL 实际结果为准）。 +- 不改口令、启用状态、角色、API 关联（`cl_user_apis`）或其他用户；不涉及表结构变更。 +- 包括软删除的 admin 行（`deleted_at <> 0`）：草稿默认一并更正，保持与代码一致；如审批人希望只改有效行，在 WHERE 中追加 `AND deleted_at = 0`。 +- 没有开启过 mock、或 admin 由 `BOOTSTRAP_ADMIN_PASSWORD` 创建的库，核对结果应为 0 行，无需执行。 + +## 3. 执行前备份（必须） + +1. 按 `docs/release/backup-restore.md` 使用 `scripts/release-backup.py` 生成全库单库备份，并完成恢复核验。 +2. 另做一份最小范围的行级备份，便于精确回滚（只保存 id 与原值，不含口令等字段）： + +```sql +-- 草稿，不执行，等用户批准 +CREATE TABLE cl_system_users_bak_sec35 AS +SELECT id, username, account_type, updated_at, deleted_at +FROM cl_system_users +WHERE username = 'admin' AND account_type = 2; +``` + +## 4. 只读核对 SQL + +```sql +-- 草稿，不执行，等用户批准（只读） +-- 4.1 计数：待更正的 admin 行 +SELECT COUNT(*) AS cnt +FROM cl_system_users +WHERE username = 'admin' AND account_type = 2; + +-- 4.2 明细：列出所有 admin 行，便于确认同名多行与软删除情况 +SELECT id, username, account_type, deleted_at +FROM cl_system_users +WHERE username = 'admin' +ORDER BY id; +``` + +执行 UPDATE 前记录 4.1 的计数 N 与 4.2 的 id 列表，并与备份表行数核对一致。 + +## 5. UPDATE 草稿 + +```sql +-- 草稿，不执行，等用户批准 +START TRANSACTION; + +UPDATE cl_system_users +SET account_type = 1 -- constants.AccountTypeUser +WHERE username = 'admin' AND account_type = 2; +-- 期望影响行数 = 第4.1节计数 N；不一致则 ROLLBACK + +-- 复核：应返回 0 +SELECT COUNT(*) FROM cl_system_users WHERE username = 'admin' AND account_type = 2; + +COMMIT; -- 确认影响行数与复核结果后再提交，否则 ROLLBACK; +``` + +说明：不显式修改 `updated_at`，保持与应用层原样；如审批要求留痕，可追加 `, updated_at = NOW()`（回滚时由备份表恢复）。 + +## 6. 回滚 SQL + +```sql +-- 草稿，不执行，等用户批准 +START TRANSACTION; + +UPDATE cl_system_users u +JOIN cl_system_users_bak_sec35 b ON b.id = u.id +SET u.account_type = b.account_type, + u.updated_at = b.updated_at +WHERE u.username = 'admin'; +-- 期望影响行数 = 备份表行数 + +COMMIT; + +-- 确认无需回滚后（观察期结束），再清理备份表： +-- DROP TABLE cl_system_users_bak_sec35; +``` + +若行级备份不可用，用第 3 节的全库备份在独立库恢复后按 id 比对恢复。 + +## 7. 对权限的影响 + +- 权限判断按**用户名**而不是 `account_type`：`internal/middleware/auth.go:155` 只允许 `admin` 调用 `/api/user/signup`，`internal/middleware/auth.go:162` 对 `user.Username == "admin"` 直接放行。 +- `internal/middleware/login.go:93-95` 每次请求会用库中的 `account_type` 覆盖 JWT 中的值，但当前非测试代码中没有任何按 `AccountType` 做的权限分支（服务账号已不再自动绕过角色权限，见 README「生产启动安全要求」）。 +- 因此本次更正**不改变 admin 的任何权限**，也不需要 admin 重新登录；影响仅限于展示与按 `account_type` 过滤的用户列表（`internal/system/service/user_service.go:146-154`），admin 将从「服务账号」筛选结果移到「普通用户」中。 +- 反向风险：若将来引入按 `account_type` 的权限逻辑，未更正的存量 admin 会与新建 admin 行为不一致——这正是 SEC-35 要消除的差异。 + +## 8. 执行顺序（批准后） + +1. 维护窗口内完成第 3 节两份备份； +2. 执行第 4 节只读核对，记录 N 与 id 列表； +3. 执行第 5 节事务，核对影响行数 = N 后提交； +4. 用 admin 登录验证可正常访问用户管理与 `/api/user/signup`； +5. 观察期结束后清理备份表。 + +**草稿，不执行，等用户批准。** diff --git a/docs/release/w1-jwt-config.md b/docs/release/w1-jwt-config.md index 7d78926e..5fe3d8c8 100644 --- a/docs/release/w1-jwt-config.md +++ b/docs/release/w1-jwt-config.md @@ -12,11 +12,11 @@ - 刷新令牌有效期不变（7 天）。前端（`AI-CloudOps-web`）已开启 `enableRefreshToken`：收到 401 后调用 `POST /api/user/refresh_token` 换取新访问令牌并重放原请求，刷新失败才回到登录页。普通 HTTP 请求对用户无感。 - 若部署环境显式设置了 `JWT_EXPIRATION`，以环境变量为准；请勿按秒填写。 -- **上限 1440 分钟（24 小时）**：`Config.Validate` 中超过 1440 时，`ENV=production` 拒绝启动并报错； - 其他环境在启动时记一条告警日志（标准错误输出，`[配置告警]` 开头）后继续启动。 - 生产环境判断统一使用 `pkg/runenv.IsProduction()`：`ENV` 去首尾空白、忽略大小写后等于 `production` - （`production`、` Production `、`PRODUCTION` 均视为生产）。配置文件选择（`internal/config/loader.go`）、 - `Config.Validate` 和 SSH `known_hosts` 要求（`pkg/ssh/hostkey.go`）使用同一判断。 +- **上限 1440 分钟（24 小时）**：`Config.Validate` 中超过 1440 时，生产环境拒绝启动并报错； + 开发环境（`development`/`dev`）在启动时记一条告警日志（标准错误输出，`[配置告警]` 开头）后继续启动。 + 生产环境判断统一使用 `pkg/envmode`（失败即关闭）：`ENV` 去首尾空白、忽略大小写后为 `production`/`prod`/`prd` + 视为生产；未设置、为空或无法识别（如 `test`、`production-x`）时启动入口直接拒绝，`Config.Validate` 也按生产校验。 + 配置文件选择（`internal/config/loader.go`）、`Config.Validate` 和 SSH `known_hosts` 要求（`pkg/ssh/hostkey.go`）使用同一判断。 - `JWT_EXPIRATION` 小于等于 0 时不报错，但记一条告警日志，提示“将使用默认值 30”（`pkg/jwt` 对非正数回退到 30 分钟）。 因此按旧习惯填 `3600` 的生产环境升级后会启动失败，需改为 `30`。 @@ -26,7 +26,8 @@ 需要**重新登录一次**；之后签发的令牌都带 iss，正常续期。建议在发布公告中提前告知。 此外 `pkg/jwt.NewJWTHandler` 现在在 issuer 为空（或全是空白）时返回 `ErrEmptyIssuer`，后端启动失败， -不再签发没有 iss 的令牌。`jwt.issuer` 取自 `config/*.yaml`，可用 `JWT_ISSUER` 覆盖；修改该值同样会使已签发令牌全部失效。 +不再签发没有 iss 的令牌。`jwt.issuer` 取自 `config/*.yaml`，可用 `JWT_ISSUER` 覆盖；发布镜像不含配置文件， +因此发布 compose 显式注入与仓库 YAML 相同的签发者默认值。修改该值同样会使已签发令牌全部失效。 ## 3. JWT 密钥校验（所有环境，含开发环境） diff --git a/docs/security-baseline.md b/docs/security-baseline.md index a4dd7780..06836802 100644 --- a/docs/security-baseline.md +++ b/docs/security-baseline.md @@ -32,3 +32,23 @@ AI网关内部凭据隔离、生产默认密钥/Mock配置校验。 历史 `mock/k8s_mock.go` 曾包含嵌入式客户端私钥和集群地址；当前文件已替换为 无凭据、不可路由的 `example.invalid` 配置。删除当前内容不能撤销 Git 历史中的凭据， 相关集群管理员必须轮换曾公开的客户端证书/密钥并审查访问记录；本次未改写历史或操作真实集群。 + +运行环境由 `ENV` 统一判定（`pkg/envmode`，SEC-27）：取值去除首尾空白并忽略大小写，`production`/`prod`/`prd` 为生产，`development`/`dev`、`test`/`testing` 为非生产；未设置或无法识别时 `config.Load` 拒绝启动，其他调用方（如 `pkg/ssh` 主机密钥策略、mock 兜底）一律按生产处理。 + +历史 `mock/user_mock.go` 曾内置 mock 管理员口令（SEC-27）。当前 mock 在生产环境默认关闭，口令只从 `MOCK_ADMIN_PASSWORD` 读取，未设置时不创建管理员。曾经开启过 mock 的环境中，`admin` 可能仍在使用旧的内置口令；删除代码不会修改已有账号，需要由负责人确认并另行修改口令。 + +启动时会检测用户名为 `admin` 的所有账号是否仍在使用该旧内置口令（源码只保存遮蔽后的字节，运行时在内存中还原并用 bcrypt 比对，用完即清零）：生产环境命中或检查本身读库失败时拒绝启动，非生产环境打印 `[SECURITY]` 告警后继续；不存在 `admin` 时跳过。处置方式：升级前先在当前运行的版本中修改 `admin` 口令（至少12字符）；若已无法启动，由负责人在维护窗口用 bcrypt 生成新口令的哈希，更新用户表中该账号的 `password` 字段后再启动。提示与日志均不包含口令。 + +### 运行环境与弱口令检测的非生产行为 / Non-production behaviour (SEC-27) + +**中文** + +- `ENV=test` 或 `ENV=testing`（去除首尾空白、忽略大小写）归为测试模式（`envmode.Test`），按非生产处理。仓库没有独立的测试配置文件，测试模式与开发模式一样复用 `config/config.development.yaml`（`internal/config/loader.go` `resolveConfigFile`）。 +- 旧内置弱口令检测（`internal/app/legacy_admin_password.go` `checkLegacyAdminPassword`）在非生产环境（development/dev、test/testing）下查库失败时，只打印一条 `[SECURITY]` Warn 并继续启动，不会拒绝启动；生产环境查库失败则拒绝启动（`internal/app/app.go` 中 `Run` 直接返回该错误）。 +- 非生产环境命中旧口令同样只告警；但该库一旦迁移到生产环境就会拒绝启动，应尽早修改 `admin` 口令。 + +**English** + +- `ENV=test` or `ENV=testing` (trimmed, case-insensitive) selects test mode (`envmode.Test`), which is treated as non-production. There is no dedicated test config file: test mode reuses `config/config.development.yaml`, exactly like development (`resolveConfigFile` in `internal/config/loader.go`). +- In non-production environments (development/dev, test/testing), if the legacy built-in weak password check (`checkLegacyAdminPassword` in `internal/app/legacy_admin_password.go`) cannot query the database, it only logs a single `[SECURITY]` warning and startup continues. In production the same query failure refuses startup (`Run` in `internal/app/app.go` returns the error). +- A non-production match on the legacy password is also only a warning, but the same database will refuse to start once promoted to production, so change the `admin` password early. diff --git a/env.example b/env.example index 51f2d16a..46159e1a 100644 --- a/env.example +++ b/env.example @@ -1,4 +1,9 @@ +# 本地开发模板：复制为 .env 后填写；所有密钥类变量留空，禁止提交真实值。 +# 发布栈（deploy/release/compose.yaml）请使用 deploy/release/env.example。 +# 必填：JWT_KEY1、JWT_KEY2、MYSQL_ADDR、TREE_PASSWORD_ENCRYPTION_KEY（webhook 进程另需 WEBHOOK_MYSQL_ADDR；使用根目录 docker-compose.yaml 另需 REDIS_PASSWORD）；条件必填/可选见各项注释。 # 环境配置 +# 后端 ENV 可选值（大小写不敏感，首尾空白忽略）：production/prod/prd（生产）、development/dev（开发）。 +# ENV 未设置、为空（含只有空白）或为其他取值（包括 test/testing）时后端拒绝启动（SEC-27）；APP_ENV 不生效。 ENV=development # ============================================================================ @@ -12,22 +17,35 @@ SERVER_PORT=8889 LOG_DIR=./logs LOG_LEVEL=debug -# JWT配置 -JWT_KEY1=xxx -JWT_KEY2=xxx -JWT_ISSUER=xxx -JWT_EXPIRATION=3600 - -# Redis配置 +# JWT配置（JWT_KEY1、JWT_KEY2 必填；production 要求 ≥32 字符、互不相同） +# 生成: openssl rand -hex 24 +JWT_KEY1= +JWT_KEY2= +# 可选，用来覆盖 config yaml 里的 JWT_ISSUER，一般不要设：设了会覆盖 yaml 中的值，现有 token 将全部失效。 +# JWT_ISSUER= +# JWT_EXPIRATION：访问令牌有效期，单位是分钟（pkg/jwt 按 time.Minute 计算），30 = 30 分钟；请勿按秒填写。 +# 与发布模板 deploy/release/compose.yaml 一致。旧示例值 3600 实为 3600 分钟（60 小时）。 +# 本分支（合入 sched/w1-mantou 前）：不设时取 config/*.yaml 的 jwt.expiration（现为 3600），≤0 时回退 30。 +# 合入 sched/w1-mantou 后：代码与 config/*.yaml 默认 30；超过 1440（24 小时）时 ENV=production 拒绝启动， +# 其他环境告警后继续；≤0 时告警并使用 30。 +JWT_EXPIRATION=30 + +# Redis配置（同时供根目录 docker-compose.yaml 的 redis --requirepass 使用） REDIS_ADDR=localhost:6379 -REDIS_PASSWORD=v6SxhWHyZC7S +# 使用根目录 docker-compose.yaml 时必填：redis 以 --requirepass 启动，compose 用 ${REDIS_PASSWORD:?} 校验，留空会直接报错（生成: openssl rand -hex 24） +# 后端读取同一变量连接 Redis；不用 compose、连接无认证的 Redis 时才可留空。此处保持为空，不要提交真实值 +REDIS_PASSWORD= # MySQL配置 -MYSQL_ADDR=root:root@tcp(localhost:3306)/cloudops?charset=utf8mb4&parseTime=True&loc=Local +# 根目录 docker-compose.yaml 的 mysql root 口令（必填，生成: openssl rand -hex 24） +MYSQL_ROOT_PASSWORD= +# 必填。格式: :@tcp(localhost:3306)/cloudops?charset=utf8mb4&parseTime=True&loc=Local（勿提交真实口令） +MYSQL_ADDR= # 树形结构配置 TREE_CHECK_STATUS_CRON=@every 300s -TREE_PASSWORD_ENCRYPTION_KEY=ebe3vxIP7sblVvUHXb7ZaiMPuz4oXo0l +# 必填，必须恰好 32 字节（AES-256），生成: openssl rand -hex 16；升级时保持不变并与数据库一同备份 +TREE_PASSWORD_ENCRYPTION_KEY= # Kubernetes配置 K8S_REFRESH_CRON=@every 300s @@ -36,13 +54,26 @@ K8S_REFRESH_CRON=@every 300s PROMETHEUS_REFRESH_CRON=@every 15s PROMETHEUS_ENABLE_ALERT=0 PROMETHEUS_ENABLE_RECORD=0 +# 告警推送地址：指向 webhook 进程（cmd/webhook）的 /api/v1/alerts/receive，不是主后端；端口须与 webhook 实际监听端口一致 +# 写法：完整 URL 的 scheme 建议写小写 http:// 或 https://，不区分大小写（w1 119c332 起 alert_cache.go 用 hasPrefixFold， +# HTTP://、Https:// 也按完整 URL 处理；合入 w1 前的旧代码按区分大小写的前缀判断，所以统一写小写）；也可只写 host:port（按 http 处理）。 +# （本分支用仓库 config/webhook.yaml 时为 8889，没有配置文件时为 8888，见下方 WEBHOOK_PORT 说明）。 +# 发布栈（REL-03A 合入后）compose 把 backend 的该值设为 ${PROMETHEUS_ALERT_WEBHOOK_ADDR:-http://webhook:8889/api/v1/alerts/receive}： +# 默认指向 Compose 内的 webhook 服务；只有 Alertmanager 在 Compose 网络外时才在 deploy/release/.env 里覆盖（webhook 默认不发布端口， +# 需自行暴露）。不要把这里的 localhost 值带进发布栈：.env 或 shell 里导出的同名变量都会覆盖 compose 默认值。 PROMETHEUS_ALERT_WEBHOOK_ADDR=http://localhost:8889/api/v1/alerts/receive PROMETHEUS_ALERT_WEBHOOK_FILE_DIR=/tmp/webhook_files -PROMETHEUS_HTTP_SD_API=http://localhost:8888/api/not_auth/getTreeNodeBindIps +# HTTP SD 接口在主后端上，端口与 SERVER_PORT 一致（代码默认 8889）。 +PROMETHEUS_HTTP_SD_API=http://localhost:8889/api/not_auth/getTreeNodeBindIps # Mock配置 MOCK_ENABLED=true +# 定时任务配置（可选） +# CRON_ALLOW_LOCAL_EXEC 默认 false：命令行(2)/脚本(4)类定时任务不在后端主机本地执行，已有的这两类任务会停止执行。 +# 设为 true 可保留这两类任务；scripts/integration/cron_crash_smoke.py 也要求 true。 +# CRON_ALLOW_LOCAL_EXEC= + # ============================================================================ # 通知配置 # ============================================================================ @@ -51,8 +82,9 @@ MOCK_ENABLED=true NOTIFICATION_EMAIL_ENABLED=true NOTIFICATION_EMAIL_SMTP_HOST=smtp.qq.com NOTIFICATION_EMAIL_SMTP_PORT=587 -NOTIFICATION_EMAIL_USERNAME=xxx@qq.com -NOTIFICATION_EMAIL_PASSWORD=xxx +# 条件必填：NOTIFICATION_EMAIL_ENABLED=true 时需要 USERNAME/PASSWORD +NOTIFICATION_EMAIL_USERNAME= +NOTIFICATION_EMAIL_PASSWORD= NOTIFICATION_EMAIL_FROM_NAME=AI-CloudOps NOTIFICATION_EMAIL_MAX_RETRIES=3 NOTIFICATION_EMAIL_RETRY_INTERVAL=5m @@ -61,8 +93,9 @@ NOTIFICATION_EMAIL_USE_TLS=true # 飞书通知配置 NOTIFICATION_FEISHU_ENABLED=true -NOTIFICATION_FEISHU_APP_ID=xxx -NOTIFICATION_FEISHU_APP_SECRET=xxx +# 条件必填：NOTIFICATION_FEISHU_ENABLED=true 时需要 APP_ID/APP_SECRET +NOTIFICATION_FEISHU_APP_ID= +NOTIFICATION_FEISHU_APP_SECRET= NOTIFICATION_FEISHU_WEBHOOK_URL=https://open.feishu.cn/open-apis/bot/v2/hook/ NOTIFICATION_FEISHU_PRIVATE_MESSAGE_API=https://open.feishu.cn/open-apis/im/v1/messages NOTIFICATION_FEISHU_TENANT_ACCESS_TOKEN_API=https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal @@ -75,7 +108,20 @@ NOTIFICATION_FEISHU_TIMEOUT=10s # ============================================================================ # Webhook基础配置 -WEBHOOK_PORT=8888 +# webhook 进程（cmd/webhook）必填：完整 MySQL DSN，格式同 MYSQL_ADDR；webhook 不读 MYSQL_ADDR，需单独设置。 +# 发布栈 deploy/release/compose.yaml 有 webhook 服务：该值由 scripts/release-env.py 写入 deploy/release/.env，经 compose 注入； +# 本机直接运行或单独部署 webhook 时自行设置。 +WEBHOOK_MYSQL_ADDR= +# WEBHOOK_PORT：webhook（cmd/webhook）监听端口；优先级 WEBHOOK_PORT > 配置文件 > 默认 8889。 +# 空串视为未设置；端口须为 1-65535 的纯数字，前导零按十进制处理（检查脚本会更保守地拒绝前导零）。 +# 用 --config 指定配置文件时，port 写在 config/webhook.yaml 同样的 webhook: 根节点下。 +# 平铺 webhook 字段、大小写重复键和多文档会拒绝启动；没有根节点时输出 [配置告警] 后继续启动。 +# 主后端（SERVER_PORT）默认同为 8889；同一台机器同时运行时用 WEBHOOK_PORT 错开端口， +# 同步修改 PROMETHEUS_ALERT_WEBHOOK_ADDR。发布 compose 不传 WEBHOOK_PORT，容器监听镜像配置的 8889。 +WEBHOOK_PORT=8889 +# 整数字段只接受十进制整数（拒绝负数、10.0、前导零及进制前缀），空环境变量视为未设置。 +# fixed_workers: 1-1024；default_upgrade_minutes: 1-10080；common_map_renew_interval_seconds: 1-86400； +# request_timeout_seconds: 1-600，0 不允许；超出范围时拒绝启动。 WEBHOOK_FIXED_WORKERS=10 WEBHOOK_FRONT_DOMAIN=http://localhost:3000 WEBHOOK_BACKEND_DOMAIN=http://localhost:8889 @@ -84,6 +130,9 @@ WEBHOOK_ALERT_MANAGER_API=http://localhost:9093 WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS=300 # Webhook飞书配置 +# 群聊消息地址：webhook 按 content.go SentFeishuGroup 用 "<本值>/<发送组的群机器人 token>" 拼接， +# 所以写到 bot/v2/hook 为止、末尾不加斜杠，拼出 https://open.feishu.cn/open-apis/bot/v2/hook/。 +# 不要写 im/v1/messages（那是私聊/应用消息接口）。 WEBHOOK_IM_FEISHU_GROUP_MESSAGE_API=https://open.feishu.cn/open-apis/bot/v2/hook WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS=10 WEBHOOK_IM_FEISHU_PRIVATE_ROBOT_APP_ID= @@ -95,21 +144,23 @@ WEBHOOK_IM_FEISHU_TENANT_ACCESS_TOKEN_API=https://open.feishu.cn/open-apis/auth/ # ============================================================================ # 阿里云配置 -ALIYUN_ACCESS_KEY_ID=xxx -ALIYUN_ACCESS_KEY_SECRET=xxx +ALIYUN_ACCESS_KEY_ID= +ALIYUN_ACCESS_KEY_SECRET= # LLM配置 -LLM_API_KEY=sk-xxx +LLM_API_KEY= LLM_BASE_URL=https://xxx/v1 # 飞书Webhook -FEISHU_WEBHOOK=https://open.feishu.cn/open-apis/bot/v2/hook/your-webhook-url +# 形如 https://open.feishu.cn/open-apis/bot/v2/hook/，token 属于凭据 +FEISHU_WEBHOOK= # Tavily搜索配置 TAVILY_API_KEY= # AI service gateway (same key as Python service; never expose to the browser) AIOPS_BASE_URL=http://cloudops-aiops:8080 +# 条件必填：production 下配置了 AIOPS_BASE_URL 时需要，≥32 字符 AIOPS_API_KEY= # Exact browser origins, comma-separated. Empty allows same-origin requests only. SERVER_ALLOWED_ORIGINS=http://localhost:5666 diff --git a/go.mod b/go.mod index 626d9faa..7062da3d 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.3.183 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/region v1.3.194 go.uber.org/zap v1.27.1 + go.yaml.in/yaml/v2 v2.4.4 golang.org/x/crypto v0.57.0 golang.org/x/sync v0.23.0 golang.org/x/sys v0.48.0 @@ -224,7 +225,6 @@ require ( go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/arch v0.16.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect diff --git a/internal/app/admin_bootstrap_test.go b/internal/app/admin_bootstrap_test.go new file mode 100644 index 00000000..5956c053 --- /dev/null +++ b/internal/app/admin_bootstrap_test.go @@ -0,0 +1,227 @@ +package app + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/constants" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/mock" + databaseLogger "github.com/GoSimplicity/AI-CloudOps/pkg/logger" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + "golang.org/x/crypto/bcrypt" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// 测试用口令，运行时拼接，不是任何环境的真实口令。 +var ( + bootstrapFixturePassword = "bootstrap-fixture-" + strings.Repeat("b", 6) + mockFixturePassword = "mock-fixture-" + strings.Repeat("m", 8) +) + +func newAdminTestApp(t *testing.T, models ...any) (*App, *gorm.DB, *observer.ObservedLogs) { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: databaseLogger.NewGORMLogger(zap.NewNop())}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + pool.SetMaxOpenConns(1) + t.Cleanup(func() { _ = pool.Close() }) + if len(models) > 0 { + if err := db.AutoMigrate(models...); err != nil { + t.Fatal(err) + } + } + core, logs := observer.New(zapcore.DebugLevel) + return &App{db: db, logger: zap.New(core)}, db, logs +} + +func assertEnvCleared(t *testing.T, keys ...string) { + t.Helper() + for _, key := range keys { + if v := os.Getenv(key); v != "" { + t.Fatalf("%s 读取后应已清除", key) + } + if _, ok := os.LookupEnv(key); ok { + t.Fatalf("%s 应从进程环境中删除", key) + } + } +} + +func assertLogsExclude(t *testing.T, logs *observer.ObservedLogs, secrets ...string) { + t.Helper() + for _, entry := range logs.All() { + text := entry.Message + for _, f := range entry.Context { + text += " " + f.Key + "=" + f.String + if f.Interface != nil { + if e, ok := f.Interface.(error); ok { + text += " " + e.Error() + } + } + } + for _, s := range secrets { + if s != "" && strings.Contains(text, s) { + t.Fatal("日志不得包含口令") + } + } + } +} + +func TestBootstrapAdminQueryErrorPropagates(t *testing.T) { + t.Setenv("ENV", "development") + t.Setenv(BootstrapAdminPasswordEnv, bootstrapFixturePassword) + a, _, logs := newAdminTestApp(t) // 无用户表 + err := a.bootstrapAdmin(context.Background()) + if err == nil || !strings.Contains(err.Error(), "查询用户失败") { + t.Fatalf("用户计数查询失败应透传错误: %v", err) + } + if strings.Contains(err.Error(), bootstrapFixturePassword) { + t.Fatal("错误不得包含口令") + } + assertLogsExclude(t, logs, bootstrapFixturePassword) + assertEnvCleared(t, BootstrapAdminPasswordEnv) +} + +func TestBootstrapAdminShortPassword(t *testing.T) { + short := strings.Repeat("s", MinBootstrapAdminPasswordLen-1) + t.Run("production refuses", func(t *testing.T) { + t.Setenv("ENV", "production") + t.Setenv(BootstrapAdminPasswordEnv, short) + a, db, logs := newAdminTestApp(t, &model.User{}) + err := a.bootstrapAdmin(context.Background()) + if err == nil || !strings.Contains(err.Error(), BootstrapAdminPasswordEnv) || strings.Contains(err.Error(), short) { + t.Fatalf("生产环境短口令应报错且不含口令: %v", err) + } + if n := countAppUsers(t, db); n != 0 { + t.Fatalf("users=%d", n) + } + assertLogsExclude(t, logs, short) + assertEnvCleared(t, BootstrapAdminPasswordEnv) + }) + t.Run("development warns and skips", func(t *testing.T) { + t.Setenv("ENV", "development") + t.Setenv(BootstrapAdminPasswordEnv, short) + a, db, logs := newAdminTestApp(t, &model.User{}) + if err := a.bootstrapAdmin(context.Background()); err != nil { + t.Fatalf("非生产环境短口令只告警: %v", err) + } + if n := countAppUsers(t, db); n != 0 { + t.Fatalf("短口令不应创建管理员, users=%d", n) + } + warns := logs.FilterLevelExact(zapcore.WarnLevel).All() + if len(warns) != 1 || !strings.HasPrefix(warns[0].Message, "[SECURITY]") || !strings.Contains(warns[0].Message, BootstrapAdminPasswordEnv) { + t.Fatalf("应打印显眼的 [SECURITY] Warn 并指明变量: %+v", warns) + } + assertLogsExclude(t, logs, short) + assertEnvCleared(t, BootstrapAdminPasswordEnv) + }) +} + +func TestBootstrapAdminUsesUserAccountTypeAndClearsEnv(t *testing.T) { + t.Setenv("ENV", "production") + t.Setenv(BootstrapAdminPasswordEnv, bootstrapFixturePassword) + a, db, logs := newAdminTestApp(t, &model.User{}) + if err := a.bootstrapAdmin(context.Background()); err != nil { + t.Fatal(err) + } + admin := loadAdmin(t, db) + if admin.AccountType != constants.AccountTypeUser { + t.Fatalf("AccountType=%d want %d", admin.AccountType, constants.AccountTypeUser) + } + assertLogsExclude(t, logs, bootstrapFixturePassword) + assertEnvCleared(t, BootstrapAdminPasswordEnv) +} + +func TestScrubAdminPasswordEnvClearsBoth(t *testing.T) { + t.Setenv(BootstrapAdminPasswordEnv, bootstrapFixturePassword) + t.Setenv(mock.MockAdminPasswordEnv, mockFixturePassword) + (&App{logger: zap.NewNop()}).scrubAdminPasswordEnv() + assertEnvCleared(t, BootstrapAdminPasswordEnv, mock.MockAdminPasswordEnv) +} + +// 与 Run 的顺序一致：bootstrapAdmin -> initMockData（mock 开启时）-> scrubAdminPasswordEnv。 +func TestBootstrapAndMockAdminCombinations(t *testing.T) { + for _, tt := range []struct { + name string + bootstrap, mockPwd string + existingUser bool + wantUsers int64 + wantPassword string // 期望 admin 对应的口令；空表示不应存在 admin + }{ + {"neither set", "", "", false, 0, ""}, + {"only bootstrap", bootstrapFixturePassword, "", false, 1, bootstrapFixturePassword}, + {"only mock", "", mockFixturePassword, false, 1, mockFixturePassword}, + {"both set bootstrap wins", bootstrapFixturePassword, mockFixturePassword, false, 1, bootstrapFixturePassword}, + {"existing users untouched", bootstrapFixturePassword, mockFixturePassword, true, 1, ""}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("ENV", "development") + t.Setenv(BootstrapAdminPasswordEnv, tt.bootstrap) + t.Setenv(mock.MockAdminPasswordEnv, tt.mockPwd) + a, db, logs := newAdminTestApp(t, &model.User{}, &model.Api{}) + if tt.existingUser { + if err := db.Create(&model.User{Username: "someone", Password: "x", RealName: "x", Enable: 1}).Error; err != nil { + t.Fatal(err) + } + } + ctx := context.Background() + if err := a.bootstrapAdmin(ctx); err != nil { + t.Fatal(err) + } + if err := a.initMockData(ctx); err != nil { + t.Fatal(err) + } + a.scrubAdminPasswordEnv() + + if n := countAppUsers(t, db); n != tt.wantUsers { + t.Fatalf("users=%d want %d", n, tt.wantUsers) + } + var admins []model.User + if err := db.Where("username = ?", mock.AdminUsername).Find(&admins).Error; err != nil { + t.Fatal(err) + } + if tt.wantPassword == "" { + if len(admins) != 0 { + t.Fatalf("不应创建 admin: %d", len(admins)) + } + } else { + if len(admins) != 1 { + t.Fatalf("应恰好有一个 admin: %d", len(admins)) + } + if bcrypt.CompareHashAndPassword([]byte(admins[0].Password), []byte(tt.wantPassword)) != nil { + t.Fatal("admin 口令来源不符合预期") + } + if admins[0].AccountType != constants.AccountTypeUser { + t.Fatalf("AccountType=%d", admins[0].AccountType) + } + } + assertLogsExclude(t, logs, bootstrapFixturePassword, mockFixturePassword) + assertEnvCleared(t, BootstrapAdminPasswordEnv, mock.MockAdminPasswordEnv) + }) + } +} + +func countAppUsers(t *testing.T, db *gorm.DB) int64 { + t.Helper() + var n int64 + if err := db.Model(&model.User{}).Count(&n).Error; err != nil { + t.Fatal(err) + } + return n +} + +func loadAdmin(t *testing.T, db *gorm.DB) model.User { + t.Helper() + var admin model.User + if err := db.Where("username = ?", mock.AdminUsername).First(&admin).Error; err != nil { + t.Fatal(err) + } + return admin +} diff --git a/internal/app/aiops.go b/internal/app/aiops.go index fef9ba3d..7d6f7936 100644 --- a/internal/app/aiops.go +++ b/internal/app/aiops.go @@ -3,6 +3,7 @@ package app import ( "context" "fmt" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" jwt "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" ws "github.com/GoSimplicity/AI-CloudOps/pkg/websocket" "net/http" @@ -61,7 +62,11 @@ func (a *App) registerAIOps() error { http.Error(w, `{"code":502,"message":"AI service unavailable"}`, http.StatusBadGateway) }, } - a.server.Any("/api/v1/*path", func(c *gin.Context) { + catalog, err := parseAIOpsCatalog() + if err != nil { + return err + } + a.server.Any(permission.AIOpsProxyTemplate, func(c *gin.Context) { value, exists := c.Get("user") claims, valid := value.(jwt.UserClaims) if !exists || !valid || claims.Uid <= 0 { @@ -71,5 +76,7 @@ func (a *App) registerAIOps() error { request := c.Request.WithContext(context.WithValue(c.Request.Context(), aiopsPrincipalKey{}, strconv.Itoa(claims.Uid))) proxy.ServeHTTP(c.Writer, request) }) + // 代理只有一个通配模板，鉴权需要按目录还原出具体接口模板 + permission.SetProxyCatalog(permission.AIOpsProxyTemplate, catalog) return nil } diff --git a/internal/app/aiops_catalog_test.go b/internal/app/aiops_catalog_test.go new file mode 100644 index 00000000..c4e91757 --- /dev/null +++ b/internal/app/aiops_catalog_test.go @@ -0,0 +1,105 @@ +package app + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/config" + "github.com/GoSimplicity/AI-CloudOps/internal/middleware" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" + "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" +) + +type grantRoles struct { + service.RoleService + apis []*model.Api +} + +func (g grantRoles) GetUserRoles(_ context.Context, _ int) (*model.ListResp[*model.Role], error) { + return &model.ListResp[*model.Role]{Items: []*model.Role{{Status: 1, Apis: g.apis}}, Total: 1}, nil +} + +func TestEmbeddedAIOpsCatalogResolvesEveryEntryToItself(t *testing.T) { + routes, err := parseAIOpsCatalog() + if err != nil { + t.Fatal(err) + } + if len(routes) == 0 { + t.Fatal("empty AIOps catalog") + } + permission.SetProxyCatalog(permission.AIOpsProxyTemplate, routes) + t.Cleanup(func() { permission.SetProxyCatalog(permission.AIOpsProxyTemplate, nil) }) + for _, route := range routes { + concrete := route.Path + for _, part := range strings.Split(route.Path, "/") { + if strings.HasPrefix(part, ":") { + concrete = strings.Replace(concrete, part, "sample-"+part[1:], 1) + } + } + if got := permission.ResolveProxyTemplate(route.Method, permission.AIOpsProxyTemplate, concrete); got != route.Path { + t.Errorf("%s %s resolved to %q", route.Method, concrete, got) + } + } +} + +func TestRegisterAIOpsInstallsAuthorizationCatalog(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) + defer upstream.Close() + t.Cleanup(func() { permission.SetProxyCatalog(permission.AIOpsProxyTemplate, nil) }) + + gin.SetMode(gin.TestMode) + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 7}); c.Next() }) + engine.Use(middleware.NewAuthMiddleware(grantRoles{apis: []*model.Api{ + {Method: 1, Path: "/api/v1/assistant/session/:session_id"}, + {Method: 1, Path: "/api/v1/inspection/report/:report_id"}, + }}).CheckAuth()) + a := &App{cfg: &config.Config{AIOps: config.AIOpsConfig{BaseURL: upstream.URL, APIKey: "test-only"}}, server: engine} + if err := a.registerAIOps(); err != nil { + t.Fatal(err) + } + // 用真实 HTTP 服务承载网关：ReverseProxy 需要 CloseNotifier，ResponseRecorder 不提供 + gateway := httptest.NewServer(engine) + defer gateway.Close() + for _, tt := range []struct { + method, path string + want int + }{ + {"GET", "/api/v1/assistant/session/s-1", 200}, + {"GET", "/api/v1/inspection/report/r-1", 200}, + {"GET", "/api/v1/inspection/report/r-1/markdown", 403}, + {"DELETE", "/api/v1/assistant/session/s-1", 403}, + {"GET", "/api/v1/assistant/session/s-1/extra", 403}, + {"POST", "/api/v1/autofix", 403}, + } { + req, err := http.NewRequest(tt.method, gateway.URL+tt.path, nil) + if err != nil { + t.Fatal(err) + } + resp, err := gateway.Client().Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != tt.want { + t.Errorf("%s %s: status=%d want=%d", tt.method, tt.path, resp.StatusCode, tt.want) + } + } +} + +func TestRegisterAIOpsWithoutBaseURLLeavesNoCatalog(t *testing.T) { + permission.SetProxyCatalog(permission.AIOpsProxyTemplate, nil) + a := &App{cfg: &config.Config{}, server: gin.New()} + if err := a.registerAIOps(); err != nil { + t.Fatal(err) + } + if got := permission.ProxyCatalog(permission.AIOpsProxyTemplate); len(got) != 0 { + t.Fatalf("catalog installed without proxy: %d entries", len(got)) + } +} diff --git a/internal/app/app.go b/internal/app/app.go index 7d7041c3..960958c0 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -138,6 +138,10 @@ func (a *App) Run(ctx context.Context) error { if err := a.bootstrapAdmin(ctx); err != nil { return err } + // SEC-27：在对外提供服务前检查 admin 是否仍在使用旧内置弱口令（生产拒绝启动，非生产告警）。 + if err := a.checkLegacyAdminPassword(ctx); err != nil { + return err + } a.server.GET("/", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{ @@ -155,6 +159,13 @@ func (a *App) Run(ctx context.Context) error { a.logger.Warn("Mock数据初始化失败", zap.Error(err)) } } + // 管理员口令只在启动阶段使用：无论是否读取过，都在启动 cron/asynq 等可能派生子进程的组件前清除。 + a.scrubAdminPasswordEnv() + + // 必须在管理员账号可能被创建（bootstrapAdmin / Mock）之后执行 + if err := a.bootstrapSuperAdmin(ctx); err != nil { + return fmt.Errorf("初始化超级管理员角色失败: %w", err) + } if err := a.initK8sClients(ctx); err != nil { a.logger.Warn("K8s客户端初始化失败", zap.Error(err)) @@ -279,6 +290,10 @@ func (a *App) initMockData(ctx context.Context) error { if a.db == nil { return fmt.Errorf("数据库连接为空") } + // 纵深防御：配置校验已拒绝生产环境开启 mock，这里再兜底一次，避免绕过 config.Load 的调用方写入 mock 数据。 + if config.IsProduction() { + return fmt.Errorf("生产环境禁止初始化 Mock 数据") + } startCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() diff --git a/internal/app/bootstrap.go b/internal/app/bootstrap.go index a4e8bc14..a30c1406 100644 --- a/internal/app/bootstrap.go +++ b/internal/app/bootstrap.go @@ -5,19 +5,52 @@ import ( "fmt" "os" + "github.com/GoSimplicity/AI-CloudOps/internal/config" + "github.com/GoSimplicity/AI-CloudOps/internal/constants" "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/mock" databaseLogger "github.com/GoSimplicity/AI-CloudOps/pkg/logger" + "go.uber.org/zap" "golang.org/x/crypto/bcrypt" ) +const ( + // BootstrapAdminPasswordEnv 是首次启动创建管理员所用口令的环境变量名。 + BootstrapAdminPasswordEnv = "BOOTSTRAP_ADMIN_PASSWORD" + // MinBootstrapAdminPasswordLen 是 BOOTSTRAP_ADMIN_PASSWORD 的最小长度。 + MinBootstrapAdminPasswordLen = 12 +) + +// takeSecretEnv 读取口令类环境变量后立即从进程环境中删除，避免被 cron 等子进程继承。 +func (a *App) takeSecretEnv(key string) string { + value := os.Getenv(key) + if err := os.Unsetenv(key); err != nil && a.logger != nil { + a.logger.Warn("[SECURITY] 清除口令环境变量失败，子进程可能继承该变量", zap.String("env", key), zap.Error(err)) + } + return value +} + +// scrubAdminPasswordEnv 兜底清除管理员口令环境变量（例如未开启 mock 时 MOCK_ADMIN_PASSWORD 不会被读取）。 +func (a *App) scrubAdminPasswordEnv() { + for _, key := range []string{BootstrapAdminPasswordEnv, mock.MockAdminPasswordEnv} { + _ = a.takeSecretEnv(key) + } +} + // bootstrapAdmin creates an administrator only on an empty installation. +// 口令读取后立即从进程环境中删除。口令过短时：生产环境拒绝启动；非生产环境打印 [SECURITY] Warn 并跳过创建。 func (a *App) bootstrapAdmin(ctx context.Context) error { - password := os.Getenv("BOOTSTRAP_ADMIN_PASSWORD") + password := a.takeSecretEnv(BootstrapAdminPasswordEnv) if password == "" { return nil } - if len(password) < 12 { - return fmt.Errorf("BOOTSTRAP_ADMIN_PASSWORD 至少12字符") + if len(password) < MinBootstrapAdminPasswordLen { + if config.IsProduction() { + return fmt.Errorf("%s 至少%d字符", BootstrapAdminPasswordEnv, MinBootstrapAdminPasswordLen) + } + a.logger.Warn(fmt.Sprintf("[SECURITY] %s 少于%d字符，非生产环境跳过创建初始管理员；请改为至少%d字符的强口令后重启（生产环境会拒绝启动）", + BootstrapAdminPasswordEnv, MinBootstrapAdminPasswordLen, MinBootstrapAdminPasswordLen)) + return nil } var count int64 if err := a.db.WithContext(ctx).Model(&model.User{}).Count(&count).Error; err != nil { @@ -30,6 +63,6 @@ func (a *App) bootstrapAdmin(ctx context.Context) error { if err != nil { return err } - err = a.db.WithContext(ctx).Create(&model.User{Username: "admin", Password: string(hash), RealName: "管理员", AccountType: 1, Enable: 1, HomePath: "/"}).Error + err = a.db.WithContext(ctx).Create(&model.User{Username: "admin", Password: string(hash), RealName: "管理员", AccountType: constants.AccountTypeUser, Enable: 1, HomePath: "/"}).Error return databaseLogger.WrapDatabaseError("初始化管理员: 创建用户失败", err) } diff --git a/internal/app/bootstrap_superadmin.go b/internal/app/bootstrap_superadmin.go new file mode 100644 index 00000000..d7375758 --- /dev/null +++ b/internal/app/bootstrap_superadmin.go @@ -0,0 +1,80 @@ +package app + +import ( + "context" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" + databaseLogger "github.com/GoSimplicity/AI-CloudOps/pkg/logger" + "go.uber.org/zap" + "gorm.io/gorm" +) + +// bootstrapSuperAdmin 幂等地确保内置超管角色存在，并把内置 admin 账号绑定到该角色。 +// 运行期鉴权只认角色，不认用户名，因此这一步是 admin 不被锁在门外的前提。 +func (a *App) bootstrapSuperAdmin(ctx context.Context) error { + return ensureSuperAdmin(ctx, a.db, a.logger) +} + +func ensureSuperAdmin(ctx context.Context, db *gorm.DB, logger *zap.Logger) error { + if logger == nil { + logger = zap.NewNop() + } + return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + var roles []model.Role + if err := tx.Where("code = ?", permission.SuperAdminRoleCode).Order("id").Find(&roles).Error; err != nil { + return databaseLogger.WrapDatabaseError("初始化超级管理员角色: 查询角色失败", err) + } + var role *model.Role + for i := range roles { + if permission.IsSuperAdminIdentity(&roles[i]) { + if role == nil { + role = &roles[i] + } + continue + } + // 普通角色借用了超管编码：不提升它，只提示人工清理 + logger.Warn("存在使用超管编码的非系统角色，该角色不具备超管权限", + zap.Int("role_id", roles[i].ID), zap.String("role_name", roles[i].Name)) + } + if role == nil { + role = &model.Role{ + Name: permission.SuperAdminRoleName, + Code: permission.SuperAdminRoleCode, + Description: permission.SuperAdminRoleDescription, + Status: 1, + IsSystem: 1, + } + if err := tx.Create(role).Error; err != nil { + return databaseLogger.WrapDatabaseError("初始化超级管理员角色: 创建角色失败", err) + } + logger.Info("已创建内置超级管理员角色", zap.Int("role_id", role.ID)) + } else if role.Status != 1 { + logger.Warn("内置超级管理员角色已被禁用，启动引导不会自动启用；持有该角色的账号当前没有超管权限", + zap.Int("role_id", role.ID)) + } + + var users []model.User + if err := tx.Select("id", "username").Where("username = ?", permission.BootstrapAdminUsername).Find(&users).Error; err != nil { + return databaseLogger.WrapDatabaseError("初始化超级管理员角色: 查询管理员失败", err) + } + for _, user := range users { + // MySQL 默认排序规则大小写不敏感，这里再按字节精确比较，避免绑定 "Admin" 等账号 + if user.Username != permission.BootstrapAdminUsername { + continue + } + var bound int64 + if err := tx.Model(&model.UserRole{}).Where("user_id = ? AND role_id = ?", user.ID, role.ID).Count(&bound).Error; err != nil { + return databaseLogger.WrapDatabaseError("初始化超级管理员角色: 查询绑定失败", err) + } + if bound > 0 { + continue + } + if err := tx.Create(&model.UserRole{UserID: user.ID, RoleID: role.ID}).Error; err != nil { + return databaseLogger.WrapDatabaseError("初始化超级管理员角色: 绑定管理员失败", err) + } + logger.Info("已将内置管理员绑定到超级管理员角色", zap.Int("user_id", user.ID), zap.Int("role_id", role.ID)) + } + return nil + }) +} diff --git a/internal/app/bootstrap_superadmin_test.go b/internal/app/bootstrap_superadmin_test.go new file mode 100644 index 00000000..6602eb33 --- /dev/null +++ b/internal/app/bootstrap_superadmin_test.go @@ -0,0 +1,180 @@ +package app + +import ( + "context" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/middleware" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" + "github.com/GoSimplicity/AI-CloudOps/internal/system/dao" + "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" + "go.uber.org/zap" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func openRBACTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "rbac.db")), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatal(err) + } + if err = db.AutoMigrate(&model.User{}, &model.Api{}, &model.Role{}, &model.RoleApi{}, &model.UserRole{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if sqlDB, err := db.DB(); err == nil { + _ = sqlDB.Close() + } + }) + return db +} + +func superRoles(t *testing.T, db *gorm.DB) []model.Role { + t.Helper() + var roles []model.Role + if err := db.Where("code = ? AND is_system = 1", permission.SuperAdminRoleCode).Find(&roles).Error; err != nil { + t.Fatal(err) + } + return roles +} + +func isSuper(t *testing.T, db *gorm.DB, userID int) bool { + t.Helper() + roles, err := dao.NewRoleDAO(db).GetRoles(context.Background(), userID) + if err != nil { + t.Fatal(err) + } + return permission.HasSuperAdminRole(roles) +} + +func TestEnsureSuperAdminIsIdempotentAndBindsOnlyAdmin(t *testing.T) { + db := openRBACTestDB(t) + admin := model.User{Username: "admin", Enable: 1} + lookalike := model.User{Username: "Admin", Enable: 1} + member := model.User{Username: "member", Enable: 1} + for _, u := range []*model.User{&admin, &lookalike, &member} { + if err := db.Create(u).Error; err != nil { + t.Fatal(err) + } + } + // 历史上手工建的同编码普通角色，不能被提升 + custom := model.Role{Name: "自建超管", Code: permission.SuperAdminRoleCode, Status: 1} + if err := db.Create(&custom).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.UserRole{UserID: member.ID, RoleID: custom.ID}).Error; err != nil { + t.Fatal(err) + } + + for i := 0; i < 3; i++ { + if err := ensureSuperAdmin(context.Background(), db, zap.NewNop()); err != nil { + t.Fatalf("run %d: %v", i, err) + } + } + + roles := superRoles(t, db) + if len(roles) != 1 || roles[0].Status != 1 || roles[0].Name != permission.SuperAdminRoleName { + t.Fatalf("unexpected system roles: %+v", roles) + } + var custom2 model.Role + if err := db.First(&custom2, custom.ID).Error; err != nil || custom2.IsSystem != 0 { + t.Fatalf("custom role promoted: %+v err=%v", custom2, err) + } + var bindings []model.UserRole + if err := db.Where("role_id = ?", roles[0].ID).Find(&bindings).Error; err != nil { + t.Fatal(err) + } + if len(bindings) != 1 || bindings[0].UserID != admin.ID { + t.Fatalf("bindings=%+v want only admin(%d)", bindings, admin.ID) + } + if !isSuper(t, db, admin.ID) || isSuper(t, db, lookalike.ID) || isSuper(t, db, member.ID) { + t.Fatal("super admin identity resolved incorrectly") + } +} + +func TestEnsureSuperAdminRebindsAndDoesNotReenable(t *testing.T) { + db := openRBACTestDB(t) + if err := ensureSuperAdmin(context.Background(), db, nil); err != nil { + t.Fatal(err) + } + if n := len(superRoles(t, db)); n != 1 { + t.Fatalf("empty install created %d roles", n) + } + // admin 在角色之后才创建（例如 Mock），下次启动应补绑 + admin := model.User{Username: "admin", Enable: 1} + if err := db.Create(&admin).Error; err != nil { + t.Fatal(err) + } + if err := ensureSuperAdmin(context.Background(), db, nil); err != nil { + t.Fatal(err) + } + if !isSuper(t, db, admin.ID) { + t.Fatal("admin not bound on second boot") + } + // 人为撤销后重启会重新绑定 + if err := db.Where("user_id = ?", admin.ID).Delete(&model.UserRole{}).Error; err != nil { + t.Fatal(err) + } + if err := ensureSuperAdmin(context.Background(), db, nil); err != nil { + t.Fatal(err) + } + if !isSuper(t, db, admin.ID) { + t.Fatal("admin not re-bound after revoke") + } + // 被直接改库禁用的超管角色不会被自动启用 + if err := db.Model(&model.Role{}).Where("code = ?", permission.SuperAdminRoleCode).Update("status", 0).Error; err != nil { + t.Fatal(err) + } + if err := ensureSuperAdmin(context.Background(), db, nil); err != nil { + t.Fatal(err) + } + if roles := superRoles(t, db); len(roles) != 1 || roles[0].Status != 0 { + t.Fatalf("disabled role changed: %+v", roles) + } + if isSuper(t, db, admin.ID) { + t.Fatal("disabled super role still grants super admin") + } +} + +// 端到端：真实 RoleDAO/RoleService + 真实鉴权中间件，admin 经引导后仍能使用 +// 仅超管可用的接口，而同名但未绑定角色的账号不能。 +func TestBootstrappedAdminKeepsAccessThroughRealMiddleware(t *testing.T) { + db := openRBACTestDB(t) + admin := model.User{Username: "admin", Enable: 1} + member := model.User{Username: "member", Enable: 1} + for _, u := range []*model.User{&admin, &member} { + if err := db.Create(u).Error; err != nil { + t.Fatal(err) + } + } + if err := ensureSuperAdmin(context.Background(), db, nil); err != nil { + t.Fatal(err) + } + roles := service.NewRoleService(dao.NewRoleDAO(db), zap.NewNop()) + for _, tt := range []struct { + claims jwt.UserClaims + want int + }{ + {jwt.UserClaims{Uid: admin.ID, Username: "admin"}, 204}, + {jwt.UserClaims{Uid: member.ID, Username: "member"}, 403}, + // 伪造用户名没有用：特权跟着 uid 的角色走 + {jwt.UserClaims{Uid: member.ID, Username: "admin"}, 403}, + } { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", tt.claims); c.Next() }) + engine.Use(middleware.NewAuthMiddleware(roles).CheckAuth()) + engine.POST("/api/user/signup", func(c *gin.Context) { c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("POST", "/api/user/signup", nil)) + if rec.Code != tt.want { + t.Fatalf("%+v: status=%d want=%d", tt.claims, rec.Code, tt.want) + } + } +} diff --git a/internal/app/legacy_admin_password.go b/internal/app/legacy_admin_password.go new file mode 100644 index 00000000..94af0ef2 --- /dev/null +++ b/internal/app/legacy_admin_password.go @@ -0,0 +1,107 @@ +/* + * MIT License + * + * Copyright (c) 2024 Bamboo + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package app + +import ( + "context" + "errors" + "fmt" + + "github.com/GoSimplicity/AI-CloudOps/internal/config" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/mock" + databaseLogger "github.com/GoSimplicity/AI-CloudOps/pkg/logger" + "go.uber.org/zap" + "golang.org/x/crypto/bcrypt" +) + +// SEC-27：SEC-27 之前 mock/user_mock.go 会用一个内置的弱口令创建 admin。删除代码不会修改已有账号， +// 因此启动时检测 admin 是否仍在使用该口令。 +// +// 为了比对 bcrypt 哈希，运行时必须拿到候选口令本身；为避免明文出现在源码、日志和扫描结果中， +// 源码只保存候选口令与 legacyMockAdminMaskKey 按字节循环异或后的结果，在内存中还原、比对后立即清零。 +// 这只是防止误抄和扫描命中的遮蔽，不是加密：任何人都能还原它，而该口令本身已存在于公开的 Git 历史中。 +var legacyMockAdminMasked = []byte{0x20, 0x2d, 0x40, 0x2a, 0x02} + +// legacyMockAdminMaskKey 是遮蔽用的固定字节序列，不是密钥。 +const legacyMockAdminMaskKey = "AI-CloudOps/SEC-27/legacy-mock-admin" + +// ErrLegacyAdminPassword 表示 admin 仍在使用旧的内置弱口令；提示中不包含口令。 +var ErrLegacyAdminPassword = errors.New("[SECURITY] admin 账号仍在使用 SEC-27 之前 mock 内置的弱口令，生产环境拒绝启动；" + + "请先为 admin 设置新的强口令（至少12字符）后再启动，处置方式见 docs/security-baseline.md（SEC-27）") + +// legacyMockAdminCandidate 在内存中还原旧内置口令，调用方用完后应清零。 +func legacyMockAdminCandidate() []byte { + out := make([]byte, len(legacyMockAdminMasked)) + for i, b := range legacyMockAdminMasked { + out[i] = b ^ legacyMockAdminMaskKey[i%len(legacyMockAdminMaskKey)] + } + return out +} + +// legacyAdminPasswordInUse 检查用户名为 admin 的账号是否仍在使用旧内置口令。 +// 用户名当前没有唯一约束，因此会检查所有同名行；不存在 admin 时返回 false。 +func (a *App) legacyAdminPasswordInUse(ctx context.Context) (bool, error) { + var admins []model.User + if err := a.db.WithContext(ctx).Select("id", "password").Where("username = ?", mock.AdminUsername).Find(&admins).Error; err != nil { + return false, databaseLogger.WrapDatabaseError("检查管理员弱口令: 查询用户失败", err) + } + if len(admins) == 0 { + return false, nil + } + candidate := legacyMockAdminCandidate() + defer clear(candidate) + for _, admin := range admins { + if bcrypt.CompareHashAndPassword([]byte(admin.Password), candidate) == nil { + return true, nil + } + } + return false, nil +} + +// checkLegacyAdminPassword 执行旧内置弱口令策略： +// - 生产环境：admin 仍在使用旧口令或检查本身失败时拒绝启动； +// - 非生产环境：只打印显眼的 [SECURITY] Warn 并继续启动； +// - admin 不存在时跳过。 +func (a *App) checkLegacyAdminPassword(ctx context.Context) error { + inUse, err := a.legacyAdminPasswordInUse(ctx) + production := config.IsProduction() + if err != nil { + if production { + return fmt.Errorf("[SECURITY] 无法确认 admin 是否仍在使用旧内置弱口令，生产环境拒绝启动: %w", err) + } + a.logger.Warn("[SECURITY] 无法确认 admin 是否仍在使用旧内置弱口令（非生产环境继续启动），请检查数据库连接与用户表", zap.Error(err)) + return nil + } + if !inUse { + return nil + } + if production { + return ErrLegacyAdminPassword + } + a.logger.Warn("[SECURITY] admin 账号仍在使用 SEC-27 之前 mock 内置的弱口令（非生产环境仅告警）；" + + "请立即在用户管理中修改 admin 口令（至少12字符），否则该库迁移到生产环境后将拒绝启动") + return nil +} diff --git a/internal/app/legacy_admin_password_test.go b/internal/app/legacy_admin_password_test.go new file mode 100644 index 00000000..6da13ad7 --- /dev/null +++ b/internal/app/legacy_admin_password_test.go @@ -0,0 +1,203 @@ +package app + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/mock" + databaseLogger "github.com/GoSimplicity/AI-CloudOps/pkg/logger" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + "golang.org/x/crypto/bcrypt" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// historicalCandidateBcrypt 是用 188e907:mock/user_mock.go 中的旧内置口令离线生成的 bcrypt 哈希（cost=4）， +// 用于确认 legacyMockAdminCandidate 的还原结果与历史值一致；测试中不出现口令明文。 +const historicalCandidateBcrypt = "$2a$04$o/sQMuK0K6ikk.gPF/52GeMMQg1k7jHR2yYmsrG777oVtYVMjEA3e" + +func newLegacyCheckApp(t *testing.T, migrate bool) (*App, *gorm.DB, *observer.ObservedLogs) { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: databaseLogger.NewGORMLogger(zap.NewNop())}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + pool.SetMaxOpenConns(1) + t.Cleanup(func() { _ = pool.Close() }) + if migrate { + if err := db.AutoMigrate(&model.User{}); err != nil { + t.Fatal(err) + } + } + core, logs := observer.New(zapcore.DebugLevel) + return &App{db: db, logger: zap.New(core)}, db, logs +} + +func legacyCandidateHash(t *testing.T) string { + t.Helper() + candidate := legacyMockAdminCandidate() + defer clear(candidate) + hash, err := bcrypt.GenerateFromPassword(candidate, bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + return string(hash) +} + +func strongHash(t *testing.T) string { + t.Helper() + hash, err := bcrypt.GenerateFromPassword([]byte("fixture-"+strings.Repeat("z", 12)), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + return string(hash) +} + +func createUser(t *testing.T, db *gorm.DB, username, hash string) { + t.Helper() + if err := db.Create(&model.User{Username: username, Password: hash, RealName: "fixture", Enable: 1}).Error; err != nil { + t.Fatal(err) + } +} + +func TestLegacyCandidateMatchesHistoricalValue(t *testing.T) { + candidate := legacyMockAdminCandidate() + defer clear(candidate) + if len(candidate) == 0 { + t.Fatal("候选值不能为空") + } + if err := bcrypt.CompareHashAndPassword([]byte(historicalCandidateBcrypt), candidate); err != nil { + t.Fatalf("候选值还原结果与 188e907 的历史值不一致: %v", err) + } + // 每次调用返回独立副本，调用方清零不影响后续比对。 + clear(candidate) + again := legacyMockAdminCandidate() + defer clear(again) + if bcrypt.CompareHashAndPassword([]byte(historicalCandidateBcrypt), again) != nil { + t.Fatal("清零调用方副本后不应影响下一次还原") + } +} + +func TestLegacyAdminPasswordRefusedInProduction(t *testing.T) { + for _, env := range []string{"production", "PROD", " prd ", "", "test", "testing"} { + t.Run(env, func(t *testing.T) { + t.Setenv("ENV", env) + a, db, logs := newLegacyCheckApp(t, true) + createUser(t, db, mock.AdminUsername, legacyCandidateHash(t)) + err := a.checkLegacyAdminPassword(context.Background()) + if !errors.Is(err, ErrLegacyAdminPassword) { + t.Fatalf("生产环境应拒绝启动, err=%v", err) + } + if !strings.Contains(err.Error(), "[SECURITY]") || !strings.Contains(err.Error(), "强口令") { + t.Fatalf("提示应显眼并说明如何修复: %v", err) + } + if logs.Len() != 0 { + t.Fatalf("拒绝启动时不应额外打印日志: %d", logs.Len()) + } + }) + } +} + +func TestLegacyAdminPasswordWarnsOutsideProduction(t *testing.T) { + for _, env := range []string{"development", "dev", " DEV "} { + t.Run(env, func(t *testing.T) { + t.Setenv("ENV", env) + a, db, logs := newLegacyCheckApp(t, true) + createUser(t, db, mock.AdminUsername, legacyCandidateHash(t)) + if err := a.checkLegacyAdminPassword(context.Background()); err != nil { + t.Fatalf("非生产环境只告警: %v", err) + } + entries := logs.FilterLevelExact(zapcore.WarnLevel).All() + if len(entries) != 1 || !strings.HasPrefix(entries[0].Message, "[SECURITY]") || !strings.Contains(entries[0].Message, "修改 admin 口令") { + t.Fatalf("应打印一条显眼的 [SECURITY] Warn: %+v", entries) + } + // 告警只包含固定文案，不携带任何口令或哈希字段。 + if len(entries[0].Context) != 0 { + t.Fatalf("告警不应携带字段: %+v", entries[0].Context) + } + }) + } +} + +func TestLegacyAdminPasswordSkipsWhenAdminMissing(t *testing.T) { + for _, env := range []string{"production", "development"} { + t.Run(env, func(t *testing.T) { + t.Setenv("ENV", env) + a, db, logs := newLegacyCheckApp(t, true) + // 其他用户使用旧口令不在本检查范围内。 + createUser(t, db, "someone", legacyCandidateHash(t)) + if err := a.checkLegacyAdminPassword(context.Background()); err != nil { + t.Fatal(err) + } + if logs.Len() != 0 { + t.Fatalf("admin 不存在时应静默跳过: %d", logs.Len()) + } + }) + } +} + +func TestLegacyAdminPasswordMismatchPasses(t *testing.T) { + t.Setenv("ENV", "production") + a, db, logs := newLegacyCheckApp(t, true) + createUser(t, db, mock.AdminUsername, strongHash(t)) + if err := a.checkLegacyAdminPassword(context.Background()); err != nil { + t.Fatal(err) + } + if logs.Len() != 0 { + t.Fatalf("口令不匹配时不应告警: %d", logs.Len()) + } +} + +func TestLegacyAdminPasswordChecksEveryAdminRow(t *testing.T) { + t.Setenv("ENV", "production") + a, db, _ := newLegacyCheckApp(t, true) + createUser(t, db, mock.AdminUsername, strongHash(t)) + createUser(t, db, mock.AdminUsername, legacyCandidateHash(t)) + if err := a.checkLegacyAdminPassword(context.Background()); !errors.Is(err, ErrLegacyAdminPassword) { + t.Fatalf("任一同名 admin 使用旧口令都应拒绝, err=%v", err) + } +} + +func TestLegacyAdminPasswordQueryErrorPropagates(t *testing.T) { + t.Run("production refuses", func(t *testing.T) { + t.Setenv("ENV", "production") + a, _, _ := newLegacyCheckApp(t, false) // 无用户表，查询必然失败 + err := a.checkLegacyAdminPassword(context.Background()) + if err == nil || errors.Is(err, ErrLegacyAdminPassword) || !strings.Contains(err.Error(), "查询用户失败") { + t.Fatalf("生产环境读库失败应透传错误并拒绝启动: %v", err) + } + }) + t.Run("production keeps cancellation", func(t *testing.T) { + t.Setenv("ENV", "production") + a, _, _ := newLegacyCheckApp(t, true) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := a.checkLegacyAdminPassword(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("应保留原始错误链: %v", err) + } + }) + t.Run("development warns", func(t *testing.T) { + t.Setenv("ENV", "development") + a, _, logs := newLegacyCheckApp(t, false) + if err := a.checkLegacyAdminPassword(context.Background()); err != nil { + t.Fatalf("非生产环境读库失败只告警: %v", err) + } + entries := logs.FilterLevelExact(zapcore.WarnLevel).All() + if len(entries) != 1 || !strings.HasPrefix(entries[0].Message, "[SECURITY]") { + t.Fatalf("应打印 [SECURITY] Warn: %+v", entries) + } + }) + t.Run("in-use helper returns error", func(t *testing.T) { + a, _, _ := newLegacyCheckApp(t, false) + inUse, err := a.legacyAdminPasswordInUse(context.Background()) + if err == nil || inUse { + t.Fatalf("inUse=%v err=%v", inUse, err) + } + }) +} diff --git a/internal/app/mock_guard_test.go b/internal/app/mock_guard_test.go new file mode 100644 index 00000000..650e912e --- /dev/null +++ b/internal/app/mock_guard_test.go @@ -0,0 +1,27 @@ +package app + +import ( + "context" + "strings" + "testing" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestInitMockDataRefusedInProduction(t *testing.T) { + t.Setenv("ENV", "production") + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + defer pool.Close() + err = (&App{db: db}).initMockData(context.Background()) + if err == nil || !strings.Contains(err.Error(), "生产环境") { + t.Fatalf("err=%v", err) + } + if tables, _ := db.Migrator().GetTables(); len(tables) != 0 { + t.Fatalf("生产环境不应写入任何 mock 数据: %v", tables) + } +} diff --git a/internal/app/permissions.go b/internal/app/permissions.go index eb2cd5bd..29840326 100644 --- a/internal/app/permissions.go +++ b/internal/app/permissions.go @@ -6,38 +6,93 @@ import ( _ "embed" "encoding/json" "fmt" + "sort" "strings" "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" "github.com/gin-gonic/gin" + "go.uber.org/zap" "gorm.io/gorm/clause" ) //go:embed aiops_permissions.json var aiopsPermissionCatalog []byte +// namedPermission 需要稳定权限名和说明的接口（敏感接口），其余接口按路由自动生成名称 +type namedPermission struct { + Name string + Description string +} + +// namedPermissions 以 "METHOD 路由" 为键登记的权限码 +var namedPermissions = map[string]namedPermission{ + "GET " + model.K8sSecretRevealRoute: { + Name: model.K8sSecretRevealPermissionName, + Description: "查看 K8s Secret 明文（SEC-25，每次调用写审计；仅显式授权生效，通配符授权不覆盖）", + }, + "GET " + model.K8sServiceAccountTokenRoute: { + Name: model.K8sServiceAccountTokenPermissionName, + Description: "读取 K8s ServiceAccount token Secret（SEC-29，每次调用写审计；仅显式授权生效，通配符授权不覆盖）", + }, + "POST " + model.K8sServiceAccountTokenRoute: { + Name: model.K8sServiceAccountTokenCreatePermissionName, + Description: "通过 TokenRequest 签发 K8s ServiceAccount token（SEC-29，每次调用写审计；仅显式授权生效，通配符授权不覆盖）", + }, +} + +// autoPermissionName 未登记权限码的接口按路由自动生成的名称 +func autoPermissionName(route gin.RouteInfo) string { + digest := sha256.Sum256([]byte(route.Method + " " + route.Path)) + handler := route.Handler[strings.LastIndex(route.Handler, ".")+1:] + if len(handler) > 25 { + handler = handler[:25] + } + return fmt.Sprintf("%s %s %x", route.Method, handler, digest[:4]) +} + +// autoPermissionDescription 未登记权限码的接口自动生成的说明：" " +func autoPermissionDescription(route gin.RouteInfo) string { + return route.Method + " " + route.Path +} + +// parseAIOpsCatalog 解析内嵌的 AI 接口目录。该目录既用于补充接口元数据， +// 也登记为 /api/v1/*path 代理的鉴权目录（见 permission.ResolveProxyTemplate）。 +func parseAIOpsCatalog() ([]permission.Route, error) { + var routes []permission.Route + if err := json.Unmarshal(aiopsPermissionCatalog, &routes); err != nil { + return nil, fmt.Errorf("解析AI接口权限目录: %w", err) + } + prefix := strings.TrimSuffix(permission.AIOpsProxyTemplate, "*path") + for _, route := range routes { + if _, ok := permission.MethodCode(route.Method); !ok || !strings.HasPrefix(route.Path, prefix) { + return nil, fmt.Errorf("AI接口权限目录包含非法条目: %s %s", route.Method, route.Path) + } + } + return routes, nil +} + // bootstrapPermissionCatalog 补充可发现接口，不授予角色或覆盖管理员维护的元数据 func (a *App) bootstrapPermissionCatalog(ctx context.Context) error { methods := map[string]int8{"GET": 1, "POST": 2, "PUT": 3, "DELETE": 4} public := map[string]bool{"/api/user/login": true, "/api/user/logout": true, "/api/user/refresh_token": true, "/api/not_auth/getBindIps": true, "/api/not_auth/getTreeNodeBindIps": true} var existing []model.Api - if err := a.db.WithContext(ctx).Select("path", "method").Find(&existing).Error; err != nil { + if err := a.db.WithContext(ctx).Select("id", "name", "path", "method", "description").Find(&existing).Error; err != nil { return err } seen := make(map[string]bool, len(existing)) + existingByKey := make(map[string]model.Api, len(existing)) key := func(method int8, path string) string { return fmt.Sprintf("%d %s", method, path) } for _, api := range existing { seen[key(api.Method, api.Path)] = true + existingByKey[key(api.Method, api.Path)] = api } routes := a.server.Routes() for _, route := range routes { - if route.Path == "/api/v1/*path" { - var aiRoutes []struct { - Method string `json:"method"` - Path string `json:"path"` - } - if err := json.Unmarshal(aiopsPermissionCatalog, &aiRoutes); err != nil { - return fmt.Errorf("解析AI接口权限目录: %w", err) + if route.Path == permission.AIOpsProxyTemplate { + aiRoutes, err := parseAIOpsCatalog() + if err != nil { + return err } for _, aiRoute := range aiRoutes { routes = append(routes, gin.RouteInfo{Method: aiRoute.Method, Path: aiRoute.Path, Handler: "AIOpsProxy"}) @@ -48,19 +103,96 @@ func (a *App) bootstrapPermissionCatalog(ctx context.Context) error { pending := make([]model.Api, 0) for _, route := range routes { method, ok := methods[route.Method] - if !ok || !strings.HasPrefix(route.Path, "/api/") || public[route.Path] || seen[key(method, route.Path)] { + if !ok || !strings.HasPrefix(route.Path, "/api/") || public[route.Path] { + continue + } + if seen[key(method, route.Path)] { + a.renameAutoNamedPermission(ctx, route, existingByKey[key(method, route.Path)]) continue } - digest := sha256.Sum256([]byte(route.Method + " " + route.Path)) - handler := route.Handler[strings.LastIndex(route.Handler, ".")+1:] - if len(handler) > 25 { - handler = handler[:25] + entry := model.Api{Name: autoPermissionName(route), Path: route.Path, Method: method, Description: autoPermissionDescription(route), Version: "v1", Category: 2, IsPublic: 0} + if named, ok := namedPermissions[route.Method+" "+route.Path]; ok { + entry.Name, entry.Description = named.Name, named.Description } - pending = append(pending, model.Api{Name: fmt.Sprintf("%s %s %x", route.Method, handler, digest[:4]), Path: route.Path, Method: method, Description: route.Method + " " + route.Path, Version: "v1", Category: 2, IsPublic: 0}) + pending = append(pending, entry) seen[key(method, route.Path)] = true } if len(pending) == 0 { return nil } - return a.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(pending, 100).Error + if err := a.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(pending, 100).Error; err != nil { + return err + } + a.warnMissingNamedPermissions(ctx, pending) + return nil +} + +// warnMissingNamedPermissions OnConflict DoNothing 遇到唯一键冲突（例如权限码已被其他接口或已软删除的行占用）会静默跳过插入。 +// 插入后逐个核对本次尝试登记的权限码：表中不存在 name + path + method 均一致的行时记一条 Warn（只记权限码），不影响启动。 +func (a *App) warnMissingNamedPermissions(ctx context.Context, pending []model.Api) { + expected := make(map[string]model.Api) + for _, entry := range pending { + for _, named := range namedPermissions { + if entry.Name == named.Name { + expected[entry.Name] = entry + } + } + } + if len(expected) == 0 { + return + } + names := make([]string, 0, len(expected)) + for name := range expected { + names = append(names, name) + } + sort.Strings(names) + var rows []model.Api + if err := a.db.WithContext(ctx).Select("id", "name", "path", "method").Where("name IN ?", names).Find(&rows).Error; err != nil { + if a.logger != nil { + a.logger.Warn("核对敏感接口权限码失败", zap.Strings("permissions", names), zap.Error(err)) + } + return + } + present := make(map[string]bool, len(rows)) + for _, row := range rows { + if want, ok := expected[row.Name]; ok && row.Path == want.Path && row.Method == want.Method { + present[row.Name] = true + } + } + for _, name := range names { + if !present[name] && a.logger != nil { + a.logger.Warn("敏感接口权限码未登记到接口目录（插入因唯一键冲突被跳过），请人工核对 cl_system_apis", + zap.String("permission", name)) + } + } +} + +// renameAutoNamedPermission 已登记过的敏感接口（如 SEC-29 之前就存在的 SA token 路由）在目录中仍是自动生成的 +// 名称与说明时，改为稳定的权限码与说明。只在名称和说明都仍等于自动生成值时更新 +// （WHERE id = ? AND name = <自动名> AND description = " "），管理员改过名称或说明（任一项）的行整行不动； +// 更新失败（例如权限码已被其他记录占用）只记日志，不影响启动。 +func (a *App) renameAutoNamedPermission(ctx context.Context, route gin.RouteInfo, row model.Api) { + named, ok := namedPermissions[route.Method+" "+route.Path] + if !ok || row.ID == 0 || row.Name == named.Name { + return + } + auto := autoPermissionName(route) + autoDescription := autoPermissionDescription(route) + if row.Name != auto { + return + } + if row.Description != autoDescription { + if a.logger != nil { + a.logger.Info("敏感接口权限行的说明已被管理员修改，保留原名称与说明", + zap.String("path", route.Path), zap.String("method", route.Method), zap.String("permission", named.Name)) + } + return + } + err := a.db.WithContext(ctx).Model(&model.Api{}). + Where("id = ? AND name = ? AND description = ?", row.ID, auto, autoDescription). + Updates(map[string]interface{}{"name": named.Name, "description": named.Description}).Error + if err != nil && a.logger != nil { + a.logger.Warn("更新敏感接口权限名失败，保留原名称", + zap.String("path", route.Path), zap.String("method", route.Method), zap.Error(err)) + } } diff --git a/internal/app/permissions_named_test.go b/internal/app/permissions_named_test.go new file mode 100644 index 00000000..99f0c4ca --- /dev/null +++ b/internal/app/permissions_named_test.go @@ -0,0 +1,46 @@ +package app + +import ( + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" +) + +func TestSecretRevealIsRegisteredAsNamedPermission(t *testing.T) { + named, ok := namedPermissions["GET "+model.K8sSecretRevealRoute] + if !ok { + t.Fatal("reveal 接口未登记到接口目录") + } + if named.Name != model.K8sSecretRevealPermissionName || named.Description == "" { + t.Fatalf("权限码不符: %+v", named) + } + // cl_system_apis.name 为 varchar(50) + if len(named.Name) > 50 { + t.Fatal("权限名超过 50 字符") + } +} + +func TestServiceAccountTokenIsRegisteredAsNamedPermission(t *testing.T) { + names := map[string]bool{} + for _, method := range []string{"GET", "POST"} { + named, ok := namedPermissions[method+" "+model.K8sServiceAccountTokenRoute] + if !ok { + t.Fatalf("%s token 接口未登记到接口目录", method) + } + if named.Description == "" || len(named.Name) > 50 { + t.Fatalf("权限码不符: %+v", named) + } + names[named.Name] = true + } + if !names[model.K8sServiceAccountTokenPermissionName] || !names[model.K8sServiceAccountTokenCreatePermissionName] { + t.Fatalf("权限码不符: %v", names) + } + // cl_system_apis.name 有唯一索引，所有登记的权限码不能重复 + seen := map[string]bool{} + for _, named := range namedPermissions { + if seen[named.Name] { + t.Fatalf("权限码重复: %s", named.Name) + } + seen[named.Name] = true + } +} diff --git a/internal/app/permissions_test.go b/internal/app/permissions_test.go index 94822bea..62b0fbc5 100644 --- a/internal/app/permissions_test.go +++ b/internal/app/permissions_test.go @@ -8,6 +8,9 @@ import ( "github.com/GoSimplicity/AI-CloudOps/internal/model" "github.com/gin-gonic/gin" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" "gorm.io/driver/sqlite" "gorm.io/gorm" ) @@ -84,3 +87,207 @@ func TestAIProxyCatalogSupportsSeparateAssistantAndWorkflowGrants(t *testing.T) } } } + +func TestPermissionCatalogNamesSecretReveal(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + pool.SetMaxOpenConns(1) + defer pool.Close() + if err := db.AutoMigrate(&model.Api{}); err != nil { + t.Fatal(err) + } + engine := gin.New() + engine.GET(model.K8sSecretRevealRoute, func(c *gin.Context) { c.Status(200) }) + app := &App{db: db, server: engine} + if err := app.bootstrapPermissionCatalog(context.Background()); err != nil { + t.Fatal(err) + } + var row model.Api + if err := db.Where("path = ? AND method = ?", model.K8sSecretRevealRoute, 1).First(&row).Error; err != nil { + t.Fatal(err) + } + if row.Name != model.K8sSecretRevealPermissionName { + t.Fatalf("权限名=%q", row.Name) + } +} + +func TestPermissionCatalogNamesServiceAccountToken(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + pool.SetMaxOpenConns(1) + defer pool.Close() + if err := db.AutoMigrate(&model.Api{}); err != nil { + t.Fatal(err) + } + engine := gin.New() + handler := func(c *gin.Context) { c.Status(200) } + engine.GET(model.K8sServiceAccountTokenRoute, handler) + engine.POST(model.K8sServiceAccountTokenRoute, handler) + app := &App{db: db, server: engine} + for i := 0; i < 2; i++ { + if err := app.bootstrapPermissionCatalog(context.Background()); err != nil { + t.Fatal(err) + } + } + for method, want := range map[int8]string{1: model.K8sServiceAccountTokenPermissionName, 2: model.K8sServiceAccountTokenCreatePermissionName} { + var rows []model.Api + db.Where("path = ? AND method = ?", model.K8sServiceAccountTokenRoute, method).Find(&rows) + if len(rows) != 1 || rows[0].Name != want { + t.Fatalf("method=%d 权限目录不符: %+v", method, rows) + } + } +} + +// 升级场景：SEC-29 之前 token 路由已按自动名称登记过；自动名称改为权限码，管理员改过的名称保持不变 +func TestPermissionCatalogRenamesAutoNamedServiceAccountToken(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + pool.SetMaxOpenConns(1) + defer pool.Close() + if err := db.AutoMigrate(&model.Api{}); err != nil { + t.Fatal(err) + } + engine := gin.New() + handler := func(c *gin.Context) { c.Status(200) } + engine.GET(model.K8sServiceAccountTokenRoute, handler) + engine.POST(model.K8sServiceAccountTokenRoute, handler) + var getRoute gin.RouteInfo + for _, r := range engine.Routes() { + if r.Method == "GET" { + getRoute = r + } + } + legacy := []model.Api{ + {Name: autoPermissionName(getRoute), Path: model.K8sServiceAccountTokenRoute, Method: 1, Description: "GET " + model.K8sServiceAccountTokenRoute, Version: "v1", Category: 2}, + {Name: "admin custom token name", Path: model.K8sServiceAccountTokenRoute, Method: 2, Description: "custom metadata", Version: "v1", Category: 2}, + } + if err := db.Create(&legacy).Error; err != nil { + t.Fatal(err) + } + app := &App{db: db, server: engine} + if err := app.bootstrapPermissionCatalog(context.Background()); err != nil { + t.Fatal(err) + } + var count int64 + db.Model(&model.Api{}).Count(&count) + var get, post model.Api + db.First(&get, legacy[0].ID) + db.First(&post, legacy[1].ID) + if count != 2 || get.Name != model.K8sServiceAccountTokenPermissionName || post.Name != "admin custom token name" || post.Description != "custom metadata" { + t.Fatalf("升级后权限目录不符: count=%d get=%q post=%q/%q", count, get.Name, post.Name, post.Description) + } + if get.Description != namedPermissions["GET "+model.K8sServiceAccountTokenRoute].Description { + t.Fatalf("自动命名的行应同时更新为登记的说明: %q", get.Description) + } +} + +func newPermissionTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + pool.SetMaxOpenConns(1) + t.Cleanup(func() { pool.Close() }) + if err := db.AutoMigrate(&model.Api{}); err != nil { + t.Fatal(err) + } + return db +} + +func newTokenRouteEngine() (*gin.Engine, map[string]gin.RouteInfo) { + engine := gin.New() + handler := func(c *gin.Context) { c.Status(200) } + engine.GET(model.K8sServiceAccountTokenRoute, handler) + engine.POST(model.K8sServiceAccountTokenRoute, handler) + routes := map[string]gin.RouteInfo{} + for _, r := range engine.Routes() { + routes[r.Method] = r + } + return engine, routes +} + +// SEC-29 P2：管理员只改了说明（名称仍是自动名）时，启动迁移不能覆盖该行的名称和说明 +func TestPermissionCatalogKeepsAdminEditedDescription(t *testing.T) { + db := newPermissionTestDB(t) + engine, routes := newTokenRouteEngine() + legacy := []model.Api{ + // 管理员只改了说明 + {Name: autoPermissionName(routes["GET"]), Path: model.K8sServiceAccountTokenRoute, Method: 1, Description: "运维自定义说明", Version: "v1", Category: 2}, + // 完全是自动生成的行，应被改名 + {Name: autoPermissionName(routes["POST"]), Path: model.K8sServiceAccountTokenRoute, Method: 2, Description: "POST " + model.K8sServiceAccountTokenRoute, Version: "v1", Category: 2}, + } + if err := db.Create(&legacy).Error; err != nil { + t.Fatal(err) + } + app := &App{db: db, server: engine, logger: zap.NewNop()} + for i := 0; i < 2; i++ { + if err := app.bootstrapPermissionCatalog(context.Background()); err != nil { + t.Fatal(err) + } + } + var get, post model.Api + db.First(&get, legacy[0].ID) + db.First(&post, legacy[1].ID) + if get.Name != autoPermissionName(routes["GET"]) || get.Description != "运维自定义说明" { + t.Fatalf("管理员改过说明的行不应被覆盖: %q/%q", get.Name, get.Description) + } + wantPost := namedPermissions["POST "+model.K8sServiceAccountTokenRoute] + if post.Name != wantPost.Name || post.Description != wantPost.Description { + t.Fatalf("自动命名的行应改为权限码与登记说明: %q/%q", post.Name, post.Description) + } + var count int64 + db.Model(&model.Api{}).Count(&count) + if count != 2 { + t.Fatalf("不应新增行: %d", count) + } +} + +// SEC-29 P3-1：权限码被其他行占用导致 OnConflict DoNothing 静默跳过时，记 Warn（只含权限码） +func TestPermissionCatalogWarnsWhenNamedInsertSkipped(t *testing.T) { + db := newPermissionTestDB(t) + engine, _ := newTokenRouteEngine() + occupant := model.Api{Name: model.K8sServiceAccountTokenPermissionName, Path: "/api/other/occupant", Method: 1, Description: "occupant", Version: "v1", Category: 2} + if err := db.Create(&occupant).Error; err != nil { + t.Fatal(err) + } + core, logs := observer.New(zapcore.InfoLevel) + app := &App{db: db, server: engine, logger: zap.New(core)} + if err := app.bootstrapPermissionCatalog(context.Background()); err != nil { + t.Fatal(err) + } + var rows []model.Api + db.Where("path = ?", model.K8sServiceAccountTokenRoute).Find(&rows) + if len(rows) != 1 || rows[0].Name != model.K8sServiceAccountTokenCreatePermissionName { + t.Fatalf("前提：GET 行应因权限码冲突被跳过，只插入 POST 行: %+v", rows) + } + warns := logs.FilterLevelExact(zapcore.WarnLevel).All() + if len(warns) != 1 { + t.Fatalf("应只记一条 Warn: %+v", warns) + } + fields := warns[0].ContextMap() + if len(fields) != 1 || fields["permission"] != model.K8sServiceAccountTokenPermissionName { + t.Fatalf("Warn 只应包含缺失的权限码: %+v", fields) + } + + // 全部登记成功时不记 Warn + db2 := newPermissionTestDB(t) + core2, logs2 := observer.New(zapcore.InfoLevel) + app2 := &App{db: db2, server: engine, logger: zap.New(core2)} + if err := app2.bootstrapPermissionCatalog(context.Background()); err != nil { + t.Fatal(err) + } + if n := logs2.FilterLevelExact(zapcore.WarnLevel).Len(); n != 0 { + t.Fatalf("权限码均已登记时不应告警: %d", n) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 8cc248a1..45ff667a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -35,7 +35,7 @@ import ( "time" "unicode" - "github.com/GoSimplicity/AI-CloudOps/pkg/runenv" + "github.com/GoSimplicity/AI-CloudOps/pkg/envmode" ) type Config struct { @@ -92,7 +92,8 @@ func (c *Config) Validate() error { if err := c.JWT.validateKeys(); err != nil { return err } - isProduction := runenv.IsProduction() + // 失败即关闭：以 pkg/envmode 为准，ENV 未设置/无法识别时同样按生产校验（启动入口 Load 已先行拒绝这两种情况） + isProduction := envmode.IsProduction() if err := c.JWT.validateExpiration(isProduction); err != nil { return err } @@ -132,7 +133,7 @@ const MaxJWTExpirationMinutes = 1440 var configWarnf = log.Printf // validateExpiration 校验访问令牌有效期上限（单位分钟）。 -// 小于等于 0 时 pkg/jwt 会回退到默认值 30 分钟，这里只记告警。环境判断使用 runenv.IsProduction +// 小于等于 0 时 pkg/jwt 会回退到默认值 30 分钟，这里只记告警。环境判断使用 envmode.IsProduction（失败即关闭） func (j JWTConfig) validateExpiration(isProduction bool) error { if j.Expiration <= 0 { configWarnf("[配置告警] JWT_EXPIRATION（单位分钟）为%d，不是正数，将使用默认值 30", j.Expiration) @@ -325,7 +326,8 @@ type CronConfig struct { // MockConfig Mock配置 type MockConfig struct { - Enabled bool `mapstructure:"enabled" env:"MOCK_ENABLED" default:"true"` + // 默认值由 setDefaults 决定：生产环境（ENV=production）为 false，其余环境为 true + Enabled bool `mapstructure:"enabled" env:"MOCK_ENABLED"` } type NotificationConfig struct { diff --git a/internal/config/env_mode_test.go b/internal/config/env_mode_test.go new file mode 100644 index 00000000..ee36e896 --- /dev/null +++ b/internal/config/env_mode_test.go @@ -0,0 +1,96 @@ +package config + +import ( + "errors" + "os" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/pkg/envmode" +) + +func setOrUnsetEnv(t *testing.T, raw string, set bool) { + t.Helper() + t.Setenv("ENV", raw) + if !set { + if err := os.Unsetenv("ENV"); err != nil { + t.Fatal(err) + } + } +} + +// 生产密钥校验必须跟随规范化后的 ENV：别名、大小写、空白都算生产； +// 未设置、空白、未知取值失败即关闭，同样执行生产校验。 +func TestProductionChecksFollowNormalizedEnv(t *testing.T) { + weak := func() *Config { + return &Config{Server: ServerConfig{Port: "8889"}, Log: LogConfig{Dir: "logs"}, MySQL: MySQLConfig{Addr: "db"}, Redis: RedisConfig{Addr: "redis"}, + // JWT 密钥强度与 issuer 在所有环境都校验（w1 validateKeys），这里给合规值；只留生产专属的弱项（TREE 密钥过短、mock 开启） + JWT: JWTConfig{Key1: randomTestKey(), Key2: randomTestKey(), Issuer: "fake-issuer-value"}, Tree: TreeConfig{PasswordEncryptionKey: "short3"}, Mock: MockConfig{Enabled: true}} + } + for _, tt := range []struct { + raw string + set bool + enforce bool + }{ + {"production", true, true}, {"prod", true, true}, {"prd", true, true}, + {"PROD", true, true}, {"Production", true, true}, {" prd\t", true, true}, {"\nPROD \r\n", true, true}, + {"", false, true}, {"", true, true}, {" ", true, true}, + {"test", true, true}, {"testing", true, true}, {"staging", true, true}, + {"development", true, false}, {"dev", true, false}, {" DEV ", true, false}, {"Development", true, false}, + } { + setOrUnsetEnv(t, tt.raw, tt.set) + err := weak().Validate() + if (err != nil) != tt.enforce { + t.Errorf("ENV=%q(set=%v): Validate() = %v，期望执行生产校验=%v", tt.raw, tt.set, err, tt.enforce) + } + } + // 合规的生产配置在别名下也能通过 + setOrUnsetEnv(t, " Prod ", true) + ok := &Config{Server: ServerConfig{Port: "8889"}, Log: LogConfig{Dir: "logs"}, MySQL: MySQLConfig{Addr: "db"}, Redis: RedisConfig{Addr: "redis"}, + JWT: JWTConfig{Key1: randomTestKey(), Key2: randomTestKey(), Issuer: "fake-issuer-value"}, Tree: TreeConfig{PasswordEncryptionKey: strings.Repeat("c", 32)}} + if err := ok.Validate(); err != nil { + t.Errorf("ENV=\" Prod \" 下合规配置不应失败：%v", err) + } +} + +// 启动入口：ENV 无论是否传了 --config 都要校验；默认配置文件按规范化后的 ENV 选择。 +func TestResolveConfigFile(t *testing.T) { + for _, tt := range []struct { + raw, flag, want string + set bool + }{ + {"production", "", "config/config.production.yaml", true}, + {"PRD", "", "config/config.production.yaml", true}, + {" prod ", "", "config/config.production.yaml", true}, + {"development", "", "config/config.development.yaml", true}, + {"Dev", "", "config/config.development.yaml", true}, + {"prod", "custom.yaml", "custom.yaml", true}, + {"dev", "custom.yaml", "custom.yaml", true}, + } { + setOrUnsetEnv(t, tt.raw, tt.set) + got, err := resolveConfigFile(tt.flag) + if err != nil || got != tt.want { + t.Errorf("ENV=%q flag=%q: resolveConfigFile = %q, %v；want %q", tt.raw, tt.flag, got, err, tt.want) + } + } + for _, tt := range []struct { + raw, flag string + set bool + want error + }{ + {"", "", false, envmode.ErrUnset}, + {"", "", true, envmode.ErrUnset}, + {" \t ", "", true, envmode.ErrUnset}, + {"", "custom.yaml", false, envmode.ErrUnset}, + {"test", "", true, envmode.ErrUnknown}, + {"testing", "", true, envmode.ErrUnknown}, + {"staging", "custom.yaml", true, envmode.ErrUnknown}, + {"productio", "", true, envmode.ErrUnknown}, + } { + setOrUnsetEnv(t, tt.raw, tt.set) + got, err := resolveConfigFile(tt.flag) + if !errors.Is(err, tt.want) || got != "" { + t.Errorf("ENV=%q(set=%v) flag=%q: resolveConfigFile = %q, %v；want 拒绝启动(%v)", tt.raw, tt.set, tt.flag, got, err, tt.want) + } + } +} diff --git a/internal/config/env_test.go b/internal/config/env_test.go new file mode 100644 index 00000000..9c3cc9e6 --- /dev/null +++ b/internal/config/env_test.go @@ -0,0 +1,115 @@ +package config + +import ( + "errors" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/pkg/envmode" +) + +func TestIsProductionNormalizesEnv(t *testing.T) { + for _, tt := range []struct { + env string + want bool + }{ + {"production", true}, + {"Production", true}, + {" production", true}, + {"prod", true}, + {"prd", true}, + {"PROD", true}, + {"development", false}, + {"dev", false}, + {"test", true}, + {"testing", true}, + {"", true}, + {"staging", true}, + } { + t.Run(tt.env, func(t *testing.T) { + t.Setenv("ENV", tt.env) + if got := IsProduction(); got != tt.want { + t.Fatalf("ENV=%q IsProduction=%v want %v", tt.env, got, tt.want) + } + }) + } +} + +func TestResolveConfigFileByEnv(t *testing.T) { + for env, want := range map[string]string{ + "production": "config/config.production.yaml", + "Production": "config/config.production.yaml", + " production": "config/config.production.yaml", + "prod": "config/config.production.yaml", + "prd": "config/config.production.yaml", + "PROD": "config/config.production.yaml", + "development": "config/config.development.yaml", + "dev": "config/config.development.yaml", + } { + t.Run(env, func(t *testing.T) { + t.Setenv("ENV", env) + got, err := resolveConfigFile("") + if err != nil || got != want { + t.Fatalf("ENV=%q got=%q err=%v want %q", env, got, err, want) + } + }) + } +} + +func TestResolveConfigFileRejectsUnsetAndUnknownEnv(t *testing.T) { + for _, tt := range []struct { + env string + want error + }{ + {"", envmode.ErrUnset}, + {" ", envmode.ErrUnset}, + {"staging", envmode.ErrUnknown}, + {"test", envmode.ErrUnknown}, + {"testing", envmode.ErrUnknown}, + {"produciton", envmode.ErrUnknown}, + } { + t.Run(tt.env, func(t *testing.T) { + t.Setenv("ENV", tt.env) + for _, explicit := range []string{"", "custom.yaml"} { + got, err := resolveConfigFile(explicit) + if !errors.Is(err, tt.want) || got != "" { + t.Fatalf("ENV=%q explicit=%q got=%q err=%v", tt.env, explicit, got, err) + } + if !strings.Contains(err.Error(), "ENV") { + t.Fatalf("错误提示应指明 ENV: %v", err) + } + } + }) + } + t.Setenv("ENV", "") + _, err := resolveConfigFile("") + if !strings.Contains(err.Error(), envmode.Allowed) { + t.Fatalf("未设置时应提示允许的 ENV 取值: %v", err) + } +} + +func TestResolveConfigFileKeepsExplicitFlag(t *testing.T) { + t.Setenv("ENV", "Prod") + if got, err := resolveConfigFile("custom.yaml"); err != nil || got != "custom.yaml" { + t.Fatalf("got=%q err=%v", got, err) + } +} + +func TestValidateAppliesProductionChecksForAliases(t *testing.T) { + for _, env := range []string{"Production", " production", "prod", "prd", "PROD", ""} { + t.Run(env, func(t *testing.T) { + t.Setenv("ENV", env) + c := prodTestConfig() + c.Mock.Enabled = true + if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "MOCK_ENABLED") { + t.Fatalf("ENV=%q 应按生产校验拒绝 mock: %v", env, err) + } + }) + } + t.Setenv("ENV", "development") + c := validConfigForTest() + c.Mock.Enabled = true + if err := c.Validate(); err != nil { + t.Fatalf("开发环境允许 mock: %v", err) + } +} diff --git a/internal/config/jwt_expiration_limit_test.go b/internal/config/jwt_expiration_limit_test.go index 6f2d478e..d6e9291d 100644 --- a/internal/config/jwt_expiration_limit_test.go +++ b/internal/config/jwt_expiration_limit_test.go @@ -50,8 +50,23 @@ func TestJWTExpirationUpperBound1440Passes(t *testing.T) { } } +// 只有 envmode 识别为开发环境（development/dev，去首尾空白、忽略大小写）时 1441 分钟才告警继续； +// 空值、test、production-x 失败即关闭，按生产拒绝（不告警） func TestJWTExpirationUpperBoundNonProductionWarns(t *testing.T) { - for _, env := range []string{"", "development", "test"} { + for _, env := range []string{"", "test", "production-x"} { + t.Run("fail-closed/"+env, func(t *testing.T) { + t.Setenv("ENV", env) + warns := captureConfigWarn(t) + err := expirationTestConfig(MaxJWTExpirationMinutes + 1).Validate() + if err == nil || !strings.Contains(err.Error(), "JWT_EXPIRATION") { + t.Fatalf("ENV=%q 应按生产拒绝 1441 分钟, got %v", env, err) + } + if len(*warns) != 0 { + t.Fatalf("ENV=%q 应直接返回错误而非告警: %v", env, *warns) + } + }) + } + for _, env := range []string{"development", "dev", " DEV "} { t.Run(env, func(t *testing.T) { t.Setenv("ENV", env) warns := captureConfigWarn(t) diff --git a/internal/config/load_test.go b/internal/config/load_test.go new file mode 100644 index 00000000..7d5259c8 --- /dev/null +++ b/internal/config/load_test.go @@ -0,0 +1,100 @@ +package config + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/pkg/envmode" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +// callLoadIsolated 在隔离的命令行环境里调用 Load： +// os.Args 只留程序名和给定参数（不带 go test 的 -test.* 参数），pflag.CommandLine 每次换新 +// （Load 会重复定义 --config），结束后恢复全局状态并清空 viper。 +func callLoadIsolated(t *testing.T, args ...string) (*Config, *ExternalConfig, error) { + t.Helper() + origArgs, origFlags := os.Args, pflag.CommandLine + t.Cleanup(func() { + os.Args, pflag.CommandLine = origArgs, origFlags + viper.Reset() + }) + os.Args = append([]string{"cloudops-load-test"}, args...) + fs := pflag.NewFlagSet("cloudops-load-test", pflag.ContinueOnError) + fs.SetOutput(io.Discard) + pflag.CommandLine = fs + viper.Reset() + return Load() +} + +// P2-3：启动入口 Load 必须把 ENV 校验错误原样返回（不论是否传了 --config），不能吞掉后继续按默认文件加载。 +func TestLoadRejectsInvalidEnv(t *testing.T) { + cfgFile := filepath.Join(t.TempDir(), "custom.yaml") + if err := os.WriteFile(cfgFile, []byte("server:\n port: \"8889\"\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, tt := range []struct { + name string + raw string + set bool + args []string + want error + }{ + {"unset", "", false, nil, envmode.ErrUnset}, + {"empty", "", true, nil, envmode.ErrUnset}, + {"blank", " \t ", true, nil, envmode.ErrUnset}, + {"test", "test", true, nil, envmode.ErrUnknown}, + {"production-x", "production-x", true, nil, envmode.ErrUnknown}, + {"long-17", strings.Repeat("q", 17), true, nil, envmode.ErrUnknown}, + {"unset+config", "", false, []string{"--config", cfgFile}, envmode.ErrUnset}, + {"staging+config", "staging", true, []string{"--config=" + cfgFile}, envmode.ErrUnknown}, + } { + t.Run(tt.name, func(t *testing.T) { + setOrUnsetEnv(t, tt.raw, tt.set) + cfg, ext, err := callLoadIsolated(t, tt.args...) + if !errors.Is(err, tt.want) { + t.Fatalf("ENV=%q(set=%v) args=%v: Load() err=%v，期望包装 %v", tt.raw, tt.set, tt.args, err, tt.want) + } + if IsNonFatal(err) { + t.Fatalf("ENV 校验失败不能按“可继续运行”的告警返回：%v", err) + } + if cfg != nil || ext != nil { + t.Fatalf("ENV 校验失败时不应返回配置：cfg=%v ext=%v", cfg != nil, ext != nil) + } + if !strings.Contains(err.Error(), "启动环境校验失败") { + t.Fatalf("错误信息应说明是启动环境校验失败：%v", err) + } + }) + } + // 对照：合法 ENV 能越过 ENV 校验（之后的结果与本用例无关，只要不是 ENV 错误） + setOrUnsetEnv(t, " Dev ", true) + if _, _, err := callLoadIsolated(t, "--config", cfgFile); errors.Is(err, envmode.ErrUnset) || errors.Is(err, envmode.ErrUnknown) { + t.Fatalf("ENV=\" Dev \" 不应报 ENV 错误：%v", err) + } +} + +// Published images contain only the executable, so all required configuration +// must be loadable from the environment when the YAML file is absent. +func TestLoadWithoutConfigFileAcceptsReleaseEnvironment(t *testing.T) { + t.Setenv("ENV", "production") + t.Setenv("JWT_KEY1", randomTestKey()) + t.Setenv("JWT_KEY2", randomTestKey()) + t.Setenv("TREE_PASSWORD_ENCRYPTION_KEY", randomTestKey()[:32]) + t.Setenv("MYSQL_ADDR", "fixture:fixture@tcp(mysql:3306)/cloudops?parseTime=True") + t.Setenv("REDIS_ADDR", "redis:6379") + t.Setenv("JWT_ISSUER", "release-test-issuer") + t.Setenv("MOCK_ENABLED", "false") + t.Setenv("AIOPS_BASE_URL", "http://aiops:8080") + t.Setenv("AIOPS_API_KEY", randomTestKey()) + cfg, _, err := callLoadIsolated(t, "--config", filepath.Join(t.TempDir(), "missing.yaml")) + if !IsNonFatal(err) { + t.Fatalf("missing YAML should only warn after environment configuration validates: %v", err) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("release environment should supply every required field: %v", err) + } +} diff --git a/internal/config/loader.go b/internal/config/loader.go index d6f9f53f..7891f0d9 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -40,7 +40,7 @@ import ( "strings" "sync" - "github.com/GoSimplicity/AI-CloudOps/pkg/runenv" + "github.com/GoSimplicity/AI-CloudOps/pkg/envmode" "github.com/spf13/pflag" "github.com/spf13/viper" "gopkg.in/yaml.v3" @@ -69,16 +69,12 @@ func Load() (*Config, *ExternalConfig, error) { configFile := pflag.String("config", "", "配置文件路径") pflag.Parse() - if *configFile == "" { - // 与 Config.Validate 使用同一判断（去首尾空白、忽略大小写），避免选了开发配置却按生产规则校验 - if runenv.IsProduction() { - *configFile = "config/config.production.yaml" - } else { - *configFile = "config/config.development.yaml" - } + file, err := resolveConfigFile(*configFile) + if err != nil { + return nil, nil, err } - return loadFromFile(*configFile) + return loadFromFile(file) } // loadFromFile 按“环境变量 > 配置文件 > 默认值”加载指定配置文件（不解析命令行参数，便于测试） @@ -114,6 +110,22 @@ func loadFromFile(configFile string) (*Config, *ExternalConfig, error) { return cfg, ext, nil } +// resolveConfigFile 校验 ENV 并给出要读的配置文件：不论是否传了 --config， +// ENV 未设置、为空（含只有空白）或无法识别都拒绝启动；未传 --config 时按规范化后的 ENV 选默认文件。 +func resolveConfigFile(flagValue string) (string, error) { + mode, err := envmode.Current() + if err != nil { + return "", fmt.Errorf("启动环境校验失败: %w", err) + } + if flagValue != "" { + return flagValue, nil + } + if mode == envmode.Production { + return "config/config.production.yaml", nil + } + return "config/config.development.yaml", nil +} + // WebhookSecrets Webhook 进程的敏感配置，仅保存在返回值中，不写入全局 viper type WebhookSecrets struct { // MySQLAddr 来自 WEBHOOK_MYSQL_ADDR 的 MySQL DSN（含账号密码） @@ -1192,7 +1204,9 @@ func setDefaults() { viper.SetDefault("prometheus.alert_webhook_file_dir", "/tmp/webhook_files") viper.SetDefault("prometheus.httpSdAPI", "http://localhost:8889/api/not_auth/getTreeNodeBindIps") - viper.SetDefault("mock.enabled", true) + // mock 默认值随环境变化：生产环境（ENV 为 production/prod/prd，或未设置/无法识别）默认关闭，开发/测试环境保持开启。 + // 显式配置（配置文件 mock.enabled 或环境变量 MOCK_ENABLED）仍然优先；生产环境开启会被 Validate 拒绝。 + viper.SetDefault("mock.enabled", defaultMockEnabled()) viper.SetDefault(CronAllowLocalExecKey, false) @@ -1234,6 +1248,19 @@ func setWebhookDefaults(v *viper.Viper) { v.SetDefault("webhook.im_feishu.tenant_access_token_api", "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal") } +// IsProduction 判断当前进程是否应按生产环境处理，委托给 pkg/envmode， +// 与 Load 选择配置文件、Validate 的生产校验、pkg/ssh 的主机密钥策略使用同一判定。 +// ENV 会先 TrimSpace、ToLower 再匹配别名（production/prod/prd）；未设置或未知取值按生产处理（fail closed）， +// 而 Load 会对这两种情况直接拒绝启动。 +func IsProduction() bool { + return envmode.IsProduction() +} + +// defaultMockEnabled 仅在明确开发环境时开启 mock，其他环境关闭。 +func defaultMockEnabled() bool { + return !IsProduction() +} + func bindEnvVars() { bindStructEnvVars(viper.GetViper(), reflect.TypeOf(Config{}), "") } diff --git a/internal/config/mock_default_test.go b/internal/config/mock_default_test.go new file mode 100644 index 00000000..d6a468d0 --- /dev/null +++ b/internal/config/mock_default_test.go @@ -0,0 +1,63 @@ +package config + +import ( + "testing" + + "github.com/spf13/viper" +) + +func loadMockEnabledFromDefaults(t *testing.T) bool { + t.Helper() + viper.Reset() + t.Cleanup(viper.Reset) + setDefaults() + viper.AutomaticEnv() + bindEnvVars() + cfg := &Config{} + if err := viper.Unmarshal(cfg); err != nil { + t.Fatal(err) + } + return cfg.Mock.Enabled +} + +func TestMockDefaultFollowsEnvironment(t *testing.T) { + for _, tt := range []struct { + name, env, mockEnv string + want bool + }{ + {"production defaults off", "production", "", false}, + {"development defaults on", "development", "", true}, + {"unset ENV fails closed", "", "", false}, + {"unknown ENV fails closed", "staging", "", false}, + {"test is not a supported mode", "test", "", false}, + {"testing is not a supported mode", "testing", "", false}, + {"production alias defaults off", " PROD ", "", false}, + {"dev alias defaults on", "Dev", "", true}, + {"explicit off in development", "development", "false", false}, + {"explicit on in production still loads (rejected by Validate)", "production", "true", true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("ENV", tt.env) + t.Setenv("MOCK_ENABLED", tt.mockEnv) + if got := loadMockEnabledFromDefaults(t); got != tt.want { + t.Fatalf("mock.enabled=%v want=%v", got, tt.want) + } + }) + } +} + +func TestShippedConfigFilesSetMockExplicitly(t *testing.T) { + for file, want := range map[string]bool{ + "../../config/config.production.yaml": false, + "../../config/config.development.yaml": true, + } { + v := viper.New() + v.SetConfigFile(file) + if err := v.ReadInConfig(); err != nil { + t.Fatalf("%s: %v", file, err) + } + if !v.IsSet("mock.enabled") || v.GetBool("mock.enabled") != want { + t.Fatalf("%s: mock.enabled 应显式为 %v", file, want) + } + } +} diff --git a/internal/config/production_env_checks_test.go b/internal/config/production_env_checks_test.go index 79696b9e..e6ffe696 100644 --- a/internal/config/production_env_checks_test.go +++ b/internal/config/production_env_checks_test.go @@ -11,24 +11,30 @@ func prodTestConfig() *Config { return c } -// ENV 去首尾空白、忽略大小写后等于 production 即按生产环境校验 +// 生产判定以 pkg/envmode 为准（失败即关闭）：production/prod/prd 去首尾空白、忽略大小写后按生产校验； +// 空值、test、production-x 等无法识别的取值同样按生产校验（被拒）；只有 development/dev 不执行生产专属检查 func TestProductionEnvDetectionTrimAndFold(t *testing.T) { - for _, env := range []string{"production", " Production ", "PRODUCTION", "\tproduction\n"} { - t.Run(env, func(t *testing.T) { - t.Setenv("ENV", env) - c := prodTestConfig() - c.Mock.Enabled = true // 仅生产环境拒绝 - if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "MOCK_ENABLED") { - t.Fatalf("ENV=%q 应按生产环境校验, got %v", env, err) - } - c = prodTestConfig() - c.JWT.Expiration = MaxJWTExpirationMinutes + 1 - if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "JWT_EXPIRATION") { - t.Fatalf("ENV=%q 下 1441 分钟应拒绝启动, got %v", env, err) - } - }) + assertProdEnforced := func(t *testing.T, env string) { + t.Helper() + t.Setenv("ENV", env) + c := prodTestConfig() + c.Mock.Enabled = true // 仅生产环境拒绝 + if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "MOCK_ENABLED") { + t.Fatalf("ENV=%q 应按生产环境校验, got %v", env, err) + } + c = prodTestConfig() + c.JWT.Expiration = MaxJWTExpirationMinutes + 1 + if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "JWT_EXPIRATION") { + t.Fatalf("ENV=%q 下 1441 分钟应拒绝启动, got %v", env, err) + } + } + for _, env := range []string{"production", " Production ", "PRODUCTION", "\tproduction\n", "prod", " PRD "} { + t.Run("prod/"+env, func(t *testing.T) { assertProdEnforced(t, env) }) + } + for _, env := range []string{"", "test", "production-x"} { + t.Run("fail-closed/"+env, func(t *testing.T) { assertProdEnforced(t, env) }) } - for _, env := range []string{"", "development", "prod", "production-x"} { + for _, env := range []string{"development", "dev", " Dev "} { t.Run("non-prod/"+env, func(t *testing.T) { t.Setenv("ENV", env) c := validConfigForTest() diff --git a/internal/config/startup_e2e_test.go b/internal/config/startup_e2e_test.go new file mode 100644 index 00000000..f7482ff1 --- /dev/null +++ b/internal/config/startup_e2e_test.go @@ -0,0 +1,129 @@ +//go:build smoke + +package config + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// moduleRoot 从当前包目录向上找 go.mod +func moduleRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("找不到 go.mod") + } + dir = parent + } +} + +func synthStrongSecret(t *testing.T, n int) string { + t.Helper() + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(b) +} + +func synthHex(t *testing.T, n int) string { + t.Helper() + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + t.Fatal(err) + } + return hex.EncodeToString(b) +} + +// P3-8 端到端：真实主程序在 ENV=" Prod " 且密钥为弱合成值时，必须 rc=1、stderr 非空、 +// 含可读原因（“配置校验失败”+变量名）、不含任何合成密钥文本，且多次运行原因一致。 +func TestStartupFailurePrintsReasonWithoutSecrets(t *testing.T) { + goBin, err := exec.LookPath("go") + if err != nil { + t.Skip("PATH 中没有 go 工具链") + } + bin := filepath.Join(t.TempDir(), "cloudops-e2e") + build := exec.Command(goBin, "build", "-o", bin, ".") + build.Dir = moduleRoot(t) + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("构建主程序失败：%v\n%s", err, out) + } + mysqlPw := "fakepw" + synthHex(t, 4) + cases := []struct { + name string + secrets map[string]string + reason string + }{ + {"all-weak", map[string]string{ + "JWT_KEY1": "wk1" + synthHex(t, 4), "JWT_KEY2": "wk2" + synthHex(t, 4), "TREE_PASSWORD_ENCRYPTION_KEY": "wk3" + synthHex(t, 4), + }, "JWT_KEY1"}, + {"tree-weak", map[string]string{ + "JWT_KEY1": synthStrongSecret(t, 48), "JWT_KEY2": synthStrongSecret(t, 48), "TREE_PASSWORD_ENCRYPTION_KEY": "wk3" + synthHex(t, 4), + }, "TREE_PASSWORD_ENCRYPTION_KEY"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + workDir := t.TempDir() // 无 .env、无 config/ 目录 + env := []string{ + "PATH=" + os.Getenv("PATH"), "HOME=" + workDir, "TMPDIR=" + os.TempDir(), + "ENV= Prod ", "JWT_ISSUER=fake-issuer-value", "MOCK_ENABLED=false", + "MYSQL_ADDR=fakeuser:" + mysqlPw + "@tcp(127.0.0.1:9)/fakedb?charset=utf8mb4&parseTime=True&loc=Local", + } + for k, v := range tc.secrets { + env = append(env, k+"="+v) + } + var first string + for i := 0; i < 30; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + cmd := exec.CommandContext(ctx, bin) + cmd.Dir, cmd.Env = workDir, env + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + cancel() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("第 %d 次：期望 rc=1，got %v", i+1, err) + } + msg := stderr.String() + if strings.TrimSpace(msg) == "" { + t.Fatalf("第 %d 次：rc=1 但 stderr 为空", i+1) + } + for _, want := range []string{"启动失败", "配置校验失败", tc.reason} { + if !strings.Contains(msg, want) { + t.Fatalf("第 %d 次：stderr 缺少 %q：\n%s", i+1, want, msg) + } + } + for _, secret := range append([]string{mysqlPw}, tc.secrets["JWT_KEY1"], tc.secrets["JWT_KEY2"], tc.secrets["TREE_PASSWORD_ENCRYPTION_KEY"]) { + if strings.Contains(msg, secret) || strings.Contains(stdout.String(), secret) { + t.Fatalf("第 %d 次：输出包含了合成密钥文本", i+1) + } + } + if i == 0 { + first = msg + } else if msg != first { + t.Fatalf("第 %d 次 stderr 与第 1 次不同：\n第1次：%s\n第%d次：%s", i+1, first, i+1, msg) + } + } + }) + } +} diff --git a/internal/config/startup_order_test.go b/internal/config/startup_order_test.go new file mode 100644 index 00000000..0e7fc36f --- /dev/null +++ b/internal/config/startup_order_test.go @@ -0,0 +1,42 @@ +package config + +import ( + "crypto/rand" + "encoding/hex" + "strings" + "testing" +) + +// synthWeakSecret 测试专用：运行时生成的短（弱）合成密钥，只用来断言错误信息里不出现它 +func synthWeakSecret(t *testing.T, prefix string) string { + t.Helper() + b := make([]byte, 4) + if _, err := rand.Read(b); err != nil { + t.Fatal(err) + } + return prefix + hex.EncodeToString(b) +} + +// P3-8：生产环境多个密钥同时不合格时，报错必须稳定指向同一个变量（按 JWT_KEY1、JWT_KEY2、 +// TREE_PASSWORD_ENCRYPTION_KEY 的固定顺序检查），同样的输入每次得到同样的启动失败原因，且不含密钥文本。 +func TestProductionWeakKeyErrorIsDeterministic(t *testing.T) { + t.Setenv("ENV", " Prod ") + k1, k2, tree := synthWeakSecret(t, "wk1"), synthWeakSecret(t, "wk2"), synthWeakSecret(t, "wk3") + cfg := &Config{Server: ServerConfig{Port: "8889"}, Log: LogConfig{Dir: "logs"}, MySQL: MySQLConfig{Addr: "db"}, Redis: RedisConfig{Addr: "redis"}, + JWT: JWTConfig{Key1: k1, Key2: k2, Issuer: "fake-issuer-value"}, Tree: TreeConfig{PasswordEncryptionKey: tree}, Mock: MockConfig{Enabled: true}} + first := cfg.Validate() + if first == nil || !strings.Contains(first.Error(), "JWT_KEY1") { + t.Fatalf("三个密钥都弱时应先报 JWT_KEY1，got %v", first) + } + for i := 0; i < 200; i++ { + err := cfg.Validate() + if err == nil || err.Error() != first.Error() { + t.Fatalf("第 %d 次校验结果与第一次不同：%v vs %v", i+1, err, first) + } + } + for _, secret := range []string{k1, k2, tree} { + if strings.Contains(first.Error(), secret) { + t.Fatalf("错误信息包含了密钥文本") + } + } +} diff --git a/internal/constants/k8s.go b/internal/constants/k8s.go index fda8a7d2..07a86d69 100644 --- a/internal/constants/k8s.go +++ b/internal/constants/k8s.go @@ -51,4 +51,7 @@ var ( ErrK8sGetPodLogs = errors.New("k8s get pod logs failed") ErrK8sPortForward = errors.New("k8s port forward failed") ErrorK8sIngressHealthFailed = errors.New("k8s ingress check health failed") + + // ErrK8sResourceConflict 乐观锁冲突（提交的 resourceVersion 已过期），handler 映射为 HTTP 409 + ErrK8sResourceConflict = errors.New("k8s resource conflict") ) diff --git a/internal/k8s/api/cluster_kubeconfig_test.go b/internal/k8s/api/cluster_kubeconfig_test.go new file mode 100644 index 00000000..6540b890 --- /dev/null +++ b/internal/k8s/api/cluster_kubeconfig_test.go @@ -0,0 +1,257 @@ +package api + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "k8s.io/client-go/kubernetes" + + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/connection" + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/dao" + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/manager" + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/service" + "github.com/GoSimplicity/AI-CloudOps/internal/model" +) + +// 测试用的假私钥/证书内容，仅用于断言响应中不会出现，不是任何真实凭据 +const ( + fakeKeyMarker = "FIXTURE-PRIVATE-KEY-MATERIAL-DO-NOT-LEAK" + fakeCertMarker = "FIXTURE-CLIENT-CERT-MATERIAL-DO-NOT-LEAK" + fakeToken = "fixture-token-do-not-leak" +) + +func fixturePEM(kind, marker string) []byte { + return pem.EncodeToMemory(&pem.Block{Type: kind, Bytes: []byte(marker)}) +} + +func kubeconfigWithCredentials(server string, caPEM []byte) string { + b64 := base64.StdEncoding.EncodeToString + return fmt.Sprintf(`apiVersion: v1 +kind: Config +current-context: fixture +clusters: +- name: fixture + cluster: + server: %s + certificate-authority-data: %s +users: +- name: fixture + user: + token: %s +- name: cert-user + user: + client-certificate-data: %s + client-key-data: %s +contexts: +- name: fixture + context: {cluster: fixture, user: fixture} +`, server, b64(caPEM), fakeToken, b64(fixturePEM("CERTIFICATE", fakeCertMarker)), b64(fixturePEM("RSA PRIVATE KEY", fakeKeyMarker))) +} + +type connectingManager struct { + manager.ClusterManager + updated []*model.K8sCluster + connErr error +} + +// UpdateCluster 模拟真实的客户端初始化：解析 kubeconfig 并请求 API Server +func (m *connectingManager) UpdateCluster(ctx context.Context, cluster *model.K8sCluster) error { + m.updated = append(m.updated, cluster) + cfg, err := connection.Parse(cluster.KubeConfigContent) + if err != nil { + m.connErr = err + return err + } + cs, err := kubernetes.NewForConfig(cfg) + if err == nil { + _, err = cs.Discovery().ServerVersion() + } + m.connErr = err + return err +} + +func setupClusterAPI(t *testing.T) (*gin.Engine, *gorm.DB, *connectingManager, *httptest.Server) { + t.Helper() + gin.SetMode(gin.TestMode) + + apiServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer "+fakeToken { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"major":"1","minor":"30","gitVersion":"v1.30.0"}`)) + })) + t.Cleanup(apiServer.Close) + + db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s-%d?mode=memory&cache=shared", t.Name(), time.Now().UnixNano())), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err := db.AutoMigrate(&model.K8sCluster{}); err != nil { + t.Fatal(err) + } + + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: apiServer.Certificate().Raw}) + cluster := &model.K8sCluster{ + Name: "kubeconfig-fixture", + ApiServerAddr: apiServer.URL, + KubeConfigContent: kubeconfigWithCredentials(apiServer.URL, caPEM), + Status: model.StatusRunning, + Env: model.EnvDev, + } + if err := db.Create(cluster).Error; err != nil { + t.Fatal(err) + } + empty := &model.K8sCluster{Name: "no-kubeconfig", ApiServerAddr: "https://127.0.0.1:6443", Status: model.StatusStopped} + if err := db.Create(empty).Error; err != nil { + t.Fatal(err) + } + + mgr := &connectingManager{} + svc := service.NewClusterService(dao.NewClusterDAO(db, zap.NewNop()), nil, mgr, zap.NewNop()) + r := gin.New() + NewK8sClusterHandler(svc).RegisterRouters(r) + return r, db, mgr, apiServer +} + +func assertNoKubeconfigLeak(t *testing.T, body string) { + t.Helper() + b64 := base64.StdEncoding.EncodeToString + forbidden := []string{ + "kube_config_content", "client-key-data", "client-certificate-data", "certificate-authority-data", + "PRIVATE KEY", "BEGIN CERTIFICATE", fakeKeyMarker, fakeCertMarker, fakeToken, + b64(fixturePEM("RSA PRIVATE KEY", fakeKeyMarker)), b64(fixturePEM("CERTIFICATE", fakeCertMarker)), + } + for _, f := range forbidden { + if strings.Contains(body, f) { + t.Fatalf("响应泄露了 kubeconfig 敏感内容 %q: %s", f, body) + } + } +} + +func doRequest(r *gin.Engine, method, path string, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequest(method, path, bytes.NewReader(body)) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + return w +} + +func TestClusterDetailAndListDoNotExposeKubeconfig(t *testing.T) { + r, _, _, _ := setupClusterAPI(t) + + detail := doRequest(r, http.MethodGet, "/api/k8s/cluster/1/detail", nil) + if detail.Code != http.StatusOK { + t.Fatalf("detail status %d: %s", detail.Code, detail.Body.String()) + } + assertNoKubeconfigLeak(t, detail.Body.String()) + var detailResp struct { + Data map[string]any `json:"data"` + } + if err := json.Unmarshal(detail.Body.Bytes(), &detailResp); err != nil { + t.Fatal(err) + } + if detailResp.Data["has_kube_config"] != true || detailResp.Data["name"] != "kubeconfig-fixture" { + t.Fatalf("详情应返回 has_kube_config=true 及其他字段: %s", detail.Body.String()) + } + + list := doRequest(r, http.MethodGet, "/api/k8s/cluster/list?page=1&size=10", nil) + if list.Code != http.StatusOK { + t.Fatalf("list status %d: %s", list.Code, list.Body.String()) + } + assertNoKubeconfigLeak(t, list.Body.String()) + var listResp struct { + Data struct { + Items []map[string]any `json:"items"` + Total int `json:"total"` + } `json:"data"` + } + if err := json.Unmarshal(list.Body.Bytes(), &listResp); err != nil { + t.Fatal(err) + } + if listResp.Data.Total != 2 || len(listResp.Data.Items) != 2 { + t.Fatalf("列表数量不符: %s", list.Body.String()) + } + got := map[string]any{} + for _, item := range listResp.Data.Items { + got[item["name"].(string)] = item["has_kube_config"] + } + if got["kubeconfig-fixture"] != true || got["no-kubeconfig"] != false { + t.Fatalf("列表 has_kube_config 不正确: %v", got) + } +} + +func TestUpdateClusterWithoutKubeconfigKeepsStoredValueAndConnects(t *testing.T) { + r, db, mgr, _ := setupClusterAPI(t) + var before model.K8sCluster + if err := db.First(&before, 1).Error; err != nil { + t.Fatal(err) + } + + for _, body := range []string{ + `{"name":"renamed-fixture","version":"v1.30.0"}`, // 不传 + `{"name":"renamed-again","version":"v1.30.0","kube_config_content":""}`, // 空字符串 + } { + w := doRequest(r, http.MethodPut, "/api/k8s/cluster/1/update", []byte(body)) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"code":0`) { + t.Fatalf("更新失败 %d: %s", w.Code, w.Body.String()) + } + var after model.K8sCluster + if err := db.First(&after, 1).Error; err != nil { + t.Fatal(err) + } + if after.KubeConfigContent != before.KubeConfigContent { + t.Fatal("未传 kubeconfig 时数据库中的原值被修改") + } + if mgr.connErr != nil { + t.Fatalf("保留原 kubeconfig 后应仍能连接集群: %v", mgr.connErr) + } + last := mgr.updated[len(mgr.updated)-1] + if last.KubeConfigContent != before.KubeConfigContent { + t.Fatal("客户端初始化未使用保留的 kubeconfig") + } + } + + var after model.K8sCluster + if err := db.First(&after, 1).Error; err != nil { + t.Fatal(err) + } + if after.Name != "renamed-again" { + t.Fatalf("其他字段应正常更新, got %q", after.Name) + } +} + +func TestUpdateClusterRejectsInvalidReplacementKubeconfig(t *testing.T) { + r, db, mgr, _ := setupClusterAPI(t) + var before model.K8sCluster + if err := db.First(&before, 1).Error; err != nil { + t.Fatal(err) + } + w := doRequest(r, http.MethodPut, "/api/k8s/cluster/1/update", []byte(`{"name":"x","kube_config_content":"not: [valid"}`)) + if strings.Contains(w.Body.String(), `"code":0`) { + t.Fatalf("非法 kubeconfig 应被拒绝: %s", w.Body.String()) + } + var after model.K8sCluster + if err := db.First(&after, 1).Error; err != nil { + t.Fatal(err) + } + if after.KubeConfigContent != before.KubeConfigContent || len(mgr.updated) != 0 { + t.Fatal("校验失败时不应修改已存 kubeconfig") + } +} diff --git a/internal/k8s/api/secret.go b/internal/k8s/api/secret.go index 4871f9e9..3f785066 100644 --- a/internal/k8s/api/secret.go +++ b/internal/k8s/api/secret.go @@ -26,6 +26,9 @@ package api import ( + "errors" + + "github.com/GoSimplicity/AI-CloudOps/internal/constants" "github.com/GoSimplicity/AI-CloudOps/internal/k8s/service" "github.com/GoSimplicity/AI-CloudOps/internal/model" "github.com/GoSimplicity/AI-CloudOps/pkg/base" @@ -50,6 +53,7 @@ func (h *K8sSecretHandler) RegisterRouters(server *gin.Engine) { k8sGroup.GET("/secret/:cluster_id/list", h.GetSecretList) k8sGroup.GET("/secret/:cluster_id/:namespace/:name/detail", h.GetSecret) k8sGroup.GET("/secret/:cluster_id/:namespace/:name/detail/yaml", h.GetSecretYAML) + k8sGroup.GET("/secret/:cluster_id/:namespace/:name/reveal", h.RevealSecret) k8sGroup.POST("/secret/:cluster_id/create", h.CreateSecret) k8sGroup.POST("/secret/:cluster_id/create/yaml", h.CreateSecretByYaml) k8sGroup.PUT("/secret/:cluster_id/:namespace/:name/update", h.UpdateSecret) @@ -255,3 +259,109 @@ func (h *K8sSecretHandler) UpdateSecretByYaml(ctx *gin.Context) { return nil, h.secretService.UpdateSecretByYaml(ctx.Request.Context(), &req) }) } + +// RevealSecret 返回 Secret 明文（SEC-25）。list/detail/yaml 默认脱敏，明文只能经此接口获取， +// 每次调用都会写一条审计（只含操作者、集群、命名空间、名称与键名）。 +func (h *K8sSecretHandler) RevealSecret(ctx *gin.Context) { + var req model.RevealSecretReq + + clusterID, err := base.GetCustomParamID(ctx, "cluster_id") + if err != nil { + base.BadRequestError(ctx, err.Error()) + return + } + + namespace, err := base.GetParamCustomName(ctx, "namespace") + if err != nil { + base.BadRequestError(ctx, err.Error()) + return + } + + name, err := base.GetParamCustomName(ctx, "name") + if err != nil { + base.BadRequestError(ctx, err.Error()) + return + } + + req.ClusterID = clusterID + req.Namespace = namespace + req.Name = name + + user := base.GetUserInfoFromGinContext(ctx) + actor := model.SecretRevealActor{ + UserID: user.UserID, + Username: user.Username, + IP: user.IP, + UserAgent: user.UserAgent, + TraceID: ctx.GetHeader("X-Trace-ID"), + Endpoint: ctx.Request.URL.Path, + } + + // 明文响应禁止被浏览器或中间代理缓存 + ctx.Header("Cache-Control", "no-store") + ctx.Header("Pragma", "no-cache") + + handleAuditedRequest(ctx, &req, func() (interface{}, error) { + return h.secretService.RevealSecret(ctx.Request.Context(), &req, actor) + }) +} + +// handleAuditedRequest 与 base.HandleRequest 相同，但同步审计写入失败（service.ErrAuditWriteFailed）时 +// 返回 HTTP 500 而不是 200+code=1，使网关/监控与审计告警能看到真实的失败状态； +// service 返回的参数错误（BusinessError{constants.ErrInvalidParam}）返回 HTTP 400。响应中不含任何明文。 +func handleAuditedRequest(ctx *gin.Context, req interface{}, action func() (interface{}, error)) { + if req != nil { + if err := base.BindRequest(ctx, req); err != nil { + base.BadRequestWithDetails(ctx, err.Error(), "绑定数据失败") + return + } + } + data, err := action() + if err != nil { + if errors.Is(err, service.ErrAuditWriteFailed) { + base.InternalServerErrorWithDetails(ctx, map[string]interface{}{}, err.Error()) + return + } + respondBusinessError(ctx, err) + return + } + base.SuccessWithData(ctx, data) +} + +// handleMappedRequest 与 base.HandleRequest 相同，但按 respondBusinessError 把带类型的业务错误映射为对应 HTTP 状态码 +func handleMappedRequest(ctx *gin.Context, req interface{}, action func() (interface{}, error)) { + if req != nil { + if err := base.BindRequest(ctx, req); err != nil { + base.BadRequestWithDetails(ctx, err.Error(), "绑定数据失败") + return + } + } + data, err := action() + if err != nil { + respondBusinessError(ctx, err) + return + } + if data != nil { + base.SuccessWithData(ctx, data) + } else { + base.Success(ctx) + } +} + +// respondBusinessError 业务错误映射：BusinessError{constants.ErrInvalidParam} → HTTP 400； +// BusinessError{constants.ErrK8sResourceConflict}（乐观锁冲突）→ HTTP 409；其余保持项目约定的 HTTP 200 + code=1 +func respondBusinessError(ctx *gin.Context, err error) { + var bizErr *base.BusinessError + if errors.As(err, &bizErr) { + switch { + case errors.Is(bizErr.Code, constants.ErrInvalidParam): + // 参数错误（如 TokenRequest 有效期越界）按项目约定返回 HTTP 400 + code=1 + base.BadRequestError(ctx, err.Error()) + return + case errors.Is(bizErr.Code, constants.ErrK8sResourceConflict): + base.ConflictError(ctx, err.Error()) + return + } + } + base.ErrorWithMessage(ctx, err.Error()) +} diff --git a/internal/k8s/api/secret_test.go b/internal/k8s/api/secret_test.go new file mode 100644 index 00000000..e59e0294 --- /dev/null +++ b/internal/k8s/api/secret_test.go @@ -0,0 +1,161 @@ +package api + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/manager" + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/service" + "github.com/GoSimplicity/AI-CloudOps/internal/middleware" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + systemService "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const handlerFixtureValue = "plain-handler-fixture-777" + +type handlerSecretManager struct { + manager.SecretManager +} + +func (handlerSecretManager) GetSecret(_ context.Context, _ int, namespace, name string) (*corev1.Secret, error) { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"alpha": []byte(handlerFixtureValue)}, + }, nil +} + +// recordingAudit 同时记录 service 的同步审计与中间件的异步审计 +type recordingAudit struct { + systemService.AuditService + mu sync.Mutex + sync []*model.CreateAuditLogRequest + async []*model.CreateAuditLogRequest +} + +func (a *recordingAudit) CreateAuditLog(_ context.Context, req *model.CreateAuditLogRequest) error { + a.mu.Lock() + defer a.mu.Unlock() + a.sync = append(a.sync, req) + return nil +} + +func (a *recordingAudit) CreateAuditLogAsync(_ context.Context, req *model.CreateAuditLogRequest) { + a.mu.Lock() + defer a.mu.Unlock() + a.async = append(a.async, req) +} + +func TestRevealRouteMatchesPermissionCode(t *testing.T) { + gin.SetMode(gin.TestMode) + engine := gin.New() + NewK8sSecretHandler(nil).RegisterRouters(engine) + found := false + for _, r := range engine.Routes() { + if r.Method == "GET" && r.Path == model.K8sSecretRevealRoute { + found = true + } + } + if !found { + t.Fatalf("未注册 reveal 路由 %s", model.K8sSecretRevealRoute) + } +} + +func TestRevealEndpointReturnsPlaintextAndAuditsWithoutValues(t *testing.T) { + gin.SetMode(gin.TestMode) + audit := &recordingAudit{} + svc := service.NewSecretService(handlerSecretManager{}, audit, zap.NewNop()) + + engine := gin.New() + engine.Use(func(c *gin.Context) { + c.Set("user", jwt.UserClaims{Uid: 9, Username: "bob"}) + c.Next() + }) + // 与线上一致挂载审计中间件，确认它不会把响应体（明文）写入审计 + engine.Use(middleware.NewAuditLogMiddleware(audit, zap.NewNop()).AuditLog()) + NewK8sSecretHandler(svc).RegisterRouters(engine) + + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", "/api/k8s/secret/3/ops/app/reveal", nil)) + if rec.Code != 200 { + t.Fatalf("status=%d", rec.Code) + } + if rec.Header().Get("Cache-Control") != "no-store" { + t.Fatal("reveal 响应应禁止缓存") + } + var resp struct { + Code int `json:"code"` + Data model.K8sSecretReveal `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if string(resp.Data.Data["alpha"]) != handlerFixtureValue || resp.Data.Namespace != "ops" || resp.Data.Name != "app" || resp.Data.ClusterID != 3 { + t.Fatalf("reveal 响应不符: %s", rec.Body.String()) + } + + audit.mu.Lock() + defer audit.mu.Unlock() + if len(audit.sync) != 1 { + t.Fatalf("reveal 应写 1 条审计，实际 %d", len(audit.sync)) + } + all := append(append([]*model.CreateAuditLogRequest{}, audit.sync...), audit.async...) + raw, _ := json.Marshal(all) + if strings.Contains(string(raw), handlerFixtureValue) || strings.Contains(string(raw), base64.StdEncoding.EncodeToString([]byte(handlerFixtureValue))) { + t.Fatal("审计记录中出现了明文") + } + for _, l := range all { + if len(l.ResponseBody) != 0 { + t.Fatal("审计不应记录响应体") + } + } + got := audit.sync[0] + if got.UserID != 9 || got.TargetID != "3/ops/app" || got.Endpoint != "/api/k8s/secret/3/ops/app/reveal" { + t.Fatalf("审计元数据不符: %+v", got) + } + var body model.SecretRevealAuditRecord + if err := json.Unmarshal(got.RequestBody, &body); err != nil || body.Username != "bob" || strings.Join(body.Keys, ",") != "alpha" { + t.Fatalf("审计内容不符: %s", got.RequestBody) + } +} + +type failingAudit struct { + systemService.AuditService +} + +func (failingAudit) CreateAuditLog(context.Context, *model.CreateAuditLogRequest) error { + return errors.New("audit db unavailable") +} + +// 审计写入失败：HTTP 500、不返回明文、仍带 no-store +func TestRevealEndpointAuditFailureReturns500WithoutPlaintext(t *testing.T) { + gin.SetMode(gin.TestMode) + svc := service.NewSecretService(handlerSecretManager{}, failingAudit{}, zap.NewNop()) + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 9, Username: "bob"}); c.Next() }) + NewK8sSecretHandler(svc).RegisterRouters(engine) + + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", "/api/k8s/secret/3/ops/app/reveal", nil)) + if rec.Code != 500 { + t.Fatalf("审计失败应返回 500, got %d", rec.Code) + } + if rec.Header().Get("Cache-Control") != "no-store" { + t.Fatal("失败响应也应禁止缓存") + } + body := rec.Body.String() + if strings.Contains(body, handlerFixtureValue) || strings.Contains(body, base64.StdEncoding.EncodeToString([]byte(handlerFixtureValue))) { + t.Fatal("审计失败时响应中出现了明文") + } +} diff --git a/internal/k8s/api/serviceaccount.go b/internal/k8s/api/serviceaccount.go index e1a03e6f..56e46ab9 100644 --- a/internal/k8s/api/serviceaccount.go +++ b/internal/k8s/api/serviceaccount.go @@ -251,7 +251,8 @@ func (h *K8sServiceAccountHandler) UpdateServiceAccountYaml(ctx *gin.Context) { req.Namespace = namespace req.Name = name - base.HandleRequest(ctx, &req, func() (interface{}, error) { + // YAML 带 resourceVersion 时走乐观锁，版本冲突返回 HTTP 409 + handleMappedRequest(ctx, &req, func() (interface{}, error) { return nil, h.serviceAccountService.UpdateServiceAccountYaml(ctx.Request.Context(), &req) }) } @@ -281,8 +282,12 @@ func (h *K8sServiceAccountHandler) GetServiceAccountToken(ctx *gin.Context) { req.Namespace = namespace req.Name = name - base.HandleRequest(ctx, &req, func() (interface{}, error) { - return h.serviceAccountService.GetServiceAccountToken(ctx.Request.Context(), &req) + actor := sensitiveReadActor(ctx) + // token 响应禁止被浏览器或中间代理缓存 + setNoStore(ctx) + + handleAuditedRequest(ctx, &req, func() (interface{}, error) { + return h.serviceAccountService.GetServiceAccountToken(ctx.Request.Context(), &req, actor) }) } @@ -311,7 +316,29 @@ func (h *K8sServiceAccountHandler) CreateServiceAccountToken(ctx *gin.Context) { req.Namespace = namespace req.ServiceAccountName = name - base.HandleRequest(ctx, &req, func() (interface{}, error) { - return h.serviceAccountService.CreateServiceAccountToken(ctx.Request.Context(), &req) + actor := sensitiveReadActor(ctx) + setNoStore(ctx) + + handleAuditedRequest(ctx, &req, func() (interface{}, error) { + return h.serviceAccountService.CreateServiceAccountToken(ctx.Request.Context(), &req, actor) }) } + +// sensitiveReadActor 从登录态中提取敏感接口（Secret reveal、SA token）审计所需的操作者信息 +func sensitiveReadActor(ctx *gin.Context) model.SecretRevealActor { + user := base.GetUserInfoFromGinContext(ctx) + return model.SecretRevealActor{ + UserID: user.UserID, + Username: user.Username, + IP: user.IP, + UserAgent: user.UserAgent, + TraceID: ctx.GetHeader("X-Trace-ID"), + Endpoint: ctx.Request.URL.Path, + } +} + +// setNoStore 敏感响应禁止被浏览器或中间代理缓存 +func setNoStore(ctx *gin.Context) { + ctx.Header("Cache-Control", "no-store") + ctx.Header("Pragma", "no-cache") +} diff --git a/internal/k8s/api/serviceaccount_route_test.go b/internal/k8s/api/serviceaccount_route_test.go new file mode 100644 index 00000000..afada2f8 --- /dev/null +++ b/internal/k8s/api/serviceaccount_route_test.go @@ -0,0 +1,129 @@ +package api + +import ( + "context" + "net/http/httptest" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/service" + "github.com/GoSimplicity/AI-CloudOps/internal/middleware" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + systemservice "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" + "go.uber.org/zap" + authv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" +) + +type routeGrantRoles struct { + systemservice.RoleService + apis []*model.Api +} + +func (r routeGrantRoles) GetUserRoles(context.Context, int) (*model.ListResp[*model.Role], error) { + return &model.ListResp[*model.Role]{Items: []*model.Role{{Status: 1, Apis: r.apis}}}, nil +} + +// routeProbeSAManager 记录 token 相关的 manager 调用：handler 被执行时必然会调用其中之一 +type routeProbeSAManager struct { + *handlerSAManager + reveals, issues int +} + +func (m *routeProbeSAManager) GetServiceAccount(ctx context.Context, clusterID int, namespace, name string) (*corev1.ServiceAccount, error) { + m.reveals++ + return m.handlerSAManager.GetServiceAccount(ctx, clusterID, namespace, name) +} + +func (m *routeProbeSAManager) CreateServiceAccountToken(ctx context.Context, clusterID int, namespace, name string, tr *authv1.TokenRequest) (*authv1.TokenRequest, error) { + m.issues++ + return m.handlerSAManager.CreateServiceAccountToken(ctx, clusterID, namespace, name, tr) +} + +// newSARouteAuthEngine 与生产一致：gin.New()（RedirectTrailingSlash=true、RedirectFixedPath=false）+ 认证中间件 + 真实路由注册 +// （不测 RedirectFixedPath=true：gin v1.10.0 的 findCaseInsensitivePath 在本路由树上会 panic，生产也未开启） +func newSARouteAuthEngine(mgr *routeProbeSAManager, apis []*model.Api) *gin.Engine { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 9, Username: "bob"}); c.Next() }) + engine.Use(middleware.NewAuthMiddleware(routeGrantRoles{apis: apis}).CheckAuth()) + NewK8sServiceAccountHandler(service.NewServiceAccountService(mgr, &recordingAudit{}, zap.NewNop())).RegisterRouters(engine) + return engine +} + +// serveFollowingRedirects 发请求并跟随最多 3 次重定向（模拟客户端自动跟随），返回最终状态码 +func serveFollowingRedirects(engine *gin.Engine, method, path string) int { + for i := 0; i < 4; i++ { + req := httptest.NewRequest(method, path, strings.NewReader(`{"expiration_seconds":600}`)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, req) + if rec.Code < 300 || rec.Code >= 400 { + return rec.Code + } + loc := rec.Header().Get("Location") + if loc == "" { + return rec.Code + } + path = loc + } + return -1 +} + +// SEC-29 P3-4：只有通配/参数化授权（没有显式授予 token 权限码）时，末尾斜杠、大小写变体、重复斜杠等路径 +// 都不能执行 k8s:serviceaccount:token（GET）或 k8s:serviceaccount:token:create（POST）的 handler； +// 即使跟随 gin 的末尾斜杠重定向，落到规范路径后仍被显式授权检查拦截。 +func TestServiceAccountTokenRoutesTrailingSlashAndCaseCannotBypassPermission(t *testing.T) { + gin.SetMode(gin.TestMode) + wildcardOnly := []*model.Api{ + {Path: "/api/k8s/*", Method: 1}, {Path: "/api/k8s/*", Method: 2}, + {Path: "/*", Method: 1}, {Path: "/*", Method: 2}, + {Path: "/api/k8s/serviceaccount/:cluster_id/:namespace/:name/:action", Method: 1}, + {Path: "/api/k8s/serviceaccount/:cluster_id/:namespace/:name/:action", Method: 2}, + } + variants := []string{ + "/api/k8s/serviceaccount/3/ops/app/token", + "/api/k8s/serviceaccount/3/ops/app/token/", + "/api/k8s/serviceaccount/3/ops/app/token//", + "/api/k8s/serviceaccount/3/ops/app//token", + "/api/k8s/serviceaccount/3/ops/app/Token", + "/api/k8s/serviceaccount/3/ops/app/TOKEN", + "/api/k8s/serviceaccount/3/ops/app/TOKEN/", + "/api/k8s/ServiceAccount/3/ops/app/token", + "/API/K8S/SERVICEACCOUNT/3/ops/app/token", + "/api/k8s/serviceaccount/3/ops/app/token/.", + "/api/k8s/serviceaccount/3/ops/app/./token", + } + for _, method := range []string{"GET", "POST"} { + for _, path := range variants { + mgr := &routeProbeSAManager{handlerSAManager: &handlerSAManager{}} + code := serveFollowingRedirects(newSARouteAuthEngine(mgr, wildcardOnly), method, path) + if mgr.reveals != 0 || mgr.issues != 0 || (code >= 200 && code < 300) { + t.Fatalf("%s %s 绕过了 token 显式授权: status=%d reveals=%d issues=%d", method, path, code, mgr.reveals, mgr.issues) + } + } + } + + // 只显式授予 GET（k8s:serviceaccount:token）时，POST 的任何路径变体都不能签发 token + getOnly := append([]*model.Api{{Path: model.K8sServiceAccountTokenRoute, Method: 1}}, wildcardOnly...) + for _, path := range variants { + mgr := &routeProbeSAManager{handlerSAManager: &handlerSAManager{}} + code := serveFollowingRedirects(newSARouteAuthEngine(mgr, getOnly), "POST", path) + if mgr.issues != 0 || (code >= 200 && code < 300) { + t.Fatalf("POST %s 未授予 token:create 却签发了 token: status=%d", path, code) + } + } + + // 对照：显式授权后规范路径可以正常访问（证明上面的拒绝来自授权检查而非路由缺失） + for _, tt := range []struct { + method string + code int8 + }{{"GET", 1}, {"POST", 2}} { + mgr := &routeProbeSAManager{handlerSAManager: &handlerSAManager{}} + engine := newSARouteAuthEngine(mgr, []*model.Api{{Path: model.K8sServiceAccountTokenRoute, Method: tt.code}}) + if code := serveFollowingRedirects(engine, tt.method, "/api/k8s/serviceaccount/3/ops/app/token"); code != 200 || mgr.reveals+mgr.issues != 1 { + t.Fatalf("%s 显式授权后应可访问: status=%d", tt.method, code) + } + } +} diff --git a/internal/k8s/api/serviceaccount_test.go b/internal/k8s/api/serviceaccount_test.go new file mode 100644 index 00000000..174f622b --- /dev/null +++ b/internal/k8s/api/serviceaccount_test.go @@ -0,0 +1,212 @@ +package api + +import ( + "context" + "encoding/json" + "net/http/httptest" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/manager" + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/service" + "github.com/GoSimplicity/AI-CloudOps/internal/middleware" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" + "go.uber.org/zap" + authv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// 测试用 token（非真实凭据） +const ( + handlerOwnToken = "fake-handler-sa-token-01" + handlerIssuedToken = "fake-handler-sa-token-02" +) + +type handlerSAManager struct { + manager.ServiceAccountManager + tokenTarget string + tokenReq *authv1.TokenRequest +} + +func (handlerSAManager) GetServiceAccount(_ context.Context, _ int, namespace, name string) (*corev1.ServiceAccount, error) { + return &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, UID: types.UID("uid-" + name)}}, nil +} + +func (handlerSAManager) GetServiceAccountSecrets(_ context.Context, _ int, sa *corev1.ServiceAccount) ([]corev1.Secret, error) { + namespace, name := sa.Namespace, sa.Name + return []corev1.Secret{{ + ObjectMeta: metav1.ObjectMeta{Name: name + "-token", Namespace: namespace, Annotations: map[string]string{ + corev1.ServiceAccountNameKey: name, corev1.ServiceAccountUIDKey: "uid-" + name, + }}, + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{corev1.ServiceAccountTokenKey: []byte(handlerOwnToken)}, + }}, nil +} + +func (m *handlerSAManager) CreateServiceAccountToken(_ context.Context, _ int, _, name string, tr *authv1.TokenRequest) (*authv1.TokenRequest, error) { + m.tokenTarget = name + m.tokenReq = tr.DeepCopy() + out := tr.DeepCopy() + out.Status.Token = handlerIssuedToken + return out, nil +} + +func TestServiceAccountTokenRoutesMatchPermissionCode(t *testing.T) { + gin.SetMode(gin.TestMode) + engine := gin.New() + NewK8sServiceAccountHandler(nil).RegisterRouters(engine) + found := map[string]bool{} + for _, r := range engine.Routes() { + if r.Path == model.K8sServiceAccountTokenRoute { + found[r.Method] = true + } + } + if !found["GET"] || !found["POST"] { + t.Fatalf("token 路由与权限码路径不一致: %v", found) + } +} + +func newSATokenEngine(audit service.ServiceAccountService) *gin.Engine { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 9, Username: "bob"}); c.Next() }) + NewK8sServiceAccountHandler(audit).RegisterRouters(engine) + return engine +} + +func TestServiceAccountTokenEndpointsNoStoreAndAuditWithoutToken(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, tt := range []struct { + method, body, token, op string + }{ + {"GET", "", handlerOwnToken, model.ServiceAccountTokenRevealAuditOperation}, + // 请求体中的 service_account_name 不能覆盖 URL 中的 SA + {"POST", `{"expiration_seconds":600,"service_account_name":"admin"}`, handlerIssuedToken, model.ServiceAccountTokenCreateAuditOperation}, + } { + t.Run(tt.method, func(t *testing.T) { + audit := &recordingAudit{} + mgr := &handlerSAManager{} + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 9, Username: "bob"}); c.Next() }) + engine.Use(middleware.NewAuditLogMiddleware(audit, zap.NewNop()).AuditLog()) + NewK8sServiceAccountHandler(service.NewServiceAccountService(mgr, audit, zap.NewNop())).RegisterRouters(engine) + + var req = httptest.NewRequest(tt.method, "/api/k8s/serviceaccount/3/ops/app/token", strings.NewReader(tt.body)) + if tt.body != "" { + req.Header.Set("Content-Type", "application/json") + } + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, req) + if rec.Code != 200 || rec.Header().Get("Cache-Control") != "no-store" || rec.Header().Get("Pragma") != "no-cache" { + t.Fatalf("status=%d headers=%v body=%s", rec.Code, rec.Header(), rec.Body.String()) + } + var resp struct { + Code int `json:"code"` + Data model.ServiceAccountTokenInfo `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil || resp.Data.Token != tt.token { + t.Fatalf("响应不符: %s", rec.Body.String()) + } + if tt.method == "POST" && mgr.tokenTarget != "app" { + t.Fatalf("TokenRequest 目标应为 URL 中的 SA, got %q", mgr.tokenTarget) + } + + audit.mu.Lock() + defer audit.mu.Unlock() + if len(audit.sync) != 1 || audit.sync[0].OperationType != tt.op || audit.sync[0].TargetID != "3/ops/app" || + audit.sync[0].Endpoint != "/api/k8s/serviceaccount/3/ops/app/token" || audit.sync[0].UserID != 9 { + t.Fatalf("同步审计不符: %+v", audit.sync) + } + raw, _ := json.Marshal(append(append([]*model.CreateAuditLogRequest{}, audit.sync...), audit.async...)) + if strings.Contains(string(raw), tt.token) { + t.Fatal("审计记录中出现了 token") + } + for _, l := range audit.async { + if len(l.ResponseBody) != 0 { + t.Fatal("审计中间件不应记录响应体") + } + } + }) + } +} + +func TestServiceAccountTokenEndpointsAuditFailureReturns500WithoutToken(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, method := range []string{"GET", "POST"} { + t.Run(method, func(t *testing.T) { + engine := newSATokenEngine(service.NewServiceAccountService(&handlerSAManager{}, failingAudit{}, zap.NewNop())) + req := httptest.NewRequest(method, "/api/k8s/serviceaccount/3/ops/app/token", strings.NewReader(`{}`)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, req) + if rec.Code != 500 { + t.Fatalf("审计失败应返回 500, got %d", rec.Code) + } + if rec.Header().Get("Cache-Control") != "no-store" { + t.Fatal("失败响应也应禁止缓存") + } + body := rec.Body.String() + if strings.Contains(body, handlerOwnToken) || strings.Contains(body, handlerIssuedToken) { + t.Fatal("审计失败时响应中出现了 token") + } + }) + } +} + +func TestServiceAccountTokenCreateExpirationValidation(t *testing.T) { + gin.SetMode(gin.TestMode) + for _, tt := range []struct { + name, body string + wantHTTP int + wantExp int64 // 传给 K8s 的有效期；0 表示不应调用 K8s + }{ + {"不传", `{}`, 200, 3600}, + {"传0", `{"expiration_seconds":0}`, 200, 3600}, + {"599", `{"expiration_seconds":599}`, 400, 0}, + {"600", `{"expiration_seconds":600}`, 200, 600}, + {"3600", `{"expiration_seconds":3600}`, 200, 3600}, + {"3601", `{"expiration_seconds":3601}`, 400, 0}, + {"负数", `{"expiration_seconds":-1}`, 400, 0}, + } { + t.Run(tt.name, func(t *testing.T) { + audit := &recordingAudit{} + mgr := &handlerSAManager{} + engine := newSATokenEngine(service.NewServiceAccountService(mgr, audit, zap.NewNop())) + req := httptest.NewRequest("POST", "/api/k8s/serviceaccount/3/ops/app/token", strings.NewReader(tt.body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, req) + if rec.Code != tt.wantHTTP || rec.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("status=%d headers=%v body=%s", rec.Code, rec.Header(), rec.Body.String()) + } + var resp struct { + Code int `json:"code"` + Data model.ServiceAccountTokenInfo `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + audit.mu.Lock() + defer audit.mu.Unlock() + if len(audit.sync) != 1 || audit.sync[0].StatusCode != tt.wantHTTP { + t.Fatalf("应写一条状态码为 %d 的审计: %+v", tt.wantHTTP, audit.sync) + } + if tt.wantExp == 0 { + if resp.Code != 1 || resp.Data.Token != "" || strings.Contains(rec.Body.String(), handlerIssuedToken) { + t.Fatalf("越界应返回参数错误且不含 token: %s", rec.Body.String()) + } + if mgr.tokenTarget != "" || mgr.tokenReq != nil { + t.Fatal("越界时不应调用 K8s TokenRequest") + } + return + } + if resp.Code != 0 || resp.Data.Token != handlerIssuedToken || mgr.tokenReq == nil || + mgr.tokenReq.Spec.ExpirationSeconds == nil || *mgr.tokenReq.Spec.ExpirationSeconds != tt.wantExp { + t.Fatalf("应以 %d 秒签发: %s %+v", tt.wantExp, rec.Body.String(), mgr.tokenReq) + } + }) + } +} diff --git a/internal/k8s/api/serviceaccount_update_yaml_test.go b/internal/k8s/api/serviceaccount_update_yaml_test.go new file mode 100644 index 00000000..cfda34d7 --- /dev/null +++ b/internal/k8s/api/serviceaccount_update_yaml_test.go @@ -0,0 +1,86 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/service" + k8sutils "github.com/GoSimplicity/AI-CloudOps/internal/k8s/utils" + "github.com/gin-gonic/gin" + "go.uber.org/zap" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// yamlUpdateSAManager 模拟 manager 层的两种 YAML 更新路径 +type yamlUpdateSAManager struct { + *handlerSAManager + strictErr error + strictCalls, retries int + strictRV string +} + +func (m *yamlUpdateSAManager) UpdateServiceAccountWithResourceVersion(_ context.Context, _ int, _, _, rv string, _ k8sutils.ServiceAccountMutateFunc) error { + m.strictCalls++ + m.strictRV = rv + return m.strictErr +} + +func (m *yamlUpdateSAManager) UpdateServiceAccountWithRetry(context.Context, int, string, string, k8sutils.ServiceAccountMutateFunc) error { + m.retries++ + return nil +} + +func putSAYaml(engine *gin.Engine, yaml string) *httptest.ResponseRecorder { + body, _ := json.Marshal(map[string]interface{}{"cluster_id": 3, "namespace": "ops", "name": "app", "yaml_content": yaml}) + req := httptest.NewRequest("PUT", "/api/k8s/serviceaccount/3/ops/app/update/yaml", strings.NewReader(string(body))) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, req) + return rec +} + +// SEC-29(A)：YAML 带过期 resourceVersion → HTTP 409 + code=1（respondBusinessError 映射 BusinessError{ErrK8sResourceConflict}）； +// 版本正确 → 200；不带版本 → 走 RetryOnConflict 路径 +func TestUpdateServiceAccountYamlConflictMapsTo409(t *testing.T) { + gin.SetMode(gin.TestMode) + withRV := "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: app\n resourceVersion: \"4\"\n" + conflict := apierrors.NewConflict(schema.GroupResource{Resource: "serviceaccounts"}, "app", errors.New("the object has been modified")) + + mgr := &yamlUpdateSAManager{handlerSAManager: &handlerSAManager{}, strictErr: conflict} + rec := putSAYaml(newSATokenEngine(service.NewServiceAccountService(mgr, &recordingAudit{}, zap.NewNop())), withRV) + var resp struct { + Code int `json:"code"` + Message string `json:"message"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &resp) + if rec.Code != 409 || resp.Code != 1 || !strings.Contains(resp.Message, "resourceVersion") { + t.Fatalf("版本冲突应返回 HTTP 409 + code=1: status=%d body=%s", rec.Code, rec.Body.String()) + } + if mgr.strictCalls != 1 || mgr.retries != 0 || mgr.strictRV != "4" { + t.Fatalf("应只走一次乐观锁更新、不重试: strict=%d retry=%d rv=%q", mgr.strictCalls, mgr.retries, mgr.strictRV) + } + + mgr = &yamlUpdateSAManager{handlerSAManager: &handlerSAManager{}} + if rec := putSAYaml(newSATokenEngine(service.NewServiceAccountService(mgr, &recordingAudit{}, zap.NewNop())), withRV); rec.Code != 200 || mgr.strictCalls != 1 { + t.Fatalf("版本正确应返回 200: status=%d body=%s", rec.Code, rec.Body.String()) + } + + mgr = &yamlUpdateSAManager{handlerSAManager: &handlerSAManager{}} + noRV := "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: app\n" + if rec := putSAYaml(newSATokenEngine(service.NewServiceAccountService(mgr, &recordingAudit{}, zap.NewNop())), noRV); rec.Code != 200 || mgr.retries != 1 || mgr.strictCalls != 0 { + t.Fatalf("不带 resourceVersion 应走重试路径: status=%d strict=%d retry=%d", rec.Code, mgr.strictCalls, mgr.retries) + } + + // 其他错误保持项目约定：HTTP 200 + code=1 + mgr = &yamlUpdateSAManager{handlerSAManager: &handlerSAManager{}, strictErr: errors.New("boom")} + rec = putSAYaml(newSATokenEngine(service.NewServiceAccountService(mgr, &recordingAudit{}, zap.NewNop())), withRV) + _ = json.Unmarshal(rec.Body.Bytes(), &resp) + if rec.Code != 200 || resp.Code != 1 { + t.Fatalf("非冲突错误应保持 200 + code=1: status=%d body=%s", rec.Code, rec.Body.String()) + } +} diff --git a/internal/k8s/manager/serviceaccount_manager.go b/internal/k8s/manager/serviceaccount_manager.go index 5ec1f1be..4257bf38 100644 --- a/internal/k8s/manager/serviceaccount_manager.go +++ b/internal/k8s/manager/serviceaccount_manager.go @@ -2,8 +2,10 @@ package manager import ( "context" + "fmt" "github.com/GoSimplicity/AI-CloudOps/internal/k8s/client" + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/utils" "go.uber.org/zap" authv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" @@ -18,12 +20,18 @@ type ServiceAccountManager interface { GetServiceAccount(ctx context.Context, clusterID int, namespace, name string) (*corev1.ServiceAccount, error) GetServiceAccountList(ctx context.Context, clusterID int, namespace string, listOptions metav1.ListOptions) (*corev1.ServiceAccountList, error) UpdateServiceAccount(ctx context.Context, clusterID int, namespace string, sa *corev1.ServiceAccount) error + // UpdateServiceAccountWithRetry 冲突时重试的更新：每次先 Get 最新版本，经 mutate 应用允许修改的字段后 Update； + // name/namespace 与 secrets 引用始终以集群最新版本为准 + UpdateServiceAccountWithRetry(ctx context.Context, clusterID int, namespace, name string, mutate utils.ServiceAccountMutateFunc) error + // UpdateServiceAccountWithResourceVersion 严格乐观锁更新：以调用方提交的 resourceVersion 只 Update 一次，冲突直接返回 Conflict，不重试 + UpdateServiceAccountWithResourceVersion(ctx context.Context, clusterID int, namespace, name, resourceVersion string, mutate utils.ServiceAccountMutateFunc) error DeleteServiceAccount(ctx context.Context, clusterID int, namespace, name string, deleteOptions metav1.DeleteOptions) error // 高级功能 PatchServiceAccount(ctx context.Context, clusterID int, namespace, name string, data []byte, patchType string) (*corev1.ServiceAccount, error) - GetServiceAccountSecrets(ctx context.Context, clusterID int, namespace, name string) ([]corev1.Secret, error) + // GetServiceAccountSecrets 读取 sa.secrets 引用的 Secret；sa 由调用方传入（复用已读取的对象，不再重复 Get） + GetServiceAccountSecrets(ctx context.Context, clusterID int, sa *corev1.ServiceAccount) ([]corev1.Secret, error) GetServiceAccountTokens(ctx context.Context, clusterID int, namespace, name string) ([]string, error) CreateServiceAccountToken(ctx context.Context, clusterID int, namespace, name string, tokenRequest *authv1.TokenRequest) (*authv1.TokenRequest, error) BindServiceAccountToRole(ctx context.Context, clusterID int, namespace, saName, roleName string) error @@ -144,6 +152,59 @@ func (m *serviceAccountManager) UpdateServiceAccount(ctx context.Context, cluste return nil } +func (m *serviceAccountManager) UpdateServiceAccountWithRetry(ctx context.Context, clusterID int, namespace, name string, mutate utils.ServiceAccountMutateFunc) error { + kubeClient, err := m.client.GetKubeClient(clusterID) + if err != nil { + m.logger.Error("获取Kubernetes客户端失败", + zap.Int("clusterID", clusterID), + zap.Error(err)) + return err + } + + if _, err := utils.UpdateServiceAccountWithRetry(ctx, kubeClient.CoreV1().ServiceAccounts(namespace), name, mutate); err != nil { + m.logger.Error("更新ServiceAccount失败", + zap.Int("clusterID", clusterID), + zap.String("namespace", namespace), + zap.String("name", name), + zap.Error(err)) + return err + } + + m.logger.Info("成功更新ServiceAccount", + zap.Int("clusterID", clusterID), + zap.String("namespace", namespace), + zap.String("name", name)) + + return nil +} + +func (m *serviceAccountManager) UpdateServiceAccountWithResourceVersion(ctx context.Context, clusterID int, namespace, name, resourceVersion string, mutate utils.ServiceAccountMutateFunc) error { + kubeClient, err := m.client.GetKubeClient(clusterID) + if err != nil { + m.logger.Error("获取Kubernetes客户端失败", + zap.Int("clusterID", clusterID), + zap.Error(err)) + return err + } + + if _, err := utils.UpdateServiceAccountWithResourceVersion(ctx, kubeClient.CoreV1().ServiceAccounts(namespace), name, resourceVersion, mutate); err != nil { + m.logger.Error("按resourceVersion更新ServiceAccount失败", + zap.Int("clusterID", clusterID), + zap.String("namespace", namespace), + zap.String("name", name), + zap.String("resourceVersion", resourceVersion), + zap.Error(err)) + return err + } + + m.logger.Info("成功更新ServiceAccount", + zap.Int("clusterID", clusterID), + zap.String("namespace", namespace), + zap.String("name", name)) + + return nil +} + func (m *serviceAccountManager) DeleteServiceAccount(ctx context.Context, clusterID int, namespace, name string, deleteOptions metav1.DeleteOptions) error { kubeClient, err := m.client.GetKubeClient(clusterID) if err != nil { @@ -201,11 +262,11 @@ func (m *serviceAccountManager) PatchServiceAccount(ctx context.Context, cluster return sa, nil } -func (m *serviceAccountManager) GetServiceAccountSecrets(ctx context.Context, clusterID int, namespace, name string) ([]corev1.Secret, error) { - sa, err := m.GetServiceAccount(ctx, clusterID, namespace, name) - if err != nil { - return nil, err +func (m *serviceAccountManager) GetServiceAccountSecrets(ctx context.Context, clusterID int, sa *corev1.ServiceAccount) ([]corev1.Secret, error) { + if sa == nil { + return nil, fmt.Errorf("ServiceAccount不能为空") } + namespace, name := sa.Namespace, sa.Name kubeClient, err := m.client.GetKubeClient(clusterID) if err != nil { @@ -239,18 +300,30 @@ func (m *serviceAccountManager) GetServiceAccountSecrets(ctx context.Context, cl } func (m *serviceAccountManager) GetServiceAccountTokens(ctx context.Context, clusterID int, namespace, name string) ([]string, error) { - secrets, err := m.GetServiceAccountSecrets(ctx, clusterID, namespace, name) + sa, err := m.GetServiceAccount(ctx, clusterID, namespace, name) if err != nil { return nil, err } + secrets, err := m.GetServiceAccountSecrets(ctx, clusterID, sa) + if err != nil { + return nil, err + } + + // SEC-29：只返回注解指向本 SA（名称 + UID）的 token，sa.secrets 中借挂的其他 SA token 一律忽略 var tokens []string - for _, secret := range secrets { - if secret.Type == corev1.SecretTypeServiceAccountToken { - if token, exists := secret.Data["token"]; exists { - tokens = append(tokens, string(token)) + for i := range secrets { + if !utils.IsServiceAccountTokenSecretFor(&secrets[i], sa) { + if secrets[i].Type == corev1.SecretTypeServiceAccountToken { + m.logger.Warn("忽略不属于该ServiceAccount的token Secret", + zap.Int("clusterID", clusterID), + zap.String("namespace", namespace), + zap.String("serviceAccountName", name), + zap.String("secretName", secrets[i].Name)) } + continue } + tokens = append(tokens, string(secrets[i].Data[corev1.ServiceAccountTokenKey])) } m.logger.Debug("获取ServiceAccount的Token", diff --git a/internal/k8s/service/cluster_service.go b/internal/k8s/service/cluster_service.go index 1f226c3e..6223f19e 100644 --- a/internal/k8s/service/cluster_service.go +++ b/internal/k8s/service/cluster_service.go @@ -77,7 +77,8 @@ func (s *clusterService) ListClusters(ctx context.Context, req *model.ListCluste return model.ListResp[*model.K8sCluster]{}, fmt.Errorf("查询所有集群失败: %w", err) } - utils.CleanClusterSensitiveInfoList(list) + // kubeconfig 由 model.K8sCluster.MarshalJSON 统一屏蔽并转换为 has_kube_config， + // 这里保留原值以便正确计算该标志。 return model.ListResp[*model.K8sCluster]{ Total: total, @@ -224,6 +225,7 @@ func (s *clusterService) UpdateCluster(ctx context.Context, req *model.UpdateClu if cluster.ApiServerAddr == "" { cluster.ApiServerAddr = existingCluster.ApiServerAddr } + // kubeconfig 选填：不传或空字符串保留原值；非空则整段替换，下方 connection.Parse 与创建接口执行相同校验 if cluster.KubeConfigContent == "" { cluster.KubeConfigContent = existingCluster.KubeConfigContent } diff --git a/internal/k8s/service/secret_service.go b/internal/k8s/service/secret_service.go index a203f1f1..e5bc6f67 100644 --- a/internal/k8s/service/secret_service.go +++ b/internal/k8s/service/secret_service.go @@ -27,7 +27,11 @@ package service import ( "context" + "encoding/json" + stderrors "errors" "fmt" + "net/http" + "sort" "strings" "time" @@ -35,6 +39,7 @@ import ( "github.com/GoSimplicity/AI-CloudOps/internal/k8s/utils" k8sutils "github.com/GoSimplicity/AI-CloudOps/internal/k8s/utils" "github.com/GoSimplicity/AI-CloudOps/internal/model" + systemService "github.com/GoSimplicity/AI-CloudOps/internal/system/service" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -51,17 +56,21 @@ type SecretService interface { UpdateSecretByYaml(ctx context.Context, req *model.UpdateSecretByYamlReq) error DeleteSecret(ctx context.Context, req *model.DeleteSecretReq) error GetSecretYAML(ctx context.Context, req *model.GetSecretYamlReq) (*model.K8sYaml, error) + // RevealSecret 返回 Secret 明文；每次调用都会写一条审计（只含元数据与键名，不含值） + RevealSecret(ctx context.Context, req *model.RevealSecretReq, actor model.SecretRevealActor) (*model.K8sSecretReveal, error) } // secretService Secret服务实现 type secretService struct { secretManager manager.SecretManager + auditService systemService.AuditService logger *zap.Logger } -func NewSecretService(secretManager manager.SecretManager, logger *zap.Logger) SecretService { +func NewSecretService(secretManager manager.SecretManager, auditService systemService.AuditService, logger *zap.Logger) SecretService { return &secretService{ secretManager: secretManager, + auditService: auditService, logger: logger, } } @@ -189,21 +198,25 @@ func (s *secretService) convertToK8sSecret(secret *corev1.Secret, clusterID int) } return &model.K8sSecret{ - Name: secret.Name, - Namespace: secret.Namespace, - ClusterID: clusterID, - UID: string(secret.UID), - Type: model.K8sSecretType(secret.Type), - Data: model.BinaryDataMap(secret.Data), - StringData: secret.StringData, // 通常为空，因为K8s会转换为Data - Labels: secret.Labels, - Annotations: secret.Annotations, - Immutable: immutable, - DataCount: dataCount, - Size: size, - Age: time.Since(secret.CreationTimestamp.Time).String(), - CreatedAt: secret.CreationTimestamp.Time, - RawSecret: secret, + Name: secret.Name, + Namespace: secret.Namespace, + ClusterID: clusterID, + UID: string(secret.UID), + Type: model.K8sSecretType(secret.Type), + // SEC-25：列表/详情默认脱敏，只返回键与值长度；明文走 reveal 接口 + Data: k8sutils.RedactSecretData(secret.Data), + StringData: k8sutils.RedactSecretStringData(secret.StringData), // 通常为空，因为K8s会转换为Data + DataKeys: k8sutils.SecretDataKeyInfos(secret.Data), + StringDataKeys: k8sutils.SecretStringDataKeyInfos(secret.StringData), + Redacted: true, + Labels: secret.Labels, + Annotations: k8sutils.RedactSecretAnnotations(secret.Annotations), + Immutable: immutable, + DataCount: dataCount, + Size: size, + Age: time.Since(secret.CreationTimestamp.Time).String(), + CreatedAt: secret.CreationTimestamp.Time, + RawSecret: secret, } } @@ -275,10 +288,11 @@ func (s *secretService) UpdateSecret(ctx context.Context, req *model.UpdateSecre updatedSecret := existingSecret.DeepCopy() - updatedSecret.Data = map[string][]byte(req.Data) - updatedSecret.StringData = req.StringData + // SEC-25：请求未携带 data（nil）时保留集群现有 data；携带空对象 {} 表示删除全部 data 键 + updatedSecret.Data = preserveRedactedData(map[string][]byte(req.Data), existingSecret.Data) + updatedSecret.Data, updatedSecret.StringData = preserveRedactedStringData(req.StringData, map[string][]byte(req.Data), updatedSecret.Data, existingSecret) updatedSecret.Labels = req.Labels - updatedSecret.Annotations = req.Annotations + updatedSecret.Annotations = preserveLastApplied(req.Annotations, existingSecret.Annotations) // Type和Immutable字段在创建后通常不能修改，保持原值 // 如果需要修改，K8s会返回错误 @@ -363,6 +377,13 @@ func (s *secretService) CreateSecretByYaml(ctx context.Context, req *model.Creat return fmt.Errorf("请求参数不能为空") } + // SEC-25：脱敏占位符不是真实值，禁止用于创建 + if hasPlaceholder, err := k8sutils.SecretYAMLHasRedactedPlaceholder(req.YAML); err != nil { + return err + } else if hasPlaceholder { + return fmt.Errorf("YAML 中包含脱敏占位符 %s，请填写真实值后再创建", k8sutils.SecretRedactedPlaceholder) + } + sec, err := k8sutils.YAMLToSecret(req.YAML) if err != nil { return fmt.Errorf("解析YAML失败: %w", err) @@ -404,8 +425,24 @@ func (s *secretService) UpdateSecretByYaml(ctx context.Context, req *model.Updat zap.String("namespace", req.Namespace), zap.String("name", req.Name)) + // 获取现有资源以获取ResourceVersion（用于乐观锁，避免并发冲突），并用于恢复脱敏占位符 + existing, err := s.secretManager.GetSecret(ctx, req.ClusterID, req.Namespace, req.Name) + if err != nil { + if errors.IsNotFound(err) { + return fmt.Errorf("Secret不存在: %s/%s", req.Namespace, req.Name) + } + s.logger.Error("获取现有Secret失败", zap.Error(err)) + return fmt.Errorf("获取现有Secret失败: %w", err) + } + + // SEC-25：详情 YAML 已脱敏，仍为占位符的键保持集群中的原值，避免占位符被写回 + restoredYAML, err := k8sutils.RestoreRedactedSecretYAML(req.YAML, existing) + if err != nil { + return err + } + // 解析YAML - sec, err := k8sutils.YAMLToSecret(req.YAML) + sec, err := k8sutils.YAMLToSecret(restoredYAML) if err != nil { s.logger.Error("解析YAML失败", zap.Error(err)) return fmt.Errorf("解析YAML失败: %w", err) @@ -415,16 +452,6 @@ func (s *secretService) UpdateSecretByYaml(ctx context.Context, req *model.Updat sec.Namespace = req.Namespace sec.Name = req.Name - // 获取现有资源以获取ResourceVersion（用于乐观锁，避免并发冲突） - existing, err := s.secretManager.GetSecret(ctx, req.ClusterID, req.Namespace, req.Name) - if err != nil { - if errors.IsNotFound(err) { - return fmt.Errorf("Secret不存在: %s/%s", req.Namespace, req.Name) - } - s.logger.Error("获取现有Secret失败", zap.Error(err)) - return fmt.Errorf("获取现有Secret失败: %w", err) - } - // 保留ResourceVersion和UID等关键元数据 sec.ResourceVersion = existing.ResourceVersion sec.UID = existing.UID @@ -445,3 +472,273 @@ func (s *secretService) UpdateSecretByYaml(ctx context.Context, req *model.Updat return nil } + +// preserveRedactedData 兼容脱敏后的详情回填：请求中值为空、而集群中该键已有非空值时保留原值。 +// 详情接口只返回空值，前端若直接回填提交，不能因此把真实值清空。 +// 如需删除某个键，请在请求中去掉该键。 +// +// SEC-25 P3-2：requested 为 nil（请求未携带 data 字段或为 null）时视为“不修改 data”，返回集群现有 data 的副本； +// requested 为非 nil 的空 map（请求携带 "data": {}）时视为显式清空，返回空 map，集群中的 data 键全部删除。 +func preserveRedactedData(requested, existing map[string][]byte) map[string][]byte { + if requested == nil { + if existing == nil { + return nil + } + kept := make(map[string][]byte, len(existing)) + for k, v := range existing { + kept[k] = append([]byte(nil), v...) + } + return kept + } + out := make(map[string][]byte, len(requested)) + for k, v := range requested { + if len(v) == 0 { + if old, ok := existing[k]; ok && len(old) > 0 { + out[k] = old + continue + } + } + out[k] = v + } + return out +} + +// preserveRedactedStringData 与 data 相同按“空值保留原值”处理 stringData。 +// K8s 写入时 stringData 会覆盖 data 中的同名键，若把脱敏回填的空字符串原样提交，会把 data 中的真实值清空。 +// 因此：stringData 中值为空、而集群中该键已有非空值（data 优先，其次 stringData）时， +// 从 stringData 中移除该键，并在 data 中保留原值（请求 data 对同名键给了非空新值时以请求为准）。 +// 集群中不存在或原值为空的键按原样写入。返回处理后的 data 与 stringData。 +// +// SEC-25 P3-1：只有请求的 data 同样“包含”该键时才恢复原值——requestedData 为 nil（未携带 data，整体保留） +// 或 requestedData 中有该键。请求 data 非 nil 且不含该键时，表示该键已被用户删除：从 stringData 中移除， +// 也不写回 data，更新后该键被删除（不会因为 stringData 中残留的空串被恢复，也不会被写成空值）。 +func preserveRedactedStringData(requested map[string]string, requestedData, data map[string][]byte, existing *corev1.Secret) (map[string][]byte, map[string]string) { + if requested == nil { + return data, nil + } + var existingData map[string][]byte + var existingStringData map[string]string + if existing != nil { + existingData, existingStringData = existing.Data, existing.StringData + } + out := make(map[string]string, len(requested)) + for k, v := range requested { + if v == "" { + old, ok := existingData[k] + if !ok || len(old) == 0 { + if s, sok := existingStringData[k]; sok && s != "" { + old, ok = []byte(s), true + } + } + if ok && len(old) > 0 { + if _, inRequest := requestedData[k]; requestedData != nil && !inRequest { + delete(data, k) + continue + } + if data == nil { + data = map[string][]byte{} + } + if cur, set := data[k]; !set || len(cur) == 0 { + data[k] = old + } + continue + } + } + out[k] = v + } + return data, out +} + +// preserveLastApplied 列表/详情不再返回 last-applied-configuration，更新时若请求未携带则保留集群中的原注解 +func preserveLastApplied(requested, existing map[string]string) map[string]string { + old, ok := existing[k8sutils.LastAppliedConfigAnnotation] + if !ok { + return requested + } + if _, provided := requested[k8sutils.LastAppliedConfigAnnotation]; provided { + return requested + } + out := make(map[string]string, len(requested)+1) + for k, v := range requested { + out[k] = v + } + out[k8sutils.LastAppliedConfigAnnotation] = old + return out +} + +// RevealSecret 返回 Secret 明文。每次调用都写审计，审计只含操作者、集群、命名空间、名称与键名，绝不含值。 +// 成功读取后若审计写入失败，则不返回明文（fail closed）。 +func (s *secretService) RevealSecret(ctx context.Context, req *model.RevealSecretReq, actor model.SecretRevealActor) (*model.K8sSecretReveal, error) { + if req == nil { + return nil, fmt.Errorf("请求参数不能为空") + } + start := time.Now() + + secret, err := s.secretManager.GetSecret(ctx, req.ClusterID, req.Namespace, req.Name) + if err != nil { + var userErr error + if errors.IsNotFound(err) { + userErr = fmt.Errorf("Secret不存在: %s/%s", req.Namespace, req.Name) + } else { + s.logger.Error("获取Secret失败", zap.Error(err), + zap.Int("cluster_id", req.ClusterID), + zap.String("namespace", req.Namespace), + zap.String("name", req.Name)) + userErr = fmt.Errorf("获取Secret失败: %w", err) + } + _ = s.writeRevealAudit(ctx, req, actor, normalizeKeys(req.Keys), start, k8sErrorStatusCode(err), userErr) + return nil, userErr + } + + result := &model.K8sSecretReveal{ + ClusterID: req.ClusterID, + Namespace: secret.Namespace, + Name: secret.Name, + Type: model.K8sSecretType(secret.Type), + } + + wanted := normalizeKeys(req.Keys) + if len(wanted) == 0 { + result.Data = model.BinaryDataMap(secret.Data) + result.StringData = secret.StringData + } else { + result.Data = model.BinaryDataMap{} + for _, k := range wanted { + found := false + if v, ok := secret.Data[k]; ok { + result.Data[k] = v + found = true + } + // data 与 stringData 存在同名键时两者都返回，与不指定 keys 时的输出保持一致 + if v, ok := secret.StringData[k]; ok { + if result.StringData == nil { + result.StringData = map[string]string{} + } + result.StringData[k] = v + found = true + } + if found { + continue + } + keyErr := fmt.Errorf("Secret %s/%s 中不存在键 %s", req.Namespace, req.Name, k) + _ = s.writeRevealAudit(ctx, req, actor, wanted, start, http.StatusBadRequest, keyErr) + return nil, keyErr + } + } + + keySet := make(map[string]struct{}, len(result.Data)+len(result.StringData)) + for k := range result.Data { + keySet[k] = struct{}{} + } + for k := range result.StringData { + keySet[k] = struct{}{} + } + result.Keys = make([]string, 0, len(keySet)) + for k := range keySet { + result.Keys = append(result.Keys, k) + } + sort.Strings(result.Keys) + + if err := s.writeRevealAudit(ctx, req, actor, result.Keys, start, http.StatusOK, nil); err != nil { + s.logger.Error("写入Secret明文查看审计失败，拒绝返回明文", zap.Error(err), + zap.Int("cluster_id", req.ClusterID), + zap.String("namespace", req.Namespace), + zap.String("name", req.Name)) + return nil, fmt.Errorf("%w，已拒绝返回Secret明文", ErrAuditWriteFailed) + } + + s.logger.Info("Secret明文已查看", + zap.Int("cluster_id", req.ClusterID), + zap.String("namespace", req.Namespace), + zap.String("name", req.Name), + zap.Int("user_id", actor.UserID), + zap.Strings("keys", result.Keys)) + + return result, nil +} + +// normalizeKeys 去重、去空并排序请求中的键，同时支持逗号分隔 +func normalizeKeys(keys []string) []string { + seen := make(map[string]struct{}, len(keys)) + out := make([]string, 0, len(keys)) + for _, raw := range keys { + for _, k := range strings.Split(raw, ",") { + k = strings.TrimSpace(k) + if k == "" { + continue + } + if _, ok := seen[k]; ok { + continue + } + seen[k] = struct{}{} + out = append(out, k) + } + } + sort.Strings(out) + return out +} + +// ErrAuditWriteFailed 敏感读取接口（Secret reveal 等）同步审计写入失败。 +// handler 据此返回 HTTP 500，且不返回任何明文。 +var ErrAuditWriteFailed = stderrors.New("写入审计失败") + +// k8sErrorStatusCode 将读取 K8s 资源的错误映射为审计记录中的 HTTP 状态码： +// 优先使用 K8s API 返回的状态码（404/403 等），无法识别时记为 500。 +func k8sErrorStatusCode(err error) int { + if err == nil { + return http.StatusOK + } + var status errors.APIStatus + if stderrors.As(err, &status) { + if code := int(status.Status().Code); code >= 400 && code <= 599 { + return code + } + } + if errors.IsNotFound(err) { + return http.StatusNotFound + } + return http.StatusInternalServerError +} + +// writeRevealAudit 同步写入 reveal 审计；内容只含元数据与键名。statusCode 为本次操作实际的结果码（成功 200，失败为 4xx/5xx） +func (s *secretService) writeRevealAudit(ctx context.Context, req *model.RevealSecretReq, actor model.SecretRevealActor, keys []string, start time.Time, statusCode int, opErr error) error { + if s.auditService == nil { + return fmt.Errorf("审计服务未初始化") + } + if keys == nil { + keys = []string{} + } + body, err := json.Marshal(model.SecretRevealAuditRecord{ + Action: model.SecretRevealAuditAction, + UserID: actor.UserID, + Username: actor.Username, + ClusterID: req.ClusterID, + Namespace: req.Namespace, + Name: req.Name, + Keys: keys, + Success: opErr == nil, + RevealedAt: start.UTC().Format(time.RFC3339), + }) + if err != nil { + return err + } + errMsg := "" + if opErr != nil { + errMsg = opErr.Error() + } + return s.auditService.CreateAuditLog(ctx, &model.CreateAuditLogRequest{ + UserID: actor.UserID, + TraceID: actor.TraceID, + IPAddress: actor.IP, + UserAgent: actor.UserAgent, + HttpMethod: "GET", + Endpoint: actor.Endpoint, + OperationType: model.SecretRevealAuditOperation, + TargetType: model.SecretRevealAuditTargetType, + TargetID: fmt.Sprintf("%d/%s/%s", req.ClusterID, req.Namespace, req.Name), + StatusCode: statusCode, + RequestBody: body, + Duration: time.Since(start).Microseconds(), + ErrorMsg: errMsg, + }) +} diff --git a/internal/k8s/service/secret_service_test.go b/internal/k8s/service/secret_service_test.go new file mode 100644 index 00000000..3dfabe3d --- /dev/null +++ b/internal/k8s/service/secret_service_test.go @@ -0,0 +1,584 @@ +package service + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/manager" + k8sutils "github.com/GoSimplicity/AI-CloudOps/internal/k8s/utils" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + systemService "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/yaml" +) + +// 测试用的明文（非真实凭据），用于断言任何脱敏输出中都不出现 +const ( + fixtureAlpha = "plain-alpha-fixture-001" + fixtureBeta = "plain-beta-fixture-0002" + fixtureApplied = "plain-applied-fixture-03" +) + +type secretManagerFixture struct { + manager.SecretManager + secret *corev1.Secret + getErr error + updated *corev1.Secret + created *corev1.Secret +} + +func (m *secretManagerFixture) GetSecret(_ context.Context, _ int, _, _ string) (*corev1.Secret, error) { + if m.getErr != nil { + return nil, m.getErr + } + return m.secret.DeepCopy(), nil +} + +func (m *secretManagerFixture) ListSecrets(_ context.Context, _ int, _ string) (*corev1.SecretList, error) { + return &corev1.SecretList{Items: []corev1.Secret{*m.secret.DeepCopy()}}, nil +} + +func (m *secretManagerFixture) UpdateSecret(_ context.Context, _ int, secret *corev1.Secret) (*corev1.Secret, error) { + m.updated = secret.DeepCopy() + return secret, nil +} + +func (m *secretManagerFixture) CreateSecret(_ context.Context, _ int, secret *corev1.Secret) (*corev1.Secret, error) { + m.created = secret.DeepCopy() + return secret, nil +} + +type auditFixture struct { + systemService.AuditService + logs []*model.CreateAuditLogRequest + err error +} + +func (a *auditFixture) CreateAuditLog(_ context.Context, req *model.CreateAuditLogRequest) error { + if a.err != nil { + return a.err + } + cp := *req + a.logs = append(a.logs, &cp) + return nil +} + +func newSecretFixture() *corev1.Secret { + applied, _ := json.Marshal(map[string]interface{}{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]interface{}{"name": "app", "namespace": "ops"}, + "data": map[string]string{"alpha": base64.StdEncoding.EncodeToString([]byte(fixtureApplied))}, + "stringData": map[string]string{"beta": fixtureApplied}, + }) + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "app", + Namespace: "ops", + UID: "uid-1", + ResourceVersion: "42", + CreationTimestamp: metav1.NewTime(time.Unix(100, 0)), + Labels: map[string]string{"app": "demo"}, + Annotations: map[string]string{ + k8sutils.LastAppliedConfigAnnotation: string(applied), + "owner": "team-a", + }, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{"alpha": []byte(fixtureAlpha), "beta": []byte(fixtureBeta)}, + StringData: map[string]string{"gamma": fixtureBeta}, + } +} + +// assertNoPlaintext 断言输出中不含任何明文及其 base64 形式 +func assertNoPlaintext(t *testing.T, where, out string) { + t.Helper() + for _, v := range []string{fixtureAlpha, fixtureBeta, fixtureApplied} { + if strings.Contains(out, v) || strings.Contains(out, base64.StdEncoding.EncodeToString([]byte(v))) { + t.Fatalf("%s 泄露了明文值", where) + } + } + if strings.Contains(out, k8sutils.LastAppliedConfigAnnotation) { + t.Fatalf("%s 仍包含 last-applied-configuration 注解", where) + } +} + +func newTestSecretService(m *secretManagerFixture, a *auditFixture) SecretService { + return NewSecretService(m, a, zap.NewNop()) +} + +func TestSecretListAndDetailAreRedacted(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{}) + + list, err := svc.GetSecretList(context.Background(), &model.GetSecretListReq{ClusterID: 1, Namespace: "ops"}) + if err != nil || len(list.Items) != 1 { + t.Fatalf("list err=%v items=%d", err, len(list.Items)) + } + detail, err := svc.GetSecret(context.Background(), &model.GetSecretDetailsReq{ClusterID: 1, Namespace: "ops", Name: "app"}) + if err != nil { + t.Fatal(err) + } + + for where, item := range map[string]*model.K8sSecret{"list": list.Items[0], "detail": detail} { + raw, err := json.Marshal(item) + if err != nil { + t.Fatal(err) + } + assertNoPlaintext(t, where, string(raw)) + + if !item.Redacted { + t.Fatalf("%s 未标记 redacted", where) + } + if len(item.Data) != 2 || len(item.Data["alpha"]) != 0 || len(item.Data["beta"]) != 0 { + t.Fatalf("%s data 应只保留键且值为空: %v", where, item.Data) + } + if v, ok := item.StringData["gamma"]; !ok || v != "" { + t.Fatalf("%s string_data 应只保留键", where) + } + wantKeys := []model.SecretKeyInfo{{Key: "alpha", Size: len(fixtureAlpha)}, {Key: "beta", Size: len(fixtureBeta)}} + if len(item.DataKeys) != 2 || item.DataKeys[0] != wantKeys[0] || item.DataKeys[1] != wantKeys[1] { + t.Fatalf("%s data_keys 不符: %+v", where, item.DataKeys) + } + if len(item.StringDataKeys) != 1 || item.StringDataKeys[0] != (model.SecretKeyInfo{Key: "gamma", Size: len(fixtureBeta)}) { + t.Fatalf("%s string_data_keys 不符: %+v", where, item.StringDataKeys) + } + if item.Annotations["owner"] != "team-a" { + t.Fatalf("%s 普通注解应保留", where) + } + if item.DataCount != 3 { + t.Fatalf("%s data_count=%d", where, item.DataCount) + } + } + + // 原始对象不被修改 + if string(m.secret.Data["alpha"]) != fixtureAlpha { + t.Fatal("脱敏不应修改原始对象") + } +} + +func TestSecretYAMLIsRedacted(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{}) + out, err := svc.GetSecretYAML(context.Background(), &model.GetSecretYamlReq{ClusterID: 1, Namespace: "ops", Name: "app"}) + if err != nil { + t.Fatal(err) + } + assertNoPlaintext(t, "yaml", out.YAML) + + var obj struct { + Data map[string]string `json:"data"` + StringData map[string]string `json:"stringData"` + Metadata struct { + Annotations map[string]string `json:"annotations"` + } `json:"metadata"` + } + if err := yaml.Unmarshal([]byte(out.YAML), &obj); err != nil { + t.Fatal(err) + } + if obj.Data["alpha"] != k8sutils.SecretRedactedPlaceholder || obj.Data["beta"] != k8sutils.SecretRedactedPlaceholder || obj.StringData["gamma"] != k8sutils.SecretRedactedPlaceholder { + t.Fatalf("data/stringData 应替换为占位符: %+v", obj) + } + if obj.Metadata.Annotations["owner"] != "team-a" { + t.Fatal("普通注解应保留") + } +} + +func TestRevealSecretReturnsValuesAndAuditsWithoutValues(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + a := &auditFixture{} + svc := newTestSecretService(m, a) + actor := model.SecretRevealActor{UserID: 7, Username: "alice", IP: "10.0.0.1", UserAgent: "ua", Endpoint: "/api/k8s/secret/1/ops/app/reveal"} + + out, err := svc.RevealSecret(context.Background(), &model.RevealSecretReq{ClusterID: 1, Namespace: "ops", Name: "app"}, actor) + if err != nil { + t.Fatal(err) + } + if string(out.Data["alpha"]) != fixtureAlpha || string(out.Data["beta"]) != fixtureBeta || out.StringData["gamma"] != fixtureBeta { + t.Fatal("reveal 未返回明文") + } + raw, _ := json.Marshal(out) + if !strings.Contains(string(raw), base64.StdEncoding.EncodeToString([]byte(fixtureAlpha))) { + t.Fatal("reveal data 应保持 base64 编码") + } + if strings.Join(out.Keys, ",") != "alpha,beta,gamma" { + t.Fatalf("keys=%v", out.Keys) + } + + if len(a.logs) != 1 { + t.Fatalf("应写 1 条审计，实际 %d", len(a.logs)) + } + log := a.logs[0] + auditRaw, _ := json.Marshal(log) + assertNoPlaintext(t, "audit", string(auditRaw)) + if log.ResponseBody != nil { + t.Fatal("审计不应记录响应体") + } + if log.UserID != 7 || log.OperationType != "REVEAL" || log.TargetType != "k8s_secret" || log.TargetID != "1/ops/app" || log.ErrorMsg != "" || log.Endpoint != actor.Endpoint { + t.Fatalf("审计元数据不符: %+v", log) + } + var rec model.SecretRevealAuditRecord + if err := json.Unmarshal(log.RequestBody, &rec); err != nil { + t.Fatal(err) + } + if rec.Username != "alice" || rec.ClusterID != 1 || rec.Namespace != "ops" || rec.Name != "app" || strings.Join(rec.Keys, ",") != "alpha,beta,gamma" || !rec.Success || rec.RevealedAt == "" { + t.Fatalf("审计内容不符: %+v", rec) + } +} + +func TestRevealSecretSelectedKeys(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + a := &auditFixture{} + svc := newTestSecretService(m, a) + + out, err := svc.RevealSecret(context.Background(), &model.RevealSecretReq{ClusterID: 1, Namespace: "ops", Name: "app", Keys: []string{"beta", "beta, alpha"}}, model.SecretRevealActor{UserID: 1}) + if err != nil { + t.Fatal(err) + } + if len(out.Data) != 2 || out.StringData != nil || strings.Join(out.Keys, ",") != "alpha,beta" { + t.Fatalf("按键过滤失败: %+v", out.Keys) + } + + if _, err := svc.RevealSecret(context.Background(), &model.RevealSecretReq{ClusterID: 1, Namespace: "ops", Name: "app", Keys: []string{"missing"}}, model.SecretRevealActor{UserID: 1}); err == nil { + t.Fatal("不存在的键应报错") + } + if len(a.logs) != 2 || a.logs[1].ErrorMsg == "" { + t.Fatal("失败的 reveal 也应写审计") + } +} + +func TestRevealSecretFailsClosedWhenAuditFails(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{err: errors.New("db down")}) + out, err := svc.RevealSecret(context.Background(), &model.RevealSecretReq{ClusterID: 1, Namespace: "ops", Name: "app"}, model.SecretRevealActor{UserID: 1}) + if err == nil || out != nil { + t.Fatal("审计写入失败时不应返回明文") + } + if strings.Contains(err.Error(), fixtureAlpha) { + t.Fatal("错误信息泄露明文") + } +} + +func TestRevealSecretAuditsGetFailure(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture(), getErr: errors.New("forbidden")} + a := &auditFixture{} + svc := newTestSecretService(m, a) + if _, err := svc.RevealSecret(context.Background(), &model.RevealSecretReq{ClusterID: 1, Namespace: "ops", Name: "app"}, model.SecretRevealActor{UserID: 1}); err == nil { + t.Fatal("应返回错误") + } + if len(a.logs) != 1 || a.logs[0].ErrorMsg == "" { + t.Fatal("读取失败也应写审计") + } +} + +func TestUpdateSecretKeepsValuesForRedactedRoundTrip(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{}) + // 模拟前端把脱敏详情原样回填，只改了 beta 与标签 + err := svc.UpdateSecret(context.Background(), &model.UpdateSecretReq{ + ClusterID: 1, Namespace: "ops", Name: "app", + Data: model.BinaryDataMap{"alpha": []byte{}, "beta": []byte("new-beta"), "delta": []byte{}}, + Labels: map[string]string{"app": "demo2"}, + Annotations: map[string]string{"owner": "team-a"}, + }) + if err != nil { + t.Fatal(err) + } + if string(m.updated.Data["alpha"]) != fixtureAlpha || string(m.updated.Data["beta"]) != "new-beta" { + t.Fatal("空值回填应保留原值，非空值应覆盖") + } + if v, ok := m.updated.Data["delta"]; !ok || len(v) != 0 { + t.Fatal("新键空值应原样写入") + } + if m.updated.Annotations[k8sutils.LastAppliedConfigAnnotation] == "" { + t.Fatal("未携带的 last-applied 注解应保留") + } +} + +func TestUpdateSecretByYamlRestoresPlaceholders(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{}) + y, err := svc.GetSecretYAML(context.Background(), &model.GetSecretYamlReq{ClusterID: 1, Namespace: "ops", Name: "app"}) + if err != nil { + t.Fatal(err) + } + // 只改 beta + edited := strings.Replace(y.YAML, "beta: "+k8sutils.SecretRedactedPlaceholder, "beta: "+base64.StdEncoding.EncodeToString([]byte("new-beta")), 1) + if edited == y.YAML { + t.Fatal("测试前提失败：未找到 beta 占位符") + } + if err := svc.UpdateSecretByYaml(context.Background(), &model.UpdateSecretByYamlReq{ClusterID: 1, Namespace: "ops", Name: "app", YAML: edited}); err != nil { + t.Fatal(err) + } + if string(m.updated.Data["alpha"]) != fixtureAlpha || string(m.updated.Data["beta"]) != "new-beta" { + t.Fatalf("占位符应恢复原值: alpha=%d beta=%q", len(m.updated.Data["alpha"]), m.updated.Data["beta"]) + } + if m.updated.StringData["gamma"] != fixtureBeta { + t.Fatal("stringData 占位符应恢复原值") + } + + // 占位符对应的键在现有 Secret 中不存在 → 报错，不会写入占位符 + m.updated = nil + withUnknown := strings.Replace(edited, "data:\n", "data:\n omega: "+k8sutils.SecretRedactedPlaceholder+"\n", 1) + if err := svc.UpdateSecretByYaml(context.Background(), &model.UpdateSecretByYamlReq{ClusterID: 1, Namespace: "ops", Name: "app", YAML: withUnknown}); err == nil || m.updated != nil { + t.Fatal("未知键的占位符应拒绝更新") + } +} + +func TestCreateSecretByYamlRejectsPlaceholder(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{}) + y := "apiVersion: v1\nkind: Secret\nmetadata:\n name: x\n namespace: ops\ndata:\n alpha: " + k8sutils.SecretRedactedPlaceholder + "\n" + if err := svc.CreateSecretByYaml(context.Background(), &model.CreateSecretByYamlReq{ClusterID: 1, YAML: y}); err == nil || m.created != nil { + t.Fatal("含占位符的 YAML 不应被创建") + } +} + +// JSON 更新：stringData 中值为空的键与 data 一样保留原值，不能借 K8s 的 stringData→data 合并把真实值清空 +func TestUpdateSecretKeepsValuesForEmptyStringData(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{}) + err := svc.UpdateSecret(context.Background(), &model.UpdateSecretReq{ + ClusterID: 1, Namespace: "ops", Name: "app", + // alpha 只在 stringData 中以空值出现（请求 data 不含）；beta 在 data 中给新值、stringData 中为空；gamma 原本只在 stringData 中 + Data: model.BinaryDataMap{"beta": []byte("new-beta")}, + StringData: map[string]string{"alpha": "", "beta": "", "gamma": "", "delta": "", "eps": "new-eps"}, + }) + if err != nil { + t.Fatal(err) + } + u := m.updated + // SEC-25 P3-1：alpha/gamma 不在请求 data 中，视为已删除，不因 stringData 空串被恢复 + for _, k := range []string{"alpha", "gamma"} { + if _, ok := u.Data[k]; ok { + t.Fatalf("请求 data 不含 %s 时不应恢复原值", k) + } + } + if string(u.Data["beta"]) != "new-beta" { + t.Fatal("data 中的新值不应被原值覆盖") + } + for _, k := range []string{"alpha", "beta", "gamma"} { + if _, ok := u.StringData[k]; ok { + t.Fatalf("保留原值的键 %s 应从 stringData 中移除，否则写入时会覆盖 data", k) + } + } + if v, ok := u.StringData["delta"]; !ok || v != "" { + t.Fatal("集群中不存在的新键空值应原样写入") + } + if u.StringData["eps"] != "new-eps" { + t.Fatal("非空 stringData 应原样写入") + } +} + +func TestUpdateSecretEmptyStringDataWithNilData(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{}) + if err := svc.UpdateSecret(context.Background(), &model.UpdateSecretReq{ + ClusterID: 1, Namespace: "ops", Name: "app", + StringData: map[string]string{"alpha": ""}, + }); err != nil { + t.Fatal(err) + } + if string(m.updated.Data["alpha"]) != fixtureAlpha || len(m.updated.StringData) != 0 { + t.Fatal("data 为空时 stringData 空值也应保留原值") + } +} + +// 审计记录中的 StatusCode 为实际结果：成功 200，失败为 4xx/5xx +func TestRevealSecretAuditStatusCodes(t *testing.T) { + ctx := context.Background() + req := &model.RevealSecretReq{ClusterID: 1, Namespace: "ops", Name: "app"} + for _, tt := range []struct { + name string + getErr error + keys []string + want int + }{ + {"success", nil, nil, 200}, + {"missing key", nil, []string{"missing"}, 400}, + {"not found", apierrors.NewNotFound(schema.GroupResource{Resource: "secrets"}, "app"), nil, 404}, + {"k8s forbidden", apierrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, "app", errors.New("rbac")), nil, 403}, + {"unknown error", errors.New("connection refused"), nil, 500}, + } { + t.Run(tt.name, func(t *testing.T) { + a := &auditFixture{} + svc := newTestSecretService(&secretManagerFixture{secret: newSecretFixture(), getErr: tt.getErr}, a) + r := *req + r.Keys = tt.keys + _, _ = svc.RevealSecret(ctx, &r, model.SecretRevealActor{UserID: 1}) + if len(a.logs) != 1 || a.logs[0].StatusCode != tt.want { + t.Fatalf("审计 StatusCode 不符: %+v", a.logs) + } + }) + } +} + +func TestRevealSecretAuditFailureIsTyped(t *testing.T) { + svc := newTestSecretService(&secretManagerFixture{secret: newSecretFixture()}, &auditFixture{err: errors.New("db down")}) + _, err := svc.RevealSecret(context.Background(), &model.RevealSecretReq{ClusterID: 1, Namespace: "ops", Name: "app"}, model.SecretRevealActor{UserID: 1}) + if !errors.Is(err, ErrAuditWriteFailed) { + t.Fatalf("审计失败应返回 ErrAuditWriteFailed, got %v", err) + } +} + +// data 与 stringData 存在同名键：脱敏、回写与 reveal 都要正确 +func TestSecretSameKeyInDataAndStringData(t *testing.T) { + newSame := func() *corev1.Secret { + s := newSecretFixture() + s.Data = map[string][]byte{"shared": []byte(fixtureAlpha)} + s.StringData = map[string]string{"shared": fixtureBeta} + return s + } + ctx := context.Background() + + t.Run("detail 脱敏", func(t *testing.T) { + svc := newTestSecretService(&secretManagerFixture{secret: newSame()}, &auditFixture{}) + d, err := svc.GetSecret(ctx, &model.GetSecretDetailsReq{ClusterID: 1, Namespace: "ops", Name: "app"}) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(d) + assertNoPlaintext(t, "detail", string(raw)) + if v, ok := d.Data["shared"]; !ok || len(v) != 0 { + t.Fatal("data 同名键应保留且值为空") + } + if v, ok := d.StringData["shared"]; !ok || v != "" { + t.Fatal("stringData 同名键应保留且值为空") + } + if len(d.DataKeys) != 1 || d.DataKeys[0].Size != len(fixtureAlpha) || len(d.StringDataKeys) != 1 || d.StringDataKeys[0].Size != len(fixtureBeta) { + t.Fatal("同名键的 size 应各自统计") + } + y, err := svc.GetSecretYAML(ctx, &model.GetSecretYamlReq{ClusterID: 1, Namespace: "ops", Name: "app"}) + if err != nil { + t.Fatal(err) + } + assertNoPlaintext(t, "yaml", y.YAML) + }) + + t.Run("JSON 原样回填保留原值", func(t *testing.T) { + m := &secretManagerFixture{secret: newSame()} + svc := newTestSecretService(m, &auditFixture{}) + if err := svc.UpdateSecret(ctx, &model.UpdateSecretReq{ClusterID: 1, Namespace: "ops", Name: "app", + Data: model.BinaryDataMap{"shared": []byte{}}, StringData: map[string]string{"shared": ""}}); err != nil { + t.Fatal(err) + } + if string(m.updated.Data["shared"]) != fixtureAlpha { + t.Fatal("同名键原样回填后 data 应保留原值") + } + if _, ok := m.updated.StringData["shared"]; ok { + t.Fatal("stringData 空值不应提交，否则会覆盖 data") + } + }) + + t.Run("YAML 原样回写保留原值", func(t *testing.T) { + m := &secretManagerFixture{secret: newSame()} + svc := newTestSecretService(m, &auditFixture{}) + y, _ := svc.GetSecretYAML(ctx, &model.GetSecretYamlReq{ClusterID: 1, Namespace: "ops", Name: "app"}) + if err := svc.UpdateSecretByYaml(ctx, &model.UpdateSecretByYamlReq{ClusterID: 1, Namespace: "ops", Name: "app", YAML: y.YAML}); err != nil { + t.Fatal(err) + } + if string(m.updated.Data["shared"]) != fixtureAlpha || m.updated.StringData["shared"] != fixtureBeta { + t.Fatal("同名键 YAML 回写后应各自恢复原值") + } + }) + + t.Run("reveal 全量与按键返回两处的值", func(t *testing.T) { + a := &auditFixture{} + svc := newTestSecretService(&secretManagerFixture{secret: newSame()}, a) + for _, keys := range [][]string{nil, {"shared"}} { + out, err := svc.RevealSecret(ctx, &model.RevealSecretReq{ClusterID: 1, Namespace: "ops", Name: "app", Keys: keys}, model.SecretRevealActor{UserID: 1}) + if err != nil { + t.Fatal(err) + } + if string(out.Data["shared"]) != fixtureAlpha || out.StringData["shared"] != fixtureBeta { + t.Fatalf("keys=%v 时应同时返回 data 与 stringData 的值", keys) + } + if strings.Join(out.Keys, ",") != "shared" { + t.Fatalf("同名键在 keys 中只应出现一次: %v", out.Keys) + } + } + raw, _ := json.Marshal(a.logs) + assertNoPlaintext(t, "audit", string(raw)) + }) +} + +// SEC-25 P3-1：stringData 空串只在请求 data 同样包含该键时恢复原值；data 不含该键时该键被删除 +func TestUpdateSecretEmptyStringDataRestoresOnlyKeysInRequestData(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{}) + if err := svc.UpdateSecret(context.Background(), &model.UpdateSecretReq{ + ClusterID: 1, Namespace: "ops", Name: "app", + // alpha 在 data 中（脱敏回填的空值）→ 恢复；beta 不在 data 中 → 删除 + Data: model.BinaryDataMap{"alpha": []byte{}}, + StringData: map[string]string{"alpha": "", "beta": ""}, + }); err != nil { + t.Fatal(err) + } + u := m.updated + if string(u.Data["alpha"]) != fixtureAlpha { + t.Fatal("请求 data 含该键时 stringData 空串应恢复原值") + } + if _, ok := u.Data["beta"]; ok { + t.Fatal("请求 data 不含该键时应删除，不应恢复原值") + } + if len(u.StringData) != 0 { + t.Fatalf("恢复或删除的键都不应留在 stringData 中（否则写入时会覆盖/写空 data）: %v", u.StringData) + } +} + +// SEC-25 P3-2：请求未携带 data（nil）时保留集群现有 data；携带空对象 {} 时删除全部 data 键 +func TestUpdateSecretNilDataKeepsExistingEmptyDataClears(t *testing.T) { + m := &secretManagerFixture{secret: newSecretFixture()} + svc := newTestSecretService(m, &auditFixture{}) + if err := svc.UpdateSecret(context.Background(), &model.UpdateSecretReq{ + ClusterID: 1, Namespace: "ops", Name: "app", Labels: map[string]string{"app": "demo2"}, + }); err != nil { + t.Fatal(err) + } + if string(m.updated.Data["alpha"]) != fixtureAlpha || string(m.updated.Data["beta"]) != fixtureBeta || len(m.updated.Data) != 2 { + t.Fatalf("data 为 nil 时应保留集群现有 data: %v", keysOf(m.updated.Data)) + } + if m.updated.Labels["app"] != "demo2" { + t.Fatal("其他字段应正常更新") + } + + // JSON 层面：缺省 data 与 "data": null 均为 nil，"data": {} 为非 nil 空 map + for _, body := range []string{`{}`, `{"data":null}`} { + var req model.UpdateSecretReq + if err := json.Unmarshal([]byte(body), &req); err != nil || req.Data != nil { + t.Fatalf("%s 应解析为 nil data: %v %v", body, req.Data, err) + } + } + var empty model.UpdateSecretReq + if err := json.Unmarshal([]byte(`{"data":{}}`), &empty); err != nil || empty.Data == nil || len(empty.Data) != 0 { + t.Fatalf(`{"data":{}} 应解析为非 nil 空 map: %v %v`, empty.Data, err) + } + + m = &secretManagerFixture{secret: newSecretFixture()} + svc = newTestSecretService(m, &auditFixture{}) + if err := svc.UpdateSecret(context.Background(), &model.UpdateSecretReq{ + ClusterID: 1, Namespace: "ops", Name: "app", Data: empty.Data, + }); err != nil { + t.Fatal(err) + } + if m.updated.Data == nil || len(m.updated.Data) != 0 { + t.Fatalf("data 为 {} 时应删除全部 data 键: %v", keysOf(m.updated.Data)) + } +} + +func keysOf(m map[string][]byte) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/k8s/service/serviceaccount_service.go b/internal/k8s/service/serviceaccount_service.go index 501f4b58..8fac2148 100644 --- a/internal/k8s/service/serviceaccount_service.go +++ b/internal/k8s/service/serviceaccount_service.go @@ -27,14 +27,22 @@ package service import ( "context" + "encoding/json" "fmt" + "net/http" + "slices" "time" + "github.com/GoSimplicity/AI-CloudOps/internal/constants" "github.com/GoSimplicity/AI-CloudOps/internal/k8s/manager" k8sutils "github.com/GoSimplicity/AI-CloudOps/internal/k8s/utils" "github.com/GoSimplicity/AI-CloudOps/internal/model" + systemService "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + "github.com/GoSimplicity/AI-CloudOps/pkg/base" "go.uber.org/zap" authv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -49,18 +57,22 @@ type ServiceAccountService interface { GetServiceAccountYaml(ctx context.Context, req *model.GetServiceAccountYamlReq) (*model.K8sYaml, error) UpdateServiceAccountYaml(ctx context.Context, req *model.UpdateServiceAccountByYamlReq) error - GetServiceAccountToken(ctx context.Context, req *model.GetServiceAccountTokenReq) (*model.ServiceAccountTokenInfo, error) - CreateServiceAccountToken(ctx context.Context, req *model.CreateServiceAccountTokenReq) (*model.ServiceAccountTokenInfo, error) + // GetServiceAccountToken 返回属于该 SA 的 token Secret 中的 token；每次调用同步写审计（不含 token），审计失败不返回 token + GetServiceAccountToken(ctx context.Context, req *model.GetServiceAccountTokenReq, actor model.SecretRevealActor) (*model.ServiceAccountTokenInfo, error) + // CreateServiceAccountToken 通过 TokenRequest 签发 token；审计规则同 GetServiceAccountToken + CreateServiceAccountToken(ctx context.Context, req *model.CreateServiceAccountTokenReq, actor model.SecretRevealActor) (*model.ServiceAccountTokenInfo, error) } type serviceAccountService struct { serviceAccountManager manager.ServiceAccountManager + auditService systemService.AuditService logger *zap.Logger } -func NewServiceAccountService(serviceAccountManager manager.ServiceAccountManager, logger *zap.Logger) ServiceAccountService { +func NewServiceAccountService(serviceAccountManager manager.ServiceAccountManager, auditService systemService.AuditService, logger *zap.Logger) ServiceAccountService { return &serviceAccountService{ serviceAccountManager: serviceAccountManager, + auditService: auditService, logger: logger, } } @@ -129,6 +141,8 @@ func (s *serviceAccountService) GetServiceAccountDetails(ctx context.Context, re func (s *serviceAccountService) CreateServiceAccount(ctx context.Context, req *model.CreateServiceAccountReq) error { sa := k8sutils.ConvertToK8sServiceAccount(req) + // SEC-29：secrets 引用由集群（token controller）维护，用户提交的引用一律忽略，防止借挂其他 SA 的 token Secret + s.dropUserSecretRefs(sa, req.ClusterID, "create") return s.serviceAccountManager.CreateServiceAccount(ctx, req.ClusterID, req.Namespace, sa) } @@ -141,10 +155,59 @@ func (s *serviceAccountService) UpdateServiceAccount(ctx context.Context, req *m Annotations: req.Annotations, AutomountServiceAccountToken: req.AutomountServiceAccountToken, ImagePullSecrets: req.ImagePullSecrets, - Secrets: req.Secrets, } - sa := k8sutils.ConvertToK8sServiceAccount(createReq) - return s.serviceAccountManager.UpdateServiceAccount(ctx, req.ClusterID, req.Namespace, sa) + desired := k8sutils.ConvertToK8sServiceAccount(createReq) + return s.updateWithRetry(ctx, req.ClusterID, req.Namespace, req.Name, desired, req.Secrets, "update") +} + +// updateWithRetry 用 RetryOnConflict 更新 SA：每次重试先读取最新版本，只把允许修改的字段 +// （labels/annotations/automountServiceAccountToken/imagePullSecrets）应用上去， +// SEC-29：请求中的 secrets 一律忽略，保留集群现有引用；name/namespace 以 URL 为准。 +func (s *serviceAccountService) updateWithRetry(ctx context.Context, clusterID int, namespace, name string, desired *corev1.ServiceAccount, requestedSecrets []string, op string) error { + err := s.serviceAccountManager.UpdateServiceAccountWithRetry(ctx, clusterID, namespace, name, func(latest *corev1.ServiceAccount) error { + s.warnIgnoredSecretRefs(latest, clusterID, requestedSecrets, op) + k8sutils.ApplyServiceAccountEditableFields(latest, desired) + return nil + }) + if err != nil { + if errors.IsNotFound(err) { + return fmt.Errorf("ServiceAccount不存在: %s/%s", namespace, name) + } + return fmt.Errorf("更新ServiceAccount失败: %w", err) + } + return nil +} + +// dropUserSecretRefs 创建时丢弃用户提交的 secrets 引用并记录告警日志（只记录名称） +func (s *serviceAccountService) dropUserSecretRefs(sa *corev1.ServiceAccount, clusterID int, op string) { + if len(sa.Secrets) == 0 { + return + } + s.logger.Warn("忽略请求中的ServiceAccount secrets引用（SEC-29：该字段由集群维护）", + zap.String("operation", op), + zap.Int("cluster_id", clusterID), + zap.String("namespace", sa.Namespace), + zap.String("name", sa.Name), + zap.Strings("ignored_secrets", k8sutils.ServiceAccountSecretRefNames(sa.Secrets))) + sa.Secrets = nil +} + +// warnIgnoredSecretRefs 请求中的 secrets 与集群现有引用不同时记录告警日志（只记录名称）；引用本身不会被修改 +func (s *serviceAccountService) warnIgnoredSecretRefs(latest *corev1.ServiceAccount, clusterID int, requested []string, op string) { + if requested == nil { + return + } + existingNames := k8sutils.ServiceAccountSecretRefNames(latest.Secrets) + if slices.Equal(requested, existingNames) { + return + } + s.logger.Warn("忽略请求中的ServiceAccount secrets引用（SEC-29：该字段由集群维护），保留集群现有引用", + zap.String("operation", op), + zap.Int("cluster_id", clusterID), + zap.String("namespace", latest.Namespace), + zap.String("name", latest.Name), + zap.Strings("ignored_secrets", requested), + zap.Strings("kept_secrets", existingNames)) } func (s *serviceAccountService) DeleteServiceAccount(ctx context.Context, req *model.DeleteServiceAccountReq) error { @@ -175,7 +238,39 @@ func (s *serviceAccountService) UpdateServiceAccountYaml(ctx context.Context, re return err } - return s.serviceAccountManager.UpdateServiceAccount(ctx, req.ClusterID, req.Namespace, serviceAccount) + // name/namespace 以 URL 为准（YAML 中的 name/namespace 不生效）； + // SEC-29：YAML 中的 secrets 同样忽略，保留集群现有引用（从详情 YAML 原样回写时不产生变化） + var requested []string + if serviceAccount.Secrets != nil { + requested = k8sutils.ServiceAccountSecretRefNames(serviceAccount.Secrets) + } + if rv := serviceAccount.ResourceVersion; rv != "" { + // YAML 带了 metadata.resourceVersion：严格乐观锁，只 Update 一次，版本过期返回 409，不重试 + return s.updateWithResourceVersion(ctx, req.ClusterID, req.Namespace, req.Name, rv, serviceAccount, requested, "update_yaml") + } + // 未带 resourceVersion：RetryOnConflict，基于最新版本应用可编辑字段 + return s.updateWithRetry(ctx, req.ClusterID, req.Namespace, req.Name, serviceAccount, requested, "update_yaml") +} + +// updateWithResourceVersion 以调用方提交的 resourceVersion 做一次乐观锁更新；可编辑字段与 secrets 规则同 updateWithRetry。 +// 版本冲突返回 BusinessError{constants.ErrK8sResourceConflict}，handler 映射为 HTTP 409。 +func (s *serviceAccountService) updateWithResourceVersion(ctx context.Context, clusterID int, namespace, name, resourceVersion string, desired *corev1.ServiceAccount, requestedSecrets []string, op string) error { + err := s.serviceAccountManager.UpdateServiceAccountWithResourceVersion(ctx, clusterID, namespace, name, resourceVersion, func(latest *corev1.ServiceAccount) error { + s.warnIgnoredSecretRefs(latest, clusterID, requestedSecrets, op) + k8sutils.ApplyServiceAccountEditableFields(latest, desired) + return nil + }) + if err != nil { + switch { + case errors.IsNotFound(err): + return fmt.Errorf("ServiceAccount不存在: %s/%s", namespace, name) + case errors.IsConflict(err): + return base.NewBusinessError(constants.ErrK8sResourceConflict, fmt.Sprintf( + "ServiceAccount %s/%s 已被修改（提交的 resourceVersion=%s 已过期），请重新获取 YAML 后再提交", namespace, name, resourceVersion)) + } + return fmt.Errorf("更新ServiceAccount失败: %w", err) + } + return nil } func (s *serviceAccountService) CreateServiceAccountByYaml(ctx context.Context, req *model.CreateServiceAccountByYamlReq) error { @@ -192,42 +287,191 @@ func (s *serviceAccountService) CreateServiceAccountByYaml(ctx context.Context, zap.String("name", serviceAccount.Name)) } + // SEC-29：YAML 中的 secrets 引用同样忽略 + s.dropUserSecretRefs(serviceAccount, req.ClusterID, "create_yaml") + return s.serviceAccountManager.CreateServiceAccount(ctx, req.ClusterID, serviceAccount.Namespace, serviceAccount) } -func (s *serviceAccountService) GetServiceAccountToken(ctx context.Context, req *model.GetServiceAccountTokenReq) (*model.ServiceAccountTokenInfo, error) { - // Get existing tokens - this is a simplified implementation - tokens, err := s.serviceAccountManager.GetServiceAccountTokens(ctx, req.ClusterID, req.Namespace, req.Name) +// GetServiceAccountToken 只返回注解 kubernetes.io/service-account.name（及存在时的 .uid）指向该 SA 的 token Secret。 +// 每次调用同步写审计（只含元数据，绝不含 token）；成功读取后审计写入失败则不返回 token（fail closed）。 +func (s *serviceAccountService) GetServiceAccountToken(ctx context.Context, req *model.GetServiceAccountTokenReq, actor model.SecretRevealActor) (*model.ServiceAccountTokenInfo, error) { + if req == nil { + return nil, fmt.Errorf("请求参数不能为空") + } + start := time.Now() + record := model.ServiceAccountTokenAuditRecord{ + Action: model.ServiceAccountTokenRevealAuditAction, + ClusterID: req.ClusterID, + Namespace: req.Namespace, + ServiceAccount: req.Name, + } + fail := func(status int, err error) error { + _ = s.writeTokenAudit(ctx, actor, record, model.ServiceAccountTokenRevealAuditOperation, "GET", start, status, err) + return err + } + + sa, err := s.serviceAccountManager.GetServiceAccount(ctx, req.ClusterID, req.Namespace, req.Name) if err != nil { - return nil, err + return nil, fail(k8sErrorStatusCode(err), serviceAccountGetError(req.Namespace, req.Name, err)) } + record.ServiceAccountUID = string(sa.UID) - if len(tokens) == 0 { - return nil, fmt.Errorf("no tokens found for service account %s", req.Name) + // 复用上面读到的 SA（归属校验用的也是同一个对象），不再重复 Get + secrets, err := s.serviceAccountManager.GetServiceAccountSecrets(ctx, req.ClusterID, sa) + if err != nil { + return nil, fail(k8sErrorStatusCode(err), fmt.Errorf("获取ServiceAccount关联的Secret失败: %w", err)) + } + + selected, rejected := k8sutils.SelectServiceAccountTokenSecret(sa, secrets) + if len(rejected) > 0 { + s.logger.Warn("忽略不属于该ServiceAccount的token Secret（注解 service-account.name/uid 不匹配）", + zap.Int("cluster_id", req.ClusterID), + zap.String("namespace", req.Namespace), + zap.String("name", req.Name), + zap.Strings("rejected_secrets", rejected)) + } + if selected == nil { + return nil, fail(http.StatusNotFound, fmt.Errorf("ServiceAccount %s/%s 没有属于自己的token Secret", req.Namespace, req.Name)) } + record.SecretName = selected.Name + + if err := s.writeTokenAudit(ctx, actor, record, model.ServiceAccountTokenRevealAuditOperation, "GET", start, http.StatusOK, nil); err != nil { + s.logger.Error("写入ServiceAccount token查看审计失败，拒绝返回token", zap.Error(err), + zap.Int("cluster_id", req.ClusterID), + zap.String("namespace", req.Namespace), + zap.String("name", req.Name)) + return nil, fmt.Errorf("%w，已拒绝返回ServiceAccount token", ErrAuditWriteFailed) + } + + s.logger.Info("ServiceAccount token已查看", + zap.Int("cluster_id", req.ClusterID), + zap.String("namespace", req.Namespace), + zap.String("name", req.Name), + zap.String("secret_name", selected.Name), + zap.Int("user_id", actor.UserID)) - // Return the first token (simplified) return &model.ServiceAccountTokenInfo{ - Token: tokens[0], + Token: string(selected.Data[corev1.ServiceAccountTokenKey]), + CreatedAt: selected.CreationTimestamp.UTC().Format(time.RFC3339), }, nil } -func (s *serviceAccountService) CreateServiceAccountToken(ctx context.Context, req *model.CreateServiceAccountTokenReq) (*model.ServiceAccountTokenInfo, error) { - // Create token request +// CreateServiceAccountToken 通过 TokenRequest 为 SA 签发 token。审计与 fail closed 规则同 GetServiceAccountToken： +// 签发成功但审计写入失败时不返回 token（已签发的 token 无法撤销，只能等待过期，见发布文档）。 +func (s *serviceAccountService) CreateServiceAccountToken(ctx context.Context, req *model.CreateServiceAccountTokenReq, actor model.SecretRevealActor) (*model.ServiceAccountTokenInfo, error) { + if req == nil { + return nil, fmt.Errorf("请求参数不能为空") + } + start := time.Now() + record := model.ServiceAccountTokenAuditRecord{ + Action: model.ServiceAccountTokenCreateAuditAction, + ClusterID: req.ClusterID, + Namespace: req.Namespace, + ServiceAccount: req.ServiceAccountName, + ExpirationSeconds: req.ExpirationSeconds, + } + + // 有效期校验必须在调用 K8s 之前完成：越界直接拒绝（400），不签发 token，但仍写审计 + expiration, err := normalizeTokenExpiration(req.ExpirationSeconds) + if err != nil { + _ = s.writeTokenAudit(ctx, actor, record, model.ServiceAccountTokenCreateAuditOperation, "POST", start, http.StatusBadRequest, err) + return nil, err + } + record.ExpirationSeconds = &expiration + tokenRequest := &authv1.TokenRequest{ Spec: authv1.TokenRequestSpec{ - ExpirationSeconds: req.ExpirationSeconds, + ExpirationSeconds: &expiration, }, } tokenResp, err := s.serviceAccountManager.CreateServiceAccountToken(ctx, req.ClusterID, req.Namespace, req.ServiceAccountName, tokenRequest) if err != nil { - return nil, err + userErr := fmt.Errorf("创建ServiceAccount token失败: %w", err) + _ = s.writeTokenAudit(ctx, actor, record, model.ServiceAccountTokenCreateAuditOperation, "POST", start, k8sErrorStatusCode(err), userErr) + return nil, userErr + } + + if err := s.writeTokenAudit(ctx, actor, record, model.ServiceAccountTokenCreateAuditOperation, "POST", start, http.StatusOK, nil); err != nil { + s.logger.Error("写入ServiceAccount token签发审计失败，拒绝返回token", zap.Error(err), + zap.Int("cluster_id", req.ClusterID), + zap.String("namespace", req.Namespace), + zap.String("name", req.ServiceAccountName)) + return nil, fmt.Errorf("%w，已拒绝返回ServiceAccount token", ErrAuditWriteFailed) } + s.logger.Info("ServiceAccount token已签发", + zap.Int("cluster_id", req.ClusterID), + zap.String("namespace", req.Namespace), + zap.String("name", req.ServiceAccountName), + zap.Int("user_id", actor.UserID)) + + if tokenResp.Spec.ExpirationSeconds != nil { + expiration = *tokenResp.Spec.ExpirationSeconds + } + if tokenResp.Status.ExpirationTimestamp.IsZero() { + return &model.ServiceAccountTokenInfo{Token: tokenResp.Status.Token, ExpirationSeconds: &expiration}, nil + } return &model.ServiceAccountTokenInfo{ Token: tokenResp.Status.Token, - ExpirationSeconds: req.ExpirationSeconds, - ExpirationTime: tokenResp.Status.ExpirationTimestamp.Time.Format("2006-01-02T15:04:05Z"), + ExpirationSeconds: &expiration, + ExpirationTime: tokenResp.Status.ExpirationTimestamp.Time.UTC().Format("2006-01-02T15:04:05Z"), }, nil } + +// normalizeTokenExpiration 校验 TokenRequest 有效期：nil 或 0 取默认值 3600； +// 小于 600 或大于 3600（含负数）返回参数错误（BusinessError{ErrInvalidParam}，handler 映射为 HTTP 400）。 +func normalizeTokenExpiration(requested *int64) (int64, error) { + if requested == nil || *requested == 0 { + return model.ServiceAccountTokenDefaultExpirationSeconds, nil + } + v := *requested + if v < model.ServiceAccountTokenMinExpirationSeconds || v > model.ServiceAccountTokenMaxExpirationSeconds { + return 0, base.NewBusinessError(constants.ErrInvalidParam, fmt.Sprintf("expiration_seconds 取值范围为 %d~%d 秒（不传或传 0 默认 %d），当前为 %d", + model.ServiceAccountTokenMinExpirationSeconds, model.ServiceAccountTokenMaxExpirationSeconds, + model.ServiceAccountTokenDefaultExpirationSeconds, v)) + } + return v, nil +} + +func serviceAccountGetError(namespace, name string, err error) error { + if errors.IsNotFound(err) { + return fmt.Errorf("ServiceAccount不存在: %s/%s", namespace, name) + } + return fmt.Errorf("获取ServiceAccount失败: %w", err) +} + +// writeTokenAudit 同步写入 token 接口审计；RequestBody 只含元数据，绝不含 token。 +func (s *serviceAccountService) writeTokenAudit(ctx context.Context, actor model.SecretRevealActor, record model.ServiceAccountTokenAuditRecord, operation, method string, start time.Time, statusCode int, opErr error) error { + if s.auditService == nil { + return fmt.Errorf("审计服务未初始化") + } + record.UserID = actor.UserID + record.Username = actor.Username + record.Success = opErr == nil + record.IssuedAt = start.UTC().Format(time.RFC3339) + body, err := json.Marshal(record) + if err != nil { + return err + } + errMsg := "" + if opErr != nil { + errMsg = opErr.Error() + } + return s.auditService.CreateAuditLog(ctx, &model.CreateAuditLogRequest{ + UserID: actor.UserID, + TraceID: actor.TraceID, + IPAddress: actor.IP, + UserAgent: actor.UserAgent, + HttpMethod: method, + Endpoint: actor.Endpoint, + OperationType: operation, + TargetType: model.ServiceAccountTokenAuditTargetType, + TargetID: fmt.Sprintf("%d/%s/%s", record.ClusterID, record.Namespace, record.ServiceAccount), + StatusCode: statusCode, + RequestBody: body, + Duration: time.Since(start).Microseconds(), + ErrorMsg: errMsg, + }) +} diff --git a/internal/k8s/service/serviceaccount_service_test.go b/internal/k8s/service/serviceaccount_service_test.go new file mode 100644 index 00000000..d37b94e5 --- /dev/null +++ b/internal/k8s/service/serviceaccount_service_test.go @@ -0,0 +1,681 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "strings" + "testing" + "time" + + "github.com/GoSimplicity/AI-CloudOps/internal/constants" + "github.com/GoSimplicity/AI-CloudOps/internal/k8s/manager" + k8sutils "github.com/GoSimplicity/AI-CloudOps/internal/k8s/utils" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/pkg/base" + "go.uber.org/zap" + authv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/retry" +) + +// 测试用的 token（非真实凭据），用于断言审计与错误中都不出现 +const ( + fixtureOwnToken = "fake-sa-token-own-0001" + fixtureVictimToken = "fake-sa-token-victim-0002" + fixtureIssuedToken = "fake-sa-token-issued-0003" +) + +type saManagerFixture struct { + manager.ServiceAccountManager + sa *corev1.ServiceAccount + secrets []corev1.Secret + getErr error + created *corev1.ServiceAccount + updated *corev1.ServiceAccount + tokenReq *authv1.TokenRequest + tokenErr error + tokenTarget string + // kube 为 UpdateServiceAccountWithRetry 使用的 fake clientset，首次使用时以 sa 为初始数据创建； + // 测试可预先设置并挂 reactor 模拟并发冲突 + kube *fake.Clientset + // getCalls 记录 GetServiceAccount 调用次数；secretsSA 记录传给 GetServiceAccountSecrets 的 SA + getCalls int + secretsSA *corev1.ServiceAccount + // tokenExpiry 非零时作为 TokenRequest 返回的过期时间（默认 metav1.Now()） + tokenExpiry time.Time +} + +func (m *saManagerFixture) GetServiceAccount(_ context.Context, _ int, _, _ string) (*corev1.ServiceAccount, error) { + m.getCalls++ + if m.getErr != nil { + return nil, m.getErr + } + return m.sa.DeepCopy(), nil +} + +func (m *saManagerFixture) GetServiceAccountSecrets(_ context.Context, _ int, sa *corev1.ServiceAccount) ([]corev1.Secret, error) { + m.secretsSA = sa + out := make([]corev1.Secret, 0, len(m.secrets)) + for i := range m.secrets { + out = append(out, *m.secrets[i].DeepCopy()) + } + return out, nil +} + +func (m *saManagerFixture) CreateServiceAccount(_ context.Context, _ int, _ string, sa *corev1.ServiceAccount) error { + m.created = sa.DeepCopy() + return nil +} + +func (m *saManagerFixture) UpdateServiceAccount(_ context.Context, _ int, _ string, sa *corev1.ServiceAccount) error { + m.updated = sa.DeepCopy() + return nil +} + +func (m *saManagerFixture) ensureKube() { + if m.kube == nil { + var objs []runtime.Object + if m.sa != nil { + objs = append(objs, m.sa.DeepCopy()) + } + m.kube = fake.NewSimpleClientset(objs...) + } +} + +func (m *saManagerFixture) UpdateServiceAccountWithResourceVersion(ctx context.Context, _ int, namespace, name, resourceVersion string, mutate k8sutils.ServiceAccountMutateFunc) error { + m.ensureKube() + updated, err := k8sutils.UpdateServiceAccountWithResourceVersion(ctx, m.kube.CoreV1().ServiceAccounts(namespace), name, resourceVersion, mutate) + if err != nil { + return err + } + m.updated = updated.DeepCopy() + return nil +} + +func (m *saManagerFixture) UpdateServiceAccountWithRetry(ctx context.Context, _ int, namespace, name string, mutate k8sutils.ServiceAccountMutateFunc) error { + m.ensureKube() + updated, err := k8sutils.UpdateServiceAccountWithRetry(ctx, m.kube.CoreV1().ServiceAccounts(namespace), name, mutate) + if err != nil { + return err + } + m.updated = updated.DeepCopy() + return nil +} + +func (m *saManagerFixture) CreateServiceAccountToken(_ context.Context, _ int, _, name string, tr *authv1.TokenRequest) (*authv1.TokenRequest, error) { + m.tokenTarget = name + if m.tokenErr != nil { + return nil, m.tokenErr + } + m.tokenReq = tr.DeepCopy() + out := tr.DeepCopy() + out.Status.Token = fixtureIssuedToken + out.Status.ExpirationTimestamp = metav1.Now() + if !m.tokenExpiry.IsZero() { + out.Status.ExpirationTimestamp = metav1.NewTime(m.tokenExpiry) + } + return out, nil +} + +func newSAFixture() *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "ops", UID: types.UID("uid-app")}, + Secrets: []corev1.ObjectReference{{Name: "app-token"}}, + } +} + +func tokenSecret(name, saName, saUID, token string) corev1.Secret { + ann := map[string]string{corev1.ServiceAccountNameKey: saName} + if saUID != "" { + ann[corev1.ServiceAccountUIDKey] = saUID + } + return corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ops", Annotations: ann}, + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{corev1.ServiceAccountTokenKey: []byte(token)}, + } +} + +var saActor = model.SecretRevealActor{UserID: 7, Username: "operator", IP: "192.0.2.10", Endpoint: "/api/k8s/serviceaccount/1/ops/app/token"} + +func assertNoToken(t *testing.T, v interface{}, tokens ...string) { + t.Helper() + raw, _ := json.Marshal(v) + for _, tok := range tokens { + if strings.Contains(string(raw), tok) { + t.Fatalf("出现了 token: %s", raw) + } + } +} + +func TestGetServiceAccountTokenRejectsBorrowedSecret(t *testing.T) { + for _, tt := range []struct { + name string + secrets []corev1.Secret + }{ + {"借挂其他 SA 的 token Secret", []corev1.Secret{tokenSecret("admin-token", "admin", "uid-admin", fixtureVictimToken)}}, + {"名称相同但 UID 不同（同名 SA 被重建）", []corev1.Secret{tokenSecret("app-token-old", "app", "uid-old", fixtureVictimToken)}}, + {"UID 注解为空串", []corev1.Secret{tokenSecret("app-token-empty", "app", "", fixtureVictimToken)}}, + {"缺少 name 注解", []corev1.Secret{{ + ObjectMeta: metav1.ObjectMeta{Name: "no-ann", Namespace: "ops"}, + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{corev1.ServiceAccountTokenKey: []byte(fixtureVictimToken)}, + }}}, + {"非 service-account-token 类型", []corev1.Secret{func() corev1.Secret { + s := tokenSecret("opaque", "app", "uid-app", fixtureVictimToken) + s.Type = corev1.SecretTypeOpaque + return s + }()}}, + {"其他命名空间", []corev1.Secret{func() corev1.Secret { + s := tokenSecret("other-ns", "app", "uid-app", fixtureVictimToken) + s.Namespace = "kube-system" + return s + }()}}, + } { + t.Run(tt.name, func(t *testing.T) { + if tt.name == "UID 注解为空串" { + tt.secrets[0].Annotations[corev1.ServiceAccountUIDKey] = "" + } + audit := &auditFixture{} + svc := NewServiceAccountService(&saManagerFixture{sa: newSAFixture(), secrets: tt.secrets}, audit, zap.NewNop()) + got, err := svc.GetServiceAccountToken(context.Background(), &model.GetServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", Name: "app"}, saActor) + if err == nil || got != nil { + t.Fatalf("借挂的 token 不应返回: %+v", got) + } + assertNoToken(t, err.Error(), fixtureVictimToken) + if len(audit.logs) != 1 || audit.logs[0].StatusCode != 404 || audit.logs[0].OperationType != model.ServiceAccountTokenRevealAuditOperation { + t.Fatalf("拒绝也要写审计: %+v", audit.logs) + } + assertNoToken(t, audit.logs, fixtureVictimToken) + }) + } +} + +func TestGetServiceAccountTokenReturnsOwnSecretOnly(t *testing.T) { + audit := &auditFixture{} + fx := &saManagerFixture{sa: newSAFixture(), secrets: []corev1.Secret{ + tokenSecret("admin-token", "admin", "uid-admin", fixtureVictimToken), + tokenSecret("app-token", "app", "uid-app", fixtureOwnToken), + }} + svc := NewServiceAccountService(fx, audit, zap.NewNop()) + got, err := svc.GetServiceAccountToken(context.Background(), &model.GetServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", Name: "app"}, saActor) + if err != nil || got == nil || got.Token != fixtureOwnToken { + t.Fatalf("应返回 SA 自己的 token: %+v %v", got, err) + } + + // 没有 uid 注解时只校验名称 + fx.secrets = []corev1.Secret{tokenSecret("app-token", "app", "", fixtureOwnToken)} + got, err = svc.GetServiceAccountToken(context.Background(), &model.GetServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", Name: "app"}, saActor) + if err != nil || got.Token != fixtureOwnToken { + t.Fatalf("无 uid 注解时名称匹配应放行: %+v %v", got, err) + } +} + +func TestGetServiceAccountTokenAuditHasMetadataOnly(t *testing.T) { + audit := &auditFixture{} + svc := NewServiceAccountService(&saManagerFixture{sa: newSAFixture(), secrets: []corev1.Secret{tokenSecret("app-token", "app", "uid-app", fixtureOwnToken)}}, audit, zap.NewNop()) + if _, err := svc.GetServiceAccountToken(context.Background(), &model.GetServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", Name: "app"}, saActor); err != nil { + t.Fatal(err) + } + if len(audit.logs) != 1 { + t.Fatalf("应同步写 1 条审计，实际 %d", len(audit.logs)) + } + l := audit.logs[0] + if l.OperationType != "TOKEN_REVEAL" || l.TargetType != model.ServiceAccountTokenAuditTargetType || l.TargetID != "1/ops/app" || + l.StatusCode != 200 || l.HttpMethod != "GET" || l.UserID != 7 || l.IPAddress != "192.0.2.10" || len(l.ResponseBody) != 0 { + t.Fatalf("审计元数据不符: %+v", l) + } + var rec model.ServiceAccountTokenAuditRecord + if err := json.Unmarshal(l.RequestBody, &rec); err != nil { + t.Fatal(err) + } + if rec.SecretName != "app-token" || rec.ServiceAccountUID != "uid-app" || !rec.Success || rec.Username != "operator" { + t.Fatalf("审计内容不符: %s", l.RequestBody) + } + assertNoToken(t, audit.logs, fixtureOwnToken) +} + +func TestGetServiceAccountTokenAuditFailureReturnsNoToken(t *testing.T) { + svc := NewServiceAccountService(&saManagerFixture{sa: newSAFixture(), secrets: []corev1.Secret{tokenSecret("app-token", "app", "uid-app", fixtureOwnToken)}}, + &auditFixture{err: errors.New("audit db unavailable")}, zap.NewNop()) + got, err := svc.GetServiceAccountToken(context.Background(), &model.GetServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", Name: "app"}, saActor) + if got != nil || !errors.Is(err, ErrAuditWriteFailed) { + t.Fatalf("审计失败不应返回 token: %+v %v", got, err) + } + assertNoToken(t, err.Error(), fixtureOwnToken) + + // 审计服务缺失同样 fail closed + svc = NewServiceAccountService(&saManagerFixture{sa: newSAFixture(), secrets: []corev1.Secret{tokenSecret("app-token", "app", "uid-app", fixtureOwnToken)}}, nil, zap.NewNop()) + if got, err := svc.GetServiceAccountToken(context.Background(), &model.GetServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", Name: "app"}, saActor); got != nil || !errors.Is(err, ErrAuditWriteFailed) { + t.Fatalf("未注入审计服务时不应返回 token: %+v %v", got, err) + } +} + +func TestGetServiceAccountTokenSANotFoundAudited(t *testing.T) { + audit := &auditFixture{} + svc := NewServiceAccountService(&saManagerFixture{getErr: apierrors.NewNotFound(schema.GroupResource{Resource: "serviceaccounts"}, "app")}, audit, zap.NewNop()) + if _, err := svc.GetServiceAccountToken(context.Background(), &model.GetServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", Name: "app"}, saActor); err == nil { + t.Fatal("SA 不存在应报错") + } + if len(audit.logs) != 1 || audit.logs[0].StatusCode != 404 { + t.Fatalf("审计状态码应为 404: %+v", audit.logs) + } +} + +func TestCreateServiceAccountTokenAudited(t *testing.T) { + exp := int64(600) + req := &model.CreateServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", ServiceAccountName: "app", ExpirationSeconds: &exp} + + audit := &auditFixture{} + fx := &saManagerFixture{} + svc := NewServiceAccountService(fx, audit, zap.NewNop()) + got, err := svc.CreateServiceAccountToken(context.Background(), req, saActor) + if err != nil || got.Token != fixtureIssuedToken || fx.tokenTarget != "app" || *fx.tokenReq.Spec.ExpirationSeconds != 600 { + t.Fatalf("TokenRequest 结果不符: %+v %v", got, err) + } + if len(audit.logs) != 1 || audit.logs[0].OperationType != "TOKEN_CREATE" || audit.logs[0].HttpMethod != "POST" || audit.logs[0].StatusCode != 200 { + t.Fatalf("TokenRequest 审计不符: %+v", audit.logs) + } + var rec model.ServiceAccountTokenAuditRecord + if err := json.Unmarshal(audit.logs[0].RequestBody, &rec); err != nil || rec.ExpirationSeconds == nil || *rec.ExpirationSeconds != 600 { + t.Fatalf("审计内容不符: %s", audit.logs[0].RequestBody) + } + assertNoToken(t, audit.logs, fixtureIssuedToken) + + // 审计失败：已签发也不返回 + svc = NewServiceAccountService(&saManagerFixture{}, &auditFixture{err: errors.New("audit db unavailable")}, zap.NewNop()) + got, err = svc.CreateServiceAccountToken(context.Background(), req, saActor) + if got != nil || !errors.Is(err, ErrAuditWriteFailed) { + t.Fatalf("审计失败不应返回签发的 token: %+v %v", got, err) + } + assertNoToken(t, err.Error(), fixtureIssuedToken) + + // K8s 拒绝：审计记录真实状态码 + audit = &auditFixture{} + svc = NewServiceAccountService(&saManagerFixture{tokenErr: apierrors.NewForbidden(schema.GroupResource{Resource: "serviceaccounts/token"}, "app", errors.New("denied"))}, audit, zap.NewNop()) + if _, err := svc.CreateServiceAccountToken(context.Background(), req, saActor); err == nil { + t.Fatal("应返回错误") + } + if len(audit.logs) != 1 || audit.logs[0].StatusCode != 403 { + t.Fatalf("审计状态码应为 403: %+v", audit.logs) + } +} + +func TestCreateServiceAccountIgnoresSecretRefs(t *testing.T) { + fx := &saManagerFixture{} + svc := NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + err := svc.CreateServiceAccount(context.Background(), &model.CreateServiceAccountReq{ + ClusterID: 1, Namespace: "ops", Name: "app", Secrets: []string{"admin-token"}, ImagePullSecrets: []string{"registry"}, + }) + if err != nil || fx.created == nil || len(fx.created.Secrets) != 0 || len(fx.created.ImagePullSecrets) != 1 { + t.Fatalf("创建时应忽略 secrets、保留 imagePullSecrets: %+v %v", fx.created, err) + } + + fx = &saManagerFixture{} + svc = NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + yamlContent := "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: app\n namespace: ops\nsecrets:\n- name: admin-token\n" + if err := svc.CreateServiceAccountByYaml(context.Background(), &model.CreateServiceAccountByYamlReq{ClusterID: 1, YamlContent: yamlContent}); err != nil { + t.Fatal(err) + } + if fx.created == nil || len(fx.created.Secrets) != 0 { + t.Fatalf("YAML 创建时应忽略 secrets: %+v", fx.created) + } +} + +func TestUpdateServiceAccountKeepsExistingSecretRefs(t *testing.T) { + fx := &saManagerFixture{sa: newSAFixture()} + svc := NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + err := svc.UpdateServiceAccount(context.Background(), &model.UpdateServiceAccountReq{ + ClusterID: 1, Namespace: "ops", Name: "app", Secrets: []string{"admin-token"}, Labels: map[string]string{"team": "ops"}, + }) + if err != nil || len(fx.updated.Secrets) != 1 || fx.updated.Secrets[0].Name != "app-token" || fx.updated.Labels["team"] != "ops" { + t.Fatalf("更新时应忽略请求 secrets、保留集群现有引用: %+v %v", fx.updated, err) + } + + // 不传 secrets 也不能清空集群维护的引用 + fx.updated = nil + if err := svc.UpdateServiceAccount(context.Background(), &model.UpdateServiceAccountReq{ClusterID: 1, Namespace: "ops", Name: "app"}); err != nil { + t.Fatal(err) + } + if len(fx.updated.Secrets) != 1 || fx.updated.Secrets[0].Name != "app-token" { + t.Fatalf("未传 secrets 时应保留现有引用: %+v", fx.updated.Secrets) + } + + // YAML 更新：secrets 被忽略，name/namespace 以 URL 为准 + fx.updated = nil + yamlContent := "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: other\n namespace: kube-system\nsecrets:\n- name: admin-token\n" + if err := svc.UpdateServiceAccountYaml(context.Background(), &model.UpdateServiceAccountByYamlReq{ClusterID: 1, Namespace: "ops", Name: "app", YamlContent: yamlContent}); err != nil { + t.Fatal(err) + } + if fx.updated.Name != "app" || fx.updated.Namespace != "ops" || len(fx.updated.Secrets) != 1 || fx.updated.Secrets[0].Name != "app-token" { + t.Fatalf("YAML 更新应忽略 secrets 并以 URL 为准: %+v", fx.updated) + } + + // SA 不存在时不更新 + fx = &saManagerFixture{getErr: apierrors.NewNotFound(schema.GroupResource{Resource: "serviceaccounts"}, "app")} + svc = NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + if err := svc.UpdateServiceAccount(context.Background(), &model.UpdateServiceAccountReq{ClusterID: 1, Namespace: "ops", Name: "app"}); err == nil || fx.updated != nil { + t.Fatalf("SA 不存在应报错且不更新: %v", err) + } +} + +func TestCreateServiceAccountTokenExpirationBounds(t *testing.T) { + i64 := func(v int64) *int64 { return &v } + for _, tt := range []struct { + name string + requested *int64 + want int64 // 期望传给 K8s 的值；0 表示应被拒绝 + }{ + {"不传默认3600", nil, 3600}, + {"传0默认3600", i64(0), 3600}, + {"599低于下限", i64(599), 0}, + {"600下限", i64(600), 600}, + {"3600上限", i64(3600), 3600}, + {"3601高于上限", i64(3601), 0}, + {"负数", i64(-1), 0}, + {"负的600", i64(-600), 0}, + } { + t.Run(tt.name, func(t *testing.T) { + audit := &auditFixture{} + fx := &saManagerFixture{} + svc := NewServiceAccountService(fx, audit, zap.NewNop()) + got, err := svc.CreateServiceAccountToken(context.Background(), &model.CreateServiceAccountTokenReq{ + ClusterID: 1, Namespace: "ops", ServiceAccountName: "app", ExpirationSeconds: tt.requested, + }, saActor) + if len(audit.logs) != 1 { + t.Fatalf("应写且只写一条审计: %+v", audit.logs) + } + var rec model.ServiceAccountTokenAuditRecord + if err := json.Unmarshal(audit.logs[0].RequestBody, &rec); err != nil { + t.Fatal(err) + } + if tt.want == 0 { + var bizErr *base.BusinessError + if got != nil || !errors.As(err, &bizErr) || !errors.Is(bizErr.Code, constants.ErrInvalidParam) { + t.Fatalf("越界应返回参数错误且不返回 token: %+v %v", got, err) + } + if fx.tokenTarget != "" || fx.tokenReq != nil { + t.Fatalf("越界时不应调用 K8s TokenRequest: target=%q", fx.tokenTarget) + } + if audit.logs[0].StatusCode != 400 || rec.Success || rec.ExpirationSeconds == nil || *rec.ExpirationSeconds != *tt.requested { + t.Fatalf("越界审计应为 400、success=false、记录请求值: %+v %s", audit.logs[0], audit.logs[0].RequestBody) + } + assertNoToken(t, audit.logs, fixtureIssuedToken) + return + } + if err != nil || got == nil || got.Token != fixtureIssuedToken { + t.Fatalf("应签发成功: %+v %v", got, err) + } + if fx.tokenReq == nil || fx.tokenReq.Spec.ExpirationSeconds == nil || *fx.tokenReq.Spec.ExpirationSeconds != tt.want { + t.Fatalf("传给 K8s 的有效期应为 %d: %+v", tt.want, fx.tokenReq) + } + if got.ExpirationSeconds == nil || *got.ExpirationSeconds != tt.want { + t.Fatalf("响应有效期应为 %d: %+v", tt.want, got.ExpirationSeconds) + } + if audit.logs[0].StatusCode != 200 || rec.ExpirationSeconds == nil || *rec.ExpirationSeconds != tt.want { + t.Fatalf("审计应记录生效有效期 %d: %s", tt.want, audit.logs[0].RequestBody) + } + }) + } +} + +func countSAUpdates(c *fake.Clientset) int { + n := 0 + for _, a := range c.Actions() { + if a.GetVerb() == "update" && a.GetResource().Resource == "serviceaccounts" { + n++ + } + } + return n +} + +// 第一次 Update 前集群中的 SA 被并发修改并返回 Conflict（YAML 不带 resourceVersion，走 RetryOnConflict），第二次成功：JSON 与 YAML 两条路径都应基于最新版本重试、保留最新 secrets +func TestUpdateServiceAccountRetriesOnConflict(t *testing.T) { + yamlContent := "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: other\n namespace: kube-system\n labels:\n team: ops\nsecrets:\n- name: admin-token\n" + for _, tt := range []struct { + name string + update func(svc ServiceAccountService) error + }{ + {"json", func(svc ServiceAccountService) error { + return svc.UpdateServiceAccount(context.Background(), &model.UpdateServiceAccountReq{ + ClusterID: 1, Namespace: "ops", Name: "app", Secrets: []string{"admin-token"}, Labels: map[string]string{"team": "ops"}, + }) + }}, + {"yaml", func(svc ServiceAccountService) error { + return svc.UpdateServiceAccountYaml(context.Background(), &model.UpdateServiceAccountByYamlReq{ClusterID: 1, Namespace: "ops", Name: "app", YamlContent: yamlContent}) + }}, + } { + t.Run(tt.name, func(t *testing.T) { + seed := newSAFixture() + seed.ResourceVersion = "1" + kube := fake.NewSimpleClientset(seed) + calls := 0 + kube.PrependReactor("update", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + calls++ + if calls > 1 { + return false, nil, nil + } + concurrent := newSAFixture() + concurrent.ResourceVersion = "2" + concurrent.Secrets = append(concurrent.Secrets, corev1.ObjectReference{Name: "app-token-rotated"}) + if err := kube.Tracker().Update(corev1.SchemeGroupVersion.WithResource("serviceaccounts"), concurrent, "ops"); err != nil { + t.Errorf("模拟并发修改失败: %v", err) + } + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "serviceaccounts"}, "app", errors.New("object has been modified")) + }) + fx := &saManagerFixture{sa: newSAFixture(), kube: kube} + if err := tt.update(NewServiceAccountService(fx, &auditFixture{}, zap.NewNop())); err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("应冲突一次后重试成功, update 调用 %d 次", calls) + } + var last *corev1.ServiceAccount + for _, a := range kube.Actions() { + if ua, ok := a.(k8stesting.UpdateAction); ok && a.GetResource().Resource == "serviceaccounts" { + last = ua.GetObject().(*corev1.ServiceAccount) + } + } + if last == nil || last.ResourceVersion != "2" { + t.Fatalf("最后一次 Update 应基于最新版本 resourceVersion=2: %+v", last) + } + if last.Name != "app" || last.Namespace != "ops" || last.Labels["team"] != "ops" { + t.Fatalf("name/namespace 应以 URL 为准、字段应被应用: %+v", last.ObjectMeta) + } + if got := k8sutils.ServiceAccountSecretRefNames(last.Secrets); len(got) != 2 || got[0] != "app-token" || got[1] != "app-token-rotated" { + t.Fatalf("应保留集群最新的 secrets 引用、忽略请求中的 admin-token: %v", got) + } + }) + } +} + +func TestUpdateServiceAccountConflictExhaustedReturnsError(t *testing.T) { + kube := fake.NewSimpleClientset(newSAFixture()) + kube.PrependReactor("update", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "serviceaccounts"}, "app", errors.New("object has been modified")) + }) + fx := &saManagerFixture{sa: newSAFixture(), kube: kube} + svc := NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + err := svc.UpdateServiceAccount(context.Background(), &model.UpdateServiceAccountReq{ClusterID: 1, Namespace: "ops", Name: "app"}) + if err == nil || !apierrors.IsConflict(err) || fx.updated != nil { + t.Fatalf("连续冲突超过重试次数应返回 Conflict 错误: %v", err) + } + if n := countSAUpdates(kube); n != retry.DefaultRetry.Steps { + t.Fatalf("应尝试 %d 次, got %d", retry.DefaultRetry.Steps, n) + } + if err := svc.UpdateServiceAccountYaml(context.Background(), &model.UpdateServiceAccountByYamlReq{ + ClusterID: 1, Namespace: "ops", Name: "app", YamlContent: "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: app\n", + }); !apierrors.IsConflict(err) { + t.Fatalf("YAML 路径连续冲突同样应返回错误: %v", err) + } +} + +func TestUpdateServiceAccountNotFoundSkipsUpdate(t *testing.T) { + for _, op := range []string{"json", "yaml"} { + t.Run(op, func(t *testing.T) { + fx := &saManagerFixture{} + svc := NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + var err error + if op == "json" { + err = svc.UpdateServiceAccount(context.Background(), &model.UpdateServiceAccountReq{ClusterID: 1, Namespace: "ops", Name: "app"}) + } else { + err = svc.UpdateServiceAccountYaml(context.Background(), &model.UpdateServiceAccountByYamlReq{ + ClusterID: 1, Namespace: "ops", Name: "app", YamlContent: "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: app\n", + }) + } + if err == nil || !strings.Contains(err.Error(), "ServiceAccount不存在") { + t.Fatalf("SA 不存在应报错: %v", err) + } + if n := countSAUpdates(fx.kube); n != 0 || fx.updated != nil { + t.Fatalf("SA 不存在时不应调用 Update, got %d", n) + } + }) + } +} + +// optimisticLockReactor 模拟 API Server 的乐观锁：Update 提交的 resourceVersion 与当前存储不一致时返回 Conflict， +// 一致时把版本号加 1 后交给默认 tracker（fake clientset 本身不校验 resourceVersion） +func optimisticLockReactor(t *testing.T, kube *fake.Clientset) { + gvr := corev1.SchemeGroupVersion.WithResource("serviceaccounts") + kube.PrependReactor("update", "serviceaccounts", func(a k8stesting.Action) (bool, runtime.Object, error) { + obj := a.(k8stesting.UpdateAction).GetObject().(*corev1.ServiceAccount) + cur, err := kube.Tracker().Get(gvr, obj.Namespace, obj.Name) + if err != nil { + return true, nil, err + } + curRV := cur.(*corev1.ServiceAccount).ResourceVersion + if obj.ResourceVersion != curRV { + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "serviceaccounts"}, obj.Name, errors.New("the object has been modified")) + } + next := obj.DeepCopy() + n, _ := strconv.Atoi(curRV) + next.ResourceVersion = strconv.Itoa(n + 1) + if err := kube.Tracker().Update(gvr, next, obj.Namespace); err != nil { + t.Errorf("tracker 更新失败: %v", err) + } + return true, next, nil + }) +} + +func saYamlWithRV(rv string) string { + y := "apiVersion: v1\nkind: ServiceAccount\nmetadata:\n name: app\n namespace: ops\n" + if rv != "" { + y += " resourceVersion: \"" + rv + "\"\n" + } + return y + " labels:\n team: yaml\nsecrets:\n- name: admin-token\n" +} + +// SEC-29(A)：YAML 带 resourceVersion 时严格乐观锁——版本过期只 Update 一次并返回 409 业务错误，不重试 +func TestUpdateServiceAccountYamlStaleResourceVersionReturnsConflict(t *testing.T) { + seed := newSAFixture() + seed.ResourceVersion = "5" + kube := fake.NewSimpleClientset(seed) + optimisticLockReactor(t, kube) + fx := &saManagerFixture{sa: newSAFixture(), kube: kube} + svc := NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + + err := svc.UpdateServiceAccountYaml(context.Background(), &model.UpdateServiceAccountByYamlReq{ClusterID: 1, Namespace: "ops", Name: "app", YamlContent: saYamlWithRV("4")}) + var bizErr *base.BusinessError + if !errors.As(err, &bizErr) || !errors.Is(bizErr.Code, constants.ErrK8sResourceConflict) { + t.Fatalf("过期 resourceVersion 应返回 ErrK8sResourceConflict 业务错误: %v", err) + } + if n := countSAUpdates(kube); n != 1 { + t.Fatalf("严格乐观锁只应 Update 一次（不重试）, got %d", n) + } + if fx.updated != nil { + t.Fatalf("冲突时不应视为更新成功: %+v", fx.updated) + } + cur, _ := kube.CoreV1().ServiceAccounts("ops").Get(context.Background(), "app", metav1.GetOptions{}) + if cur.ResourceVersion != "5" || cur.Labels["team"] != "" { + t.Fatalf("冲突时集群对象不应被修改: %+v", cur.ObjectMeta) + } +} + +func TestUpdateServiceAccountYamlCurrentResourceVersionSucceeds(t *testing.T) { + seed := newSAFixture() + seed.ResourceVersion = "5" + kube := fake.NewSimpleClientset(seed) + optimisticLockReactor(t, kube) + fx := &saManagerFixture{sa: newSAFixture(), kube: kube} + svc := NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + + if err := svc.UpdateServiceAccountYaml(context.Background(), &model.UpdateServiceAccountByYamlReq{ClusterID: 1, Namespace: "ops", Name: "app", YamlContent: saYamlWithRV("5")}); err != nil { + t.Fatal(err) + } + if n := countSAUpdates(kube); n != 1 { + t.Fatalf("应只 Update 一次, got %d", n) + } + cur, _ := kube.CoreV1().ServiceAccounts("ops").Get(context.Background(), "app", metav1.GetOptions{}) + if cur.ResourceVersion != "6" || cur.Labels["team"] != "yaml" { + t.Fatalf("应以提交的版本更新成功并应用字段: %+v", cur.ObjectMeta) + } + if got := k8sutils.ServiceAccountSecretRefNames(cur.Secrets); len(got) != 1 || got[0] != "app-token" { + t.Fatalf("乐观锁路径同样应忽略 YAML 中的 secrets、保留集群引用: %v", got) + } +} + +// 不带 resourceVersion 时保持 RetryOnConflict：即使存储中的版本与 YAML 无关，也基于最新版本重试成功 +func TestUpdateServiceAccountYamlWithoutResourceVersionUsesRetry(t *testing.T) { + seed := newSAFixture() + seed.ResourceVersion = "5" + kube := fake.NewSimpleClientset(seed) + optimisticLockReactor(t, kube) + conflicts := 0 + kube.PrependReactor("update", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + if conflicts == 0 { + conflicts++ + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "serviceaccounts"}, "app", errors.New("the object has been modified")) + } + return false, nil, nil + }) + fx := &saManagerFixture{sa: newSAFixture(), kube: kube} + svc := NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + if err := svc.UpdateServiceAccountYaml(context.Background(), &model.UpdateServiceAccountByYamlReq{ClusterID: 1, Namespace: "ops", Name: "app", YamlContent: saYamlWithRV("")}); err != nil { + t.Fatal(err) + } + if n := countSAUpdates(kube); n != 2 { + t.Fatalf("未带 resourceVersion 应走重试路径（冲突 1 次后成功）, update %d 次", n) + } + cur, _ := kube.CoreV1().ServiceAccounts("ops").Get(context.Background(), "app", metav1.GetOptions{}) + if cur.Labels["team"] != "yaml" || cur.ResourceVersion != "6" { + t.Fatalf("重试后应基于最新版本更新成功: %+v", cur.ObjectMeta) + } +} + +// SEC-29(B)：K8s 返回非 UTC 时区的过期时间时，响应中的 expiration_time 仍是正确的 UTC 字符串 +func TestCreateServiceAccountTokenExpirationTimeIsUTC(t *testing.T) { + cst := time.FixedZone("CST", 8*3600) + fx := &saManagerFixture{tokenExpiry: time.Date(2026, 10, 24, 9, 30, 15, 0, cst)} + svc := NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + got, err := svc.CreateServiceAccountToken(context.Background(), &model.CreateServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", ServiceAccountName: "app"}, saActor) + if err != nil { + t.Fatal(err) + } + if got.ExpirationTime != "2026-10-24T01:30:15Z" { + t.Fatalf("expiration_time 应转换为 UTC: %q", got.ExpirationTime) + } +} + +// SEC-29 P3-3：GetServiceAccountToken 只读取一次 SA，并把同一个对象传给 GetServiceAccountSecrets +func TestGetServiceAccountTokenReadsServiceAccountOnce(t *testing.T) { + fx := &saManagerFixture{sa: newSAFixture(), secrets: []corev1.Secret{tokenSecret("app-token", "app", "uid-app", fixtureOwnToken)}} + svc := NewServiceAccountService(fx, &auditFixture{}, zap.NewNop()) + if _, err := svc.GetServiceAccountToken(context.Background(), &model.GetServiceAccountTokenReq{ClusterID: 1, Namespace: "ops", Name: "app"}, saActor); err != nil { + t.Fatal(err) + } + if fx.getCalls != 1 { + t.Fatalf("GetServiceAccount 应只调用一次, got %d", fx.getCalls) + } + if fx.secretsSA == nil || fx.secretsSA.Name != "app" || fx.secretsSA.UID != "uid-app" { + t.Fatalf("应把已读取的 SA 传给 GetServiceAccountSecrets: %+v", fx.secretsSA) + } +} diff --git a/internal/k8s/utils/cluster.go b/internal/k8s/utils/cluster.go index d975b34a..7b7321b9 100644 --- a/internal/k8s/utils/cluster.go +++ b/internal/k8s/utils/cluster.go @@ -40,18 +40,6 @@ import ( "k8s.io/client-go/kubernetes" ) -func CleanClusterSensitiveInfo(cluster *model.K8sCluster) { - if cluster != nil { - cluster.KubeConfigContent = "" - } -} - -func CleanClusterSensitiveInfoList(clusters []*model.K8sCluster) { - for _, cluster := range clusters { - CleanClusterSensitiveInfo(cluster) - } -} - func ValidateClusterExists(cluster *model.K8sCluster, clusterID int) error { if cluster == nil { return fmt.Errorf("集群不存在，ID: %d", clusterID) diff --git a/internal/k8s/utils/secret.go b/internal/k8s/utils/secret.go index cee9848b..333e31f5 100644 --- a/internal/k8s/utils/secret.go +++ b/internal/k8s/utils/secret.go @@ -27,6 +27,7 @@ package utils import ( "fmt" + "strings" "github.com/GoSimplicity/AI-CloudOps/internal/model" corev1 "k8s.io/api/core/v1" @@ -104,19 +105,24 @@ func CleanSecretForYAML(secret *corev1.Secret) *corev1.Secret { cleaned.ObjectMeta.ManagedFields = nil if cleaned.Annotations != nil { - delete(cleaned.Annotations, "kubectl.kubernetes.io/last-applied-configuration") + delete(cleaned.Annotations, LastAppliedConfigAnnotation) } return cleaned } -// SecretToYAML 将 Secret 转换为 YAML 字符串 +// SecretToYAML 将 Secret 转换为脱敏后的 YAML 字符串： +// data/stringData 的值替换为 SecretRedactedPlaceholder，last-applied-configuration 注解被移除。 +// 明文只能通过 reveal 接口获取。 func SecretToYAML(sec *corev1.Secret) (string, error) { if sec == nil { return "", fmt.Errorf("Secret 不能为空") } - clean := CleanSecretForYAML(sec) - b, err := yaml.Marshal(clean) + obj, err := redactedSecretYAMLObject(sec) + if err != nil { + return "", fmt.Errorf("转换为YAML失败: %w", err) + } + b, err := yaml.Marshal(obj) if err != nil { return "", fmt.Errorf("转换为YAML失败: %w", err) } @@ -128,11 +134,18 @@ func YAMLToSecret(y string) (*corev1.Secret, error) { if y == "" { return nil, fmt.Errorf("YAML 字符串不能为空") } - var sec corev1.Secret - if err := yaml.Unmarshal([]byte(y), &sec); err != nil { - return nil, fmt.Errorf("解析YAML失败: %w", err) + // 与占位符检查/恢复共用同一个类型化解析：单文档、无重复键、错误不回显值 + sec, placeholders, err := parseSecretYAMLTyped(y) + if err != nil { + return nil, err + } + if len(placeholders) > 0 { + return nil, fmt.Errorf("data 中键 %s 仍为脱敏占位符 %s，请填写真实值", strings.Join(placeholders, ", "), SecretRedactedPlaceholder) + } + if err := checkNoSecretPlaceholder(sec); err != nil { + return nil, err } - return &sec, nil + return sec, nil } func ValidateSecretData(secretType corev1.SecretType, data map[string][]byte, stringData map[string]string) error { diff --git a/internal/k8s/utils/secret_redact.go b/internal/k8s/utils/secret_redact.go new file mode 100644 index 00000000..7e2fdc46 --- /dev/null +++ b/internal/k8s/utils/secret_redact.go @@ -0,0 +1,522 @@ +/* + * MIT License + * + * Copyright (c) 2024 Bamboo + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +package utils + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + yamlv2 "go.yaml.in/yaml/v2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +const ( + // SecretRedactedPlaceholder YAML 中替换 data/stringData 值的占位符。 + // 它不是合法的 base64，因此不可能被当作真实值静默写回集群。 + SecretRedactedPlaceholder = "" + // LastAppliedConfigAnnotation kubectl apply 写入的注解，内容为完整对象（含 Secret 明文） + LastAppliedConfigAnnotation = "kubectl.kubernetes.io/last-applied-configuration" +) + +// SecretDataKeyInfos 返回 data 的键与值字节数（按 key 升序），不包含值 +func SecretDataKeyInfos(data map[string][]byte) []model.SecretKeyInfo { + infos := make([]model.SecretKeyInfo, 0, len(data)) + for k, v := range data { + infos = append(infos, model.SecretKeyInfo{Key: k, Size: len(v)}) + } + sort.Slice(infos, func(i, j int) bool { return infos[i].Key < infos[j].Key }) + return infos +} + +// SecretStringDataKeyInfos 返回 stringData 的键与值字节数（按 key 升序），不包含值 +func SecretStringDataKeyInfos(data map[string]string) []model.SecretKeyInfo { + infos := make([]model.SecretKeyInfo, 0, len(data)) + for k, v := range data { + infos = append(infos, model.SecretKeyInfo{Key: k, Size: len(v)}) + } + sort.Slice(infos, func(i, j int) bool { return infos[i].Key < infos[j].Key }) + return infos +} + +// RedactSecretData 仅保留键，值置为空 +func RedactSecretData(data map[string][]byte) model.BinaryDataMap { + if data == nil { + return nil + } + out := make(model.BinaryDataMap, len(data)) + for k := range data { + out[k] = []byte{} + } + return out +} + +// RedactSecretStringData 仅保留键，值置为空 +func RedactSecretStringData(data map[string]string) map[string]string { + if data == nil { + return nil + } + out := make(map[string]string, len(data)) + for k := range data { + out[k] = "" + } + return out +} + +// RedactSecretAnnotations 复制注解并去掉可能包含明文的 last-applied-configuration +func RedactSecretAnnotations(annotations map[string]string) map[string]string { + if annotations == nil { + return nil + } + out := make(map[string]string, len(annotations)) + for k, v := range annotations { + if k == LastAppliedConfigAnnotation { + continue + } + out[k] = v + } + return out +} + +// redactedSecretYAMLObject 将清理后的 Secret 转为通用对象，并把 data/stringData 的值替换为占位符 +func redactedSecretYAMLObject(sec *corev1.Secret) (map[string]interface{}, error) { + clean := CleanSecretForYAML(sec) + raw, err := json.Marshal(clean) + if err != nil { + return nil, err + } + var obj map[string]interface{} + if err := json.Unmarshal(raw, &obj); err != nil { + return nil, err + } + for _, field := range []string{"data", "stringData"} { + values, ok := obj[field].(map[string]interface{}) + if !ok { + continue + } + for k := range values { + values[k] = SecretRedactedPlaceholder + } + } + return obj, nil +} + +// secretYAMLDocument 与 corev1.Secret 的字段及 JSON 名称一一对应（由测试保证）， +// 唯一区别是 data 的值保留原始 JSON 文本：base64 解码前先识别裸占位符， +// 其余值再按 corev1.Secret 相同的规则（encoding/json 的 []byte 解码）解码。 +// 不能用嵌入 corev1.Secret 的方式定义：嵌入字段在 sigs.k8s.io/yaml 中会丢失数值转字符串的处理。 +type secretYAMLDocument struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Immutable *bool `json:"immutable,omitempty"` + Data map[string]json.RawMessage `json:"data,omitempty"` + StringData map[string]string `json:"stringData,omitempty"` + Type corev1.SecretType `json:"type,omitempty"` +} + +// maxSecretYAMLBytes 是 Secret YAML 入口允许的最大字节数（2 MiB），按 YAML 文本字节数计，在任何解析之前检查。 +// data 的值是 base64（膨胀约 4/3），取 2 MiB 是为了让接近 Kubernetes 1 MiB 上限的 Secret 也能提交， +// Secret 本身的大小限制交给 apiserver 判定；这里只防止超大输入进入解析。 +const maxSecretYAMLBytes = 2 << 20 + +var ( + errSecretYAMLMultiDoc = errors.New("解析YAML失败: 只支持单个 YAML 文档，请去掉 --- 分隔的其他文档") + errSecretYAMLTooLarge = errors.New("解析YAML失败: YAML 超过 2 MiB 上限") + errSecretYAMLDupKeyStr = errors.New("解析YAML失败: 存在按字符串比较相同的重复键（例如 1 与 \"1\"），请只保留一个") + // 只从库报错里取行号，其余内容一律不保留 + secretYAMLErrorLine = regexp.MustCompile(`line ([0-9]+)`) +) + +// secretYAMLErrorClasses 是库报错到固定文案的白名单映射，按顺序匹配第一项。 +// 报错中只保留类别与行号，不保留任何来自输入的键或值。 +var secretYAMLErrorClasses = []struct { + marker string + text string +}{ + {"invalid map key", "映射键必须是字符串等标量"}, + {"unsupported map key", "映射键必须是字符串等标量"}, + {"already set in map", "存在重复键"}, + {"unknown anchor", "引用了未定义的锚点"}, + {"excessive aliasing", "别名展开过多"}, + {"exceeded max depth", "嵌套层级过深"}, + {"map merge requires map", "合并键 << 只能引用映射"}, + {"cannot decode", "值的类型与标签不匹配"}, + {"cannot unmarshal", "字段类型不匹配"}, + {"parsing time", "时间字段格式不正确"}, + {"illegal base64", "base64 格式不正确"}, + {"yaml:", "YAML 语法错误"}, +} + +// sanitizeSecretYAMLError 按白名单重写解析错误：只保留错误类别与行号，不回显任何键或值 +func sanitizeSecretYAMLError(err error) error { + msg := err.Error() + text := "YAML 格式或字段类型错误" + for _, c := range secretYAMLErrorClasses { + if strings.Contains(msg, c.marker) { + text = c.text + break + } + } + if m := secretYAMLErrorLine.FindStringSubmatch(msg); m != nil { + return fmt.Errorf("解析YAML失败: line %s: %s", m[1], text) + } + return fmt.Errorf("解析YAML失败: %s", text) +} + +// secretYAMLKeyString 与 sigs.k8s.io/yaml 把 YAML 键转成 JSON 键的规则一致；不支持的键类型返回 false +func secretYAMLKeyString(k interface{}) (string, bool) { + switch v := k.(type) { + case string: + return v, true + case int: + return strconv.Itoa(v), true + case int64: + return strconv.FormatInt(v, 10), true + case float64: + s := strconv.FormatFloat(v, 'g', -1, 32) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + return s, true + case bool: + if v { + return "true", true + } + return "false", true + } + return "", false +} + +// checkStringifiedDuplicateKeys 拒绝同一映射中转成字符串后相同的键（如 1 与 "1"）， +// 否则 encoding/json 合并时以哪个为准取决于 map 遍历顺序，结果不确定。 +func checkStringifiedDuplicateKeys(node interface{}) error { + switch v := node.(type) { + case map[interface{}]interface{}: + seen := make(map[string]struct{}, len(v)) + for k, child := range v { + if s, ok := secretYAMLKeyString(k); ok { + if _, dup := seen[s]; dup { + return errSecretYAMLDupKeyStr + } + seen[s] = struct{}{} + } + if err := checkStringifiedDuplicateKeys(child); err != nil { + return err + } + } + case []interface{}: + for _, child := range v { + if err := checkStringifiedDuplicateKeys(child); err != nil { + return err + } + } + } + return nil +} + +// checkSingleYAMLDocument 拒绝多文档 YAML（包括末尾多出的空文档 ---）， +// 并拒绝转成字符串后相同的重复键 +func checkSingleYAMLDocument(raw []byte) error { + dec := yamlv2.NewDecoder(bytes.NewReader(raw)) + var first interface{} + if err := dec.Decode(&first); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return sanitizeSecretYAMLError(err) + } + if err := checkStringifiedDuplicateKeys(first); err != nil { + return err + } + var next interface{} + if err := dec.Decode(&next); !errors.Is(err, io.EOF) { + return errSecretYAMLMultiDoc + } + return nil +} + +// foldSecretFieldName 与 encoding/json 匹配字段名时的大小写折叠规则一致 +func foldSecretFieldName(name string) string { + var b strings.Builder + for _, r := range name { + if r < utf8.RuneSelf { + if 'a' <= r && r <= 'z' { + r -= 'a' - 'A' + } + b.WriteRune(r) + continue + } + for { + r2 := unicode.SimpleFold(r) + if r2 <= r { + r = r2 + break + } + r = r2 + } + b.WriteRune(r) + } + return b.String() +} + +// checkFoldedDuplicateKeys 拒绝只差大小写、会被 encoding/json 合并到同一字段的键 +func checkFoldedDuplicateKeys(obj map[string]json.RawMessage, prefix string) error { + keys := make([]string, 0, len(obj)) + for k := range obj { + keys = append(keys, k) + } + sort.Strings(keys) + seen := make(map[string]string, len(keys)) + for _, k := range keys { + f := foldSecretFieldName(k) + if prev, ok := seen[f]; ok { + return fmt.Errorf("解析YAML失败: 字段 %s%s 与 %s%s 只差大小写，请只保留一个", prefix, prev, prefix, k) + } + seen[f] = k + } + return nil +} + +// checkSecretFieldCaseDuplicates 检查顶层与 metadata 下只差大小写的重复字段 +func checkSecretFieldCaseDuplicates(j []byte) error { + var top map[string]json.RawMessage + if err := json.Unmarshal(j, &top); err != nil { + // 非对象由后续类型化解析报错 + return nil + } + if err := checkFoldedDuplicateKeys(top, ""); err != nil { + return err + } + metaName := foldSecretFieldName("metadata") + for k, v := range top { + if foldSecretFieldName(k) != metaName { + continue + } + var meta map[string]json.RawMessage + if err := json.Unmarshal(v, &meta); err != nil { + continue + } + if err := checkFoldedDuplicateKeys(meta, k+"."); err != nil { + return err + } + } + return nil +} + +// parseSecretYAMLTyped 把单文档 YAML 解析为类型化的 corev1.Secret。 +// 返回值中 data 里值恰为占位符的键不放入 sec.Data，而是按升序单独返回，供调用方恢复或拒绝。 +// 所有错误都不包含 Secret 的值。 +func parseSecretYAMLTyped(y string) (*corev1.Secret, []string, error) { + if len(y) > maxSecretYAMLBytes { + return nil, nil, errSecretYAMLTooLarge + } + raw := []byte(y) + if err := checkSingleYAMLDocument(raw); err != nil { + return nil, nil, err + } + // 严格模式：任意层级完全相同的重复键直接报错（报错只含键名） + j, err := yaml.YAMLToJSONStrict(raw) + if err != nil { + return nil, nil, sanitizeSecretYAMLError(err) + } + if err := checkSecretFieldCaseDuplicates(j); err != nil { + return nil, nil, err + } + var doc secretYAMLDocument + if err := yaml.Unmarshal(raw, &doc); err != nil { + return nil, nil, sanitizeSecretYAMLError(err) + } + sec := &corev1.Secret{ + TypeMeta: doc.TypeMeta, + ObjectMeta: doc.ObjectMeta, + Immutable: doc.Immutable, + StringData: doc.StringData, + Type: doc.Type, + } + var placeholders []string + if doc.Data != nil { + sec.Data = make(map[string][]byte, len(doc.Data)) + for k, v := range doc.Data { + var s string + if json.Unmarshal(v, &s) == nil && s == SecretRedactedPlaceholder { + placeholders = append(placeholders, k) + continue + } + var b []byte + if err := json.Unmarshal(v, &b); err != nil { + return nil, nil, fmt.Errorf("解析YAML失败: data 中键 %s 的值不是合法的 base64 字符串", k) + } + sec.Data[k] = b + } + } + sort.Strings(placeholders) + return sec, placeholders, nil +} + +// isSecretPlaceholderPadding 是判断占位符前要去掉的首尾字符：空白、NUL 与零宽字符 +func isSecretPlaceholderPadding(r rune) bool { + switch r { + case 0, '\u200b', '\u200c', '\u200d', '\ufeff': + return true + } + return unicode.IsSpace(r) +} + +// isSecretPlaceholderValue 判断值去掉首尾空白、NUL、零宽字符（U+200B/U+200C/U+200D/U+FEFF）后是否为占位符 +func isSecretPlaceholderValue(v string) bool { + return strings.TrimFunc(v, isSecretPlaceholderPadding) == SecretRedactedPlaceholder +} + +// checkNoSecretPlaceholder 确认 data（解码后）与 stringData 中没有占位符或其首尾带空白/NUL/零宽字符的变体 +func checkNoSecretPlaceholder(sec *corev1.Secret) error { + dataKeys := make([]string, 0, len(sec.Data)) + for k := range sec.Data { + dataKeys = append(dataKeys, k) + } + sort.Strings(dataKeys) + for _, k := range dataKeys { + if isSecretPlaceholderValue(string(sec.Data[k])) { + return fmt.Errorf("data 中键 %s 解码后仍为脱敏占位符 %s，请填写真实值", k, SecretRedactedPlaceholder) + } + } + stringKeys := make([]string, 0, len(sec.StringData)) + for k := range sec.StringData { + stringKeys = append(stringKeys, k) + } + sort.Strings(stringKeys) + for _, k := range stringKeys { + if isSecretPlaceholderValue(sec.StringData[k]) { + return fmt.Errorf("stringData 中键 %s 仍为脱敏占位符 %s（或首尾带空白、NUL、零宽字符的变体），请填写真实值", k, SecretRedactedPlaceholder) + } + } + return nil +} + +// SecretYAMLHasRedactedPlaceholder 判断 YAML 的 data/stringData 中是否仍含脱敏占位符。 +// 先解析为类型化的 Secret（字段名大小写变体按 encoding/json 不区分大小写的规则归并； +// apiserver/kubectl 是区分大小写的，见 docs/release/k8s-secret-redaction.md）， +// data 中的裸占位符、解码后为占位符的值、stringData 中首尾带空白/NUL/零宽字符的占位符都算。 +func SecretYAMLHasRedactedPlaceholder(y string) (bool, error) { + sec, placeholders, err := parseSecretYAMLTyped(y) + if err != nil { + return false, err + } + if len(placeholders) > 0 || checkNoSecretPlaceholder(sec) != nil { + return true, nil + } + return false, nil +} + +// RestoreRedactedSecretYAML 将 YAML 中仍为占位符的键恢复为集群中的现有值。 +// 前端拿到脱敏 YAML 后只改了部分字段再提交时，未改动的键保持原值而不是被占位符覆盖。 +// 占位符对应的键在现有 Secret 中不存在、或恢复后仍残留占位符变体时返回错误。 +// 有恢复时返回由类型化 Secret 重新生成的 YAML，否则原样返回。 +func RestoreRedactedSecretYAML(y string, existing *corev1.Secret) (string, error) { + sec, placeholders, err := parseSecretYAMLTyped(y) + if err != nil { + return "", err + } + var existingData map[string][]byte + var existingStringData map[string]string + if existing != nil { + existingData = existing.Data + existingStringData = existing.StringData + } + changed := false + + for _, k := range placeholders { + var restored []byte + if old, exists := existingData[k]; exists { + restored = old + } else if old, exists := existingStringData[k]; exists { + restored = []byte(old) + } else { + return "", fmt.Errorf("键 %s 仍为脱敏占位符且在现有Secret中不存在，请填写真实值", k) + } + if sec.Data == nil { + sec.Data = map[string][]byte{} + } + sec.Data[k] = restored + changed = true + } + + stringKeys := make([]string, 0, len(sec.StringData)) + for k := range sec.StringData { + stringKeys = append(stringKeys, k) + } + sort.Strings(stringKeys) + for _, k := range stringKeys { + if sec.StringData[k] != SecretRedactedPlaceholder { + continue + } + if old, exists := existingStringData[k]; exists { + sec.StringData[k] = old + changed = true + continue + } + old, exists := existingData[k] + if !exists { + return "", fmt.Errorf("键 %s 仍为脱敏占位符且在现有Secret中不存在，请填写真实值", k) + } + // stringData 写入后会合并到 data，这里直接以 data 形式恢复原值 + delete(sec.StringData, k) + if sec.Data == nil { + sec.Data = map[string][]byte{} + } + if _, set := sec.Data[k]; !set { + sec.Data[k] = old + } + changed = true + } + + if err := checkNoSecretPlaceholder(sec); err != nil { + return "", err + } + if !changed { + return y, nil + } + out, err := yaml.Marshal(sec) + if err != nil { + return "", fmt.Errorf("生成YAML失败: %w", err) + } + return string(out), nil +} diff --git a/internal/k8s/utils/secret_redact_case_test.go b/internal/k8s/utils/secret_redact_case_test.go new file mode 100644 index 00000000..bf87fb09 --- /dev/null +++ b/internal/k8s/utils/secret_redact_case_test.go @@ -0,0 +1,370 @@ +package utils + +import ( + "encoding/base64" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SEC-26 P2：字段名大小写变体、折行 base64、多文档、重复键、错误回显。 +// 本文件只调用公开函数，可在修复前的代码上编译运行（先红后绿）。 + +const p2Head = "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\n namespace: ops\n" + +// "" 的 base64 为 PHJlZGFjdGVkPg== +const p2PlaceholderB64 = "PHJlZGFjdGVkPg==" + +func p2Existing() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "ops"}, + Data: map[string][]byte{"k": []byte(redactFixtureData)}, + } +} + +// 模拟 service 层 UpdateSecretByYaml：先恢复占位符，再解析。 +func p2Update(y string, existing *corev1.Secret) (*corev1.Secret, error) { + restored, err := RestoreRedactedSecretYAML(y, existing) + if err != nil { + return nil, err + } + return YAMLToSecret(restored) +} + +// 模拟 service 层 CreateSecretByYaml：先检查占位符，再解析。 +func p2Create(y string) (*corev1.Secret, error) { + has, err := SecretYAMLHasRedactedPlaceholder(y) + if err != nil { + return nil, err + } + if has { + return nil, errPlaceholderForTest + } + return YAMLToSecret(y) +} + +type p2Err string + +func (e p2Err) Error() string { return string(e) } + +const errPlaceholderForTest = p2Err("YAML 中包含脱敏占位符") + +// 最终要写入集群的对象里不得出现占位符（含空白变体）。 +func p2AssertNoPlaceholder(t *testing.T, sec *corev1.Secret) { + t.Helper() + for k, v := range sec.Data { + if strings.TrimSpace(string(v)) == SecretRedactedPlaceholder { + t.Fatalf("data[%s] 将以占位符写入集群", k) + } + } + for k, v := range sec.StringData { + if strings.TrimSpace(v) == SecretRedactedPlaceholder { + t.Fatalf("stringData[%s] 将以占位符写入集群", k) + } + } +} + +// 错误信息不得包含任何测试假值（明文或 base64）。只报告是否泄露，不打印内容。 +func p2AssertNoEcho(t *testing.T, err error) { + t.Helper() + if err == nil { + return + } + msg := err.Error() + for i, v := range []string{redactFixtureData, redactFixtureString, redactFixtureNew} { + if strings.Contains(msg, v) || strings.Contains(msg, base64.StdEncoding.EncodeToString([]byte(v))) { + t.Fatalf("错误信息回显了第 %d 个测试假值", i) + } + } +} + +// 应被拒绝：更新和新建两条路径都必须报错，且不回显值。 +func p2MustReject(t *testing.T, y string) { + t.Helper() + sec, err := p2Update(y, p2Existing()) + if err == nil { + p2AssertNoPlaceholder(t, sec) + t.Fatalf("更新路径应拒绝，但被接受") + } + p2AssertNoEcho(t, err) + sec, err = p2Create(y) + if err == nil { + p2AssertNoPlaceholder(t, sec) + t.Fatalf("新建路径应拒绝，但被接受") + } + p2AssertNoEcho(t, err) +} + +// 1) stringData 字段名大小写变体（e01/e02/e03/e10）：更新应恢复原值，新建应拒绝。 +func TestP2StringDataKeyVariants(t *testing.T) { + for _, field := range []string{"stringData", "StringData", "stringdata", "STRINGDATA", "ſtringData"} { + t.Run(field, func(t *testing.T) { + y := p2Head + field + ":\n k: " + SecretRedactedPlaceholder + "\n" + sec, err := p2Update(y, p2Existing()) + if err != nil { + t.Fatalf("更新路径失败: %v", err) + } + p2AssertNoPlaceholder(t, sec) + if string(sec.Data["k"]) != redactFixtureData { + t.Fatal("更新路径未从现有 Secret 恢复 k") + } + if _, ok := sec.StringData["k"]; ok { + t.Fatal("占位符键应从 stringData 移除") + } + if _, err := p2Create(y); err == nil { + t.Fatal("新建路径应拒绝占位符") + } else { + p2AssertNoEcho(t, err) + } + }) + } +} + +// 2) data 字段名大小写变体 + 裸占位符（e04）：更新应恢复原值，新建应拒绝。 +func TestP2DataKeyVariantsBarePlaceholder(t *testing.T) { + for _, field := range []string{"data", "Data", "DATA", "dAtA"} { + t.Run(field, func(t *testing.T) { + y := p2Head + field + ":\n k: " + SecretRedactedPlaceholder + "\n s: " + SecretRedactedPlaceholder + "\n" + existing := p2Existing() + existing.StringData = map[string]string{"s": redactFixtureString} + sec, err := p2Update(y, existing) + if err != nil { + t.Fatalf("更新路径失败: %v", err) + } + p2AssertNoPlaceholder(t, sec) + if string(sec.Data["k"]) != redactFixtureData { + t.Fatal("更新路径未从现有 Secret 的 data 恢复 k") + } + if string(sec.Data["s"]) != redactFixtureString { + t.Fatal("更新路径未从现有 Secret 的 stringData 恢复 s") + } + if _, err := p2Create(y); err == nil { + t.Fatal("新建路径应拒绝占位符") + } else { + p2AssertNoEcho(t, err) + } + }) + } +} + +// 3) data 下 base64 编码的占位符：折行（e05/e08）、转义（d01-d03）、单行。两条路径都应拒绝。 +func TestP2DataEncodedPlaceholderRejected(t *testing.T) { + cases := map[string]string{ + "Data_folded": "Data:\n k: |\n PHJlZGFj\n dGVkPg==\n", + "data_folded": "data:\n k: |\n PHJlZGFj\n dGVkPg==\n", + "Data_folded_trailing": "Data:\n k: |\n PHJlZGFj\n dGVkPg==\n---\n", + "DATA_escaped": "DATA:\n k: \"\\x50HJlZGFjdGVkPg==\"\n", + "Data_plain_b64": "Data:\n k: " + p2PlaceholderB64 + "\n", + "data_plain_b64": "data:\n k: " + p2PlaceholderB64 + "\n", + "data_b64_with_space": "data:\n k: " + base64.StdEncoding.EncodeToString([]byte(SecretRedactedPlaceholder+" ")) + "\n", + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { p2MustReject(t, p2Head+body) }) + } +} + +// 4) stringData 带空白的占位符变体（d09/d10）：两条路径都应拒绝。 +func TestP2StringDataWhitespacePlaceholderRejected(t *testing.T) { + cases := map[string]string{ + "trailing_space": "stringData:\n k: \"" + SecretRedactedPlaceholder + " \"\n", + "trailing_newline": "stringData:\n k: \"" + SecretRedactedPlaceholder + "\\n\"\n", + "StringData_space": "StringData:\n k: \" " + SecretRedactedPlaceholder + "\"\n", + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { p2MustReject(t, p2Head+body) }) + } +} + +// 5) 多文档 YAML（p02）：两条路径都应拒绝。 +func TestP2MultiDocumentRejected(t *testing.T) { + second := "---\n" + p2Head + "stringData:\n k: " + SecretRedactedPlaceholder + "\n" + cases := map[string]string{ + "two_docs": p2Head + "stringData:\n k: " + redactFixtureNew + "\n" + second, + "second_has_token": p2Head + "type: Opaque\n" + second, + "trailing_sep": p2Head + "type: Opaque\n---\n", + "doc_end_then_sep": p2Head + "type: Opaque\n...\n---\n" + p2Head, + "second_broken": p2Head + "type: Opaque\n---\n: : [\n", + } + for name, y := range cases { + t.Run(name, func(t *testing.T) { p2MustReject(t, y) }) + } +} + +// 6) 重复键（p01），含只差大小写：两条路径都应拒绝。 +func TestP2DuplicateKeysRejected(t *testing.T) { + cases := map[string]string{ + "exact_stringData": p2Head + "stringData:\n k: " + redactFixtureNew + "\nstringData:\n k: " + SecretRedactedPlaceholder + "\n", + "exact_data": p2Head + "data:\n k: " + p2PlaceholderB64 + "\ndata:\n j: " + p2PlaceholderB64 + "\n", + "case_stringData": p2Head + "stringData:\n k: " + redactFixtureNew + "\nStringData:\n k: " + SecretRedactedPlaceholder + "\n", + "case_data": p2Head + "data:\n k: " + SecretRedactedPlaceholder + "\nData:\n j: " + p2PlaceholderB64 + "\n", + "case_data_upper": p2Head + "data:\n k: " + SecretRedactedPlaceholder + "\nDATA:\n j: " + SecretRedactedPlaceholder + "\n", + "inner_key_exact": p2Head + "stringData:\n k: " + redactFixtureNew + "\n k: " + SecretRedactedPlaceholder + "\n", + "case_kind": p2Head + "Kind: Secret\nstringData:\n k: " + redactFixtureNew + "\n", + "case_metadata": p2Head + "Metadata:\n name: other\nstringData:\n k: " + redactFixtureNew + "\n", + "case_metadata_name": "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\n Name: other\n namespace: ops\nstringData:\n k: " + redactFixtureNew + "\n", + } + for name, y := range cases { + t.Run(name, func(t *testing.T) { p2MustReject(t, y) }) + } +} + +// 7) 解析错误不得回显值：非字符串键、类型标签错误、非法 base64。 +func TestP2ErrorsDoNotEchoValues(t *testing.T) { + newB64 := base64.StdEncoding.EncodeToString([]byte(redactFixtureNew)) + cases := map[string]string{ + "null_key_data": p2Head + "data:\n ~: " + newB64 + "\n", + "null_key_stringData": p2Head + "stringData:\n ~: " + redactFixtureNew + "\n", + "int_key_data": p2Head + "data:\n ? [a]\n : " + newB64 + "\n", + "bad_tag": p2Head + "stringData:\n k: !!int " + redactFixtureNew + "\n", + "bad_base64": p2Head + "data:\n k: " + redactFixtureNew + "\n", + } + for name, y := range cases { + t.Run(name, func(t *testing.T) { + _, err := p2Update(y, p2Existing()) + if err == nil { + t.Fatal("更新路径应报错") + } + p2AssertNoEcho(t, err) + _, err = p2Create(y) + if err == nil { + t.Fatal("新建路径应报错") + } + p2AssertNoEcho(t, err) + }) + } +} + +// 8) 正常输入保持兼容（对照组）：规范键、带 --- 前导、数值转字符串、合法折行 base64。 +func TestP2CanonicalInputsStillAccepted(t *testing.T) { + newB64 := base64.StdEncoding.EncodeToString([]byte(redactFixtureNew)) + folded := newB64[:8] + "\n " + newB64[8:] + t.Run("leading_separator", func(t *testing.T) { + y := "---\n" + p2Head + "stringData:\n k: " + SecretRedactedPlaceholder + "\n" + sec, err := p2Update(y, p2Existing()) + if err != nil { + t.Fatal(err) + } + if string(sec.Data["k"]) != redactFixtureData { + t.Fatal("未恢复") + } + }) + t.Run("doc_end_marker", func(t *testing.T) { + y := p2Head + "stringData:\n k: " + redactFixtureNew + "\n...\n" + if _, err := p2Create(y); err != nil { + t.Fatal(err) + } + }) + t.Run("number_coercion", func(t *testing.T) { + y := p2Head + " labels:\n port: 8080\nstringData:\n port: 8080\n flag: true\n" + sec, err := p2Create(y) + if err != nil { + t.Fatal(err) + } + if sec.StringData["port"] != "8080" || sec.Labels["port"] != "8080" || sec.StringData["flag"] != "true" { + t.Fatal("数值/布尔未按原逻辑转为字符串") + } + }) + t.Run("folded_real_base64", func(t *testing.T) { + y := p2Head + "data:\n k: |\n " + folded + "\n j: " + SecretRedactedPlaceholder + "\n" + existing := p2Existing() + existing.Data["j"] = []byte(redactFixtureString) + sec, err := p2Update(y, existing) + if err != nil { + t.Fatal(err) + } + if string(sec.Data["k"]) != redactFixtureNew || string(sec.Data["j"]) != redactFixtureString { + t.Fatal("合法折行 base64 或占位符恢复结果不对") + } + }) + t.Run("no_placeholder_unchanged", func(t *testing.T) { + y := p2Head + "data:\n k: " + newB64 + "\n" + out, err := RestoreRedactedSecretYAML(y, p2Existing()) + if err != nil { + t.Fatal(err) + } + if out != y { + t.Fatal("无占位符时应原样返回") + } + }) + t.Run("missing_key_error_names_key_only", func(t *testing.T) { + y := p2Head + "Data:\n absent: " + SecretRedactedPlaceholder + "\n" + _, err := p2Update(y, p2Existing()) + if err == nil || !strings.Contains(err.Error(), "absent") { + t.Fatalf("应报出缺失的键名, got err=%v", err != nil) + } + p2AssertNoEcho(t, err) + }) +} + +// 9) YAML 合并键把占位符带进 StringData：更新恢复，新建拒绝。 +func TestP2MergeKeyStringData(t *testing.T) { + y := p2Head + "x: &a\n k: " + SecretRedactedPlaceholder + "\nStringData:\n <<: *a\n" + sec, err := p2Update(y, p2Existing()) + if err != nil { + t.Fatalf("更新路径失败: %v", err) + } + p2AssertNoPlaceholder(t, sec) + if _, err := p2Create(y); err == nil { + t.Fatal("新建路径应拒绝占位符") + } +} + +// 10) 每一层单独都要拦住（纵深防御）：Has 判定、Restore 残留检查、YAMLToSecret 兜底各自独立生效。 +func TestP2EachLayerRejectsIndependently(t *testing.T) { + inputs := map[string]string{ + "Data_folded": p2Head + "Data:\n k: |\n PHJlZGFj\n dGVkPg==\n", + "stringData_space": p2Head + "stringData:\n k: \"" + SecretRedactedPlaceholder + " \"\n", + "STRINGDATA_nl": p2Head + "STRINGDATA:\n k: \"" + SecretRedactedPlaceholder + "\\n\"\n", + } + for name, y := range inputs { + t.Run(name, func(t *testing.T) { + if has, err := SecretYAMLHasRedactedPlaceholder(y); err == nil && !has { + t.Fatal("SecretYAMLHasRedactedPlaceholder 未识别") + } + if _, err := RestoreRedactedSecretYAML(y, p2Existing()); err == nil { + t.Fatal("RestoreRedactedSecretYAML 未拒绝残留占位符") + } + if _, err := YAMLToSecret(y); err == nil { + t.Fatal("YAMLToSecret 未拒绝占位符") + } + }) + } + // data 裸占位符：Has 必须为 true，YAMLToSecret 必须拒绝（Restore 会恢复，不在此列） + for _, field := range []string{"data", "Data"} { + y := p2Head + field + ":\n k: " + SecretRedactedPlaceholder + "\n" + if has, err := SecretYAMLHasRedactedPlaceholder(y); err != nil || !has { + t.Fatalf("%s: SecretYAMLHasRedactedPlaceholder 应返回 true", field) + } + if _, err := YAMLToSecret(y); err == nil { + t.Fatalf("%s: YAMLToSecret 应拒绝", field) + } + } + // 结构问题：YAMLToSecret 自身也要拒绝（不依赖前置检查） + structural := map[string]string{ + "multi_doc": p2Head + "type: Opaque\n---\n" + p2Head, + "dup_exact": p2Head + "type: Opaque\ntype: Opaque\n", + "dup_case": p2Head + "stringData:\n a: b\nStringData:\n c: d\n", + "dup_meta": p2Head + " Name: other\n", + "dup_unicod": p2Head + "stringData:\n a: b\nſtringData:\n c: d\n", + } + for name, y := range structural { + if _, err := YAMLToSecret(y); err == nil { + t.Fatalf("%s: YAMLToSecret 应拒绝", name) + } + if _, err := SecretYAMLHasRedactedPlaceholder(y); err == nil { + t.Fatalf("%s: SecretYAMLHasRedactedPlaceholder 应报错", name) + } + } +} + +// 11) 只有注释/空文档时保持原行为：Restore 原样返回，不当作多文档。 +func TestP2EmptyDocumentCompat(t *testing.T) { + for _, y := range []string{"# only comment\n", "---\n"} { + out, err := RestoreRedactedSecretYAML(y, p2Existing()) + if err != nil || out != y { + t.Fatalf("空文档应原样返回: %q", y) + } + } +} diff --git a/internal/k8s/utils/secret_redact_r2_test.go b/internal/k8s/utils/secret_redact_r2_test.go new file mode 100644 index 00000000..eb3fa685 --- /dev/null +++ b/internal/k8s/utils/secret_redact_r2_test.go @@ -0,0 +1,316 @@ +package utils + +import ( + "encoding/base64" + "errors" + "strings" + "testing" +) + +// SEC-26 P2 第一层 Review 跟进（P2-1、P3-1、P3-2、P3-4、P3-6）。 +// 只调用 f763601 上已存在的函数，可在修复前的代码上编译运行（先红后绿）。 + +// 哨兵前缀：放在值的位置，报错里出现它即视为回显。 +const r2Sentinel = "ZQXSNTL" + +func r2AssertNoSentinel(t *testing.T, err error) { + t.Helper() + if err != nil && strings.Contains(err.Error(), r2Sentinel) { + t.Fatal("错误信息回显了输入中的值") + } +} + +// 两条路径都必须报错，且报错不含哨兵。 +func r2MustErrorNoEcho(t *testing.T, y string) { + t.Helper() + _, err := p2Update(y, p2Existing()) + if err == nil { + t.Fatal("更新路径应报错") + } + r2AssertNoSentinel(t, err) + _, err = p2Create(y) + if err == nil { + t.Fatal("新建路径应报错") + } + r2AssertNoSentinel(t, err) +} + +// P2-1：invalid map key（复杂键）报错不得回显值。 +func TestR2InvalidMapKeyDoesNotEcho(t *testing.T) { + cases := map[string]string{ + "flow_double_brace_stringData": "stringData: {{password: " + r2Sentinel + "01}}\n", + "flow_double_brace_data": "data: {{password: " + r2Sentinel + "02}}\n", + "seq_key_stringData": "stringData:\n ? [" + r2Sentinel + "03]\n : x\n", + "map_key_stringData": "stringData:\n ? {a: " + r2Sentinel + "04}\n : x\n", + "seq_key_value_side": "stringData:\n ? [a]\n : " + r2Sentinel + "05\n", + "top_level_brace": "{{password: " + r2Sentinel + "06}}\n", + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { r2MustErrorNoEcho(t, p2Head+body) }) + } +} + +// P2-1：直接构造 yaml/json 库可能给出的报错形态，确认过滤器不放行值。 +func TestR2SanitizeKnownLibraryMessages(t *testing.T) { + msgs := []string{ + `yaml: invalid map key: map[interface {}]interface {}{"password":"` + r2Sentinel + `"}`, + `yaml: line 7: invalid map key: []interface {}{"` + r2Sentinel + `"}`, + `unsupported map key of type: , key: , value: "` + r2Sentinel + `"`, + "yaml: line 3: cannot decode !!str `" + r2Sentinel + "` as a !!int", + `parsing time "` + r2Sentinel + `" as "2006-01-02T15:04:05Z07:00": cannot parse`, + "yaml: unknown anchor '" + r2Sentinel + "' referenced", + `error converting YAML to JSON: yaml: line 4: key "` + r2Sentinel + `" already set in map`, + `json: cannot unmarshal string into Go struct field secretYAMLDocument.metadata.name of type ` + r2Sentinel, + "yaml: line 2: found character that cannot start any token " + r2Sentinel, + } + for i, m := range msgs { + err := sanitizeSecretYAMLError(errors.New(m)) + if strings.Contains(err.Error(), r2Sentinel) { + t.Fatalf("第 %d 条报错未过滤", i) + } + } + // 行号要保留 + if err := sanitizeSecretYAMLError(errors.New(`yaml: line 7: invalid map key: []interface {}{"x"}`)); !strings.Contains(err.Error(), "7") { + t.Fatal("应保留行号") + } +} + +// P2-1：表驱动覆盖各类 yaml/json 报错，值的位置放哨兵。 +func TestR2YAMLErrorClassesDoNotEcho(t *testing.T) { + s := r2Sentinel + cases := map[string]string{ + "unclosed_dquote": p2Head + "stringData:\n k: \"" + s + "a\n", + "unclosed_squote": p2Head + "stringData:\n k: '" + s + "b\n", + "unclosed_flow": p2Head + "stringData: {k: " + s + "c\n", + "bad_escape": p2Head + "stringData:\n k: \"\\q" + s + "d\"\n", + "tab_indent": p2Head + "stringData:\n\tk: " + s + "e\n", + "mapping_in_value": p2Head + "stringData:\n k: " + s + "f: g\n", + "tag_int": p2Head + "stringData:\n k: !!int " + s + "g\n", + "tag_float": p2Head + "stringData:\n k: !!float " + s + "h\n", + "tag_bool": p2Head + "stringData:\n k: !!bool " + s + "i\n", + "tag_binary": p2Head + "stringData:\n k: !!binary " + s + "j!!\n", + "tag_timestamp": p2Head + "stringData:\n k: !!timestamp " + s + "k\n", + "unknown_anchor": p2Head + "stringData:\n k: *" + s + "l\n", + "creation_timestamp": "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\n creationTimestamp: " + s + "m\n", + "data_is_scalar": p2Head + "data: " + s + "n\n", + "stringData_is_list": p2Head + "stringData: [" + s + "o]\n", + "stringData_val_map": p2Head + "stringData:\n k: {a: " + s + "p}\n", + "immutable_string": p2Head + "immutable: " + s + "q\n", + "name_is_list": "apiVersion: v1\nkind: Secret\nmetadata:\n name: [" + s + "r]\n", + "labels_val_list": p2Head + " labels:\n a: [" + s + "s]\n", + "top_level_list": "- " + s + "t\n", + "top_level_scalar": s + "u\n", + "bad_base64": p2Head + "data:\n k: " + s + "v!\n", + "null_key": p2Head + "stringData:\n ~: " + s + "w\n", + "dup_key_same_value": p2Head + "stringData:\n k: " + s + "x\n k: " + s + "x\n", + "second_doc_broken": p2Head + "stringData:\n k: a\n---\nk: \"" + s + "y\n", + "control_char": p2Head + "stringData:\n k: " + s + "z\x01\n", + "merge_non_map": p2Head + "stringData:\n <<: " + s + "A\n", + "directive_bad": "%YAML 9.9 " + s + "B\n" + p2Head, + "anchor_then_override": p2Head + "x: &a {k: " + s + "C}\nstringData:\n <<: *a\n k: " + s + "D\n", + } + for name, y := range cases { + t.Run(name, func(t *testing.T) { r2MustErrorNoEcho(t, y) }) + } +} + +// P2-1：模糊测试，把任意片段放到多个值位置，报错不得出现哨兵。 +func FuzzSecretYAMLErrorsDoNotEcho(f *testing.F) { + for _, seed := range []string{"x", "}}", "]", "\"", "'", "\t", ": y", "\n - z", "!!int", "*a", "{", "\\", "\x00", "{{a: b}}"} { + f.Add(seed) + } + templates := []string{ + p2Head + "stringData:\n k: %s\n", + p2Head + "stringData: {{password: %s}}\n", + p2Head + "stringData:\n ? [%s]\n : x\n", + p2Head + "data:\n k: %s\n", + p2Head + "stringData:\n k: \"%s\n", + p2Head + "stringData:\n k: !!int %s\n", + "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\n creationTimestamp: %s\n", + } + f.Fuzz(func(t *testing.T, frag string) { + for _, tpl := range templates { + y := strings.Replace(tpl, "%s", r2Sentinel+frag, 1) + if _, err := p2Update(y, p2Existing()); err != nil { + r2AssertNoSentinel(t, err) + } + if _, err := p2Create(y); err != nil { + r2AssertNoSentinel(t, err) + } + } + }) +} + +// P3-1：首尾的 NUL 与零宽字符（ZWSP/ZWNJ/ZWJ/BOM）也要去掉后再判断占位符。 +func TestR2PlaceholderWithInvisibleChars(t *testing.T) { + variants := map[string]string{ + "nul_suffix": SecretRedactedPlaceholder + "\x00", + "nul_prefix": "\x00" + SecretRedactedPlaceholder, + "zwsp": "\u200b" + SecretRedactedPlaceholder, + "zwnj": SecretRedactedPlaceholder + "\u200c", + "zwj": "\u200d" + SecretRedactedPlaceholder + "\u200d", + "bom": "\ufeff" + SecretRedactedPlaceholder, + "mixed": "\ufeff \u200b" + SecretRedactedPlaceholder + "\x00\n\u200c", + "nul_nul_ws": "\x00\x00 " + SecretRedactedPlaceholder + " \x00", + "zwsp_suffix": SecretRedactedPlaceholder + "\u200b", + } + for name, v := range variants { + t.Run("data_"+name, func(t *testing.T) { + p2MustReject(t, p2Head+"data:\n k: "+base64.StdEncoding.EncodeToString([]byte(v))+"\n") + }) + t.Run("stringData_"+name, func(t *testing.T) { + p2MustReject(t, p2Head+"stringData:\n k: \""+r2YAMLEscape(v)+"\"\n") + }) + } + // 对照：零宽字符在中间时不是占位符，按普通值接受 + t.Run("interior_not_placeholder", func(t *testing.T) { + y := p2Head + "stringData:\n k: \"\"\n" + if _, err := p2Create(y); err != nil { + t.Fatal(err) + } + }) +} + +// 把字符串转成 YAML 双引号字面量（不可见字符用转义写出） +func r2YAMLEscape(v string) string { + var b strings.Builder + for _, r := range v { + switch { + case r == 0: + b.WriteString(`\0`) + case r == '\n': + b.WriteString(`\n`) + case r == '"' || r == '\\': + b.WriteRune('\\') + b.WriteRune(r) + case r > 0x7e: + b.WriteString(`\u` + hex4(r)) + default: + b.WriteRune(r) + } + } + return b.String() +} + +func hex4(r rune) string { + const digits = "0123456789abcdef" + return string([]byte{digits[(r>>12)&0xf], digits[(r>>8)&0xf], digits[(r>>4)&0xf], digits[r&0xf]}) +} + +// P3-2：字符串化后相同的键（1 与 "1"、true 与 "true"、1.5 与 "1.5"）按重复键拒绝，多次运行结果稳定。 +func TestR2StringifiedDuplicateKeysRejectedStably(t *testing.T) { + newB64 := base64.StdEncoding.EncodeToString([]byte(redactFixtureNew)) + cases := map[string]string{ + "stringData_int": p2Head + "stringData:\n 1: \"" + SecretRedactedPlaceholder + "\"\n \"1\": " + redactFixtureNew + "\n", + "data_int": p2Head + "data:\n 1: \"" + SecretRedactedPlaceholder + "\"\n \"1\": " + newB64 + "\n", + "stringData_bool": p2Head + "stringData:\n true: \"" + SecretRedactedPlaceholder + "\"\n \"true\": " + redactFixtureNew + "\n", + "stringData_yes": p2Head + "stringData:\n yes: \"" + SecretRedactedPlaceholder + "\"\n \"true\": " + redactFixtureNew + "\n", + "stringData_float": p2Head + "stringData:\n 1.5: \"" + SecretRedactedPlaceholder + "\"\n \"1.5\": " + redactFixtureNew + "\n", + "labels_int": p2Head + " labels:\n 1: a\n \"1\": b\n", + "nested_in_list": p2Head + "extra:\n - 1: a\n \"1\": b\n", + } + for name, y := range cases { + t.Run(name, func(t *testing.T) { + for i := 0; i < 50; i++ { + if _, err := RestoreRedactedSecretYAML(y, p2Existing()); err == nil { + t.Fatalf("第 %d 次：RestoreRedactedSecretYAML 应拒绝", i) + } + if _, err := SecretYAMLHasRedactedPlaceholder(y); err == nil { + t.Fatalf("第 %d 次：SecretYAMLHasRedactedPlaceholder 应报错", i) + } + if _, err := YAMLToSecret(y); err == nil { + t.Fatalf("第 %d 次：YAMLToSecret 应拒绝", i) + } + } + }) + } + // 对照：类型不同但字符串化后不同的键照常接受 + t.Run("distinct_ok", func(t *testing.T) { + y := p2Head + "stringData:\n 1: a\n \"2\": b\n" + sec, err := YAMLToSecret(y) + if err != nil || sec.StringData["1"] != "a" || sec.StringData["2"] != "b" { + t.Fatal("不同键应照常接受") + } + }) +} + +// P3-4：metadata 字段名本身为非标准大小写时，其下只差大小写的键也要拒绝。 +func TestR2NonCanonicalMetadataFieldDuplicates(t *testing.T) { + for _, field := range []string{"Metadata", "METADATA", "metaData"} { + t.Run(field, func(t *testing.T) { + y := "apiVersion: v1\nkind: Secret\n" + field + ":\n name: app\n Name: other\n namespace: ops\nstringData:\n k: " + redactFixtureNew + "\n" + p2MustReject(t, y) + if _, err := YAMLToSecret(y); err == nil { + t.Fatal("YAMLToSecret 应拒绝") + } + }) + } + // 对照：只有非标准写法、没有重复时照常接受（按 encoding/json 不区分大小写的规则归并） + t.Run("Metadata_alone_ok", func(t *testing.T) { + sec, err := YAMLToSecret("apiVersion: v1\nkind: Secret\nMetadata:\n name: app\n") + if err != nil || sec.Name != "app" { + t.Fatal("单独的 Metadata 应照常解析") + } + }) +} + +// P3-6：入口大小上限 2 MiB（2097152 字节，按 YAML 文本字节数计），超限在任何解析前报通用错误。 +func TestR2SizeLimit(t *testing.T) { + const limit = 2097152 + build := func(total int) string { + head := p2Head + "stringData:\n k: " + r2Sentinel + pad := total - len(head) - 1 + return head + strings.Repeat("a", pad) + "\n" + } + at := build(limit) + over := build(limit + 1) + if len(at) != limit || len(over) != limit+1 { + t.Fatal("构造长度不对") + } + t.Run("exactly_limit_accepted", func(t *testing.T) { + if _, err := p2Create(at); err != nil { + t.Fatalf("新建路径: %v", err) + } + if _, err := p2Update(at, p2Existing()); err != nil { + t.Fatalf("更新路径: %v", err) + } + if _, err := YAMLToSecret(at); err != nil { + t.Fatalf("YAMLToSecret: %v", err) + } + }) + t.Run("limit_plus_one_rejected", func(t *testing.T) { + _, errCreate := p2Create(over) + _, errUpdate := p2Update(over, p2Existing()) + _, errParse := YAMLToSecret(over) + for name, err := range map[string]error{"新建路径": errCreate, "更新路径": errUpdate, "YAMLToSecret": errParse} { + if !errors.Is(err, errSecretYAMLTooLarge) { + t.Fatalf("%s 应以大小错误拒绝，实际: %v", name, err) + } + r2AssertNoSentinel(t, err) + } + }) + t.Run("checked_before_parse", func(t *testing.T) { + // 语法错误且含哨兵：恰好 limit 字节时报语法错误（进入了解析）， + // 多 1 字节时报大小错误（未进入解析），两者文案不同且都不回显 + brokenOf := func(total int) string { + head := "stringData: {{k: " + r2Sentinel + return head + strings.Repeat("b", total-len(head)) + } + _, errSyntax := SecretYAMLHasRedactedPlaceholder(brokenOf(limit)) + _, errSize := SecretYAMLHasRedactedPlaceholder(brokenOf(limit + 1)) + if errSize == nil || errSyntax == nil { + t.Fatal("两者都应报错") + } + r2AssertNoSentinel(t, errSize) + r2AssertNoSentinel(t, errSyntax) + if errors.Is(errSyntax, errSecretYAMLTooLarge) { + t.Fatal("恰好 limit 字节的输入不应以大小错误拒绝") + } + if errSize.Error() != errSecretYAMLTooLarge.Error() { + t.Fatalf("超限输入应在解析前以固定的大小错误拒绝，实际: %v", errSize) + } + if !strings.Contains(errSize.Error(), "2 MiB") { + t.Fatal("应提示 2 MiB 上限") + } + }) +} diff --git a/internal/k8s/utils/secret_redact_test.go b/internal/k8s/utils/secret_redact_test.go new file mode 100644 index 00000000..f5e8d1c8 --- /dev/null +++ b/internal/k8s/utils/secret_redact_test.go @@ -0,0 +1,202 @@ +package utils + +import ( + "encoding/base64" + "reflect" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +// 测试用的明文（非真实凭据） +const ( + redactFixtureData = "plain-redact-data-fixture-01" + redactFixtureString = "plain-redact-string-fixture-02" + redactFixtureNew = "plain-redact-new-fixture-03" +) + +func parseSecretYAML(t *testing.T, y string) *corev1.Secret { + t.Helper() + var sec corev1.Secret + if err := yaml.Unmarshal([]byte(y), &sec); err != nil { + t.Fatalf("解析恢复后的 YAML 失败: %v", err) + } + return &sec +} + +// stringData 中的占位符在现有 Secret 的 stringData 中不存在、只在 data 中存在时， +// 应从 stringData 移除并以 data 形式恢复原值（secret_redact.go 中 stringData→data 分支）。 +func TestRestoreRedactedSecretYAMLStringDataPlaceholderFallsBackToData(t *testing.T) { + existing := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "ops"}, + Data: map[string][]byte{"token": []byte(redactFixtureData)}, + } + + t.Run("data 中未出现该键时补入 data", func(t *testing.T) { + in := "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\n namespace: ops\nstringData:\n token: " + SecretRedactedPlaceholder + "\n" + out, err := RestoreRedactedSecretYAML(in, existing) + if err != nil { + t.Fatal(err) + } + sec := parseSecretYAML(t, out) + if _, ok := sec.StringData["token"]; ok { + t.Fatal("占位符键应从 stringData 中移除") + } + if string(sec.Data["token"]) != redactFixtureData { + t.Fatalf("应以 data 形式恢复原值, got %d bytes", len(sec.Data["token"])) + } + if strings.Contains(out, SecretRedactedPlaceholder) { + t.Fatal("恢复后的 YAML 不应再含占位符") + } + }) + + t.Run("data 中已有同名新值时以 data 为准", func(t *testing.T) { + in := "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\n namespace: ops\ndata:\n token: " + + base64.StdEncoding.EncodeToString([]byte(redactFixtureNew)) + "\nstringData:\n token: " + SecretRedactedPlaceholder + "\n" + out, err := RestoreRedactedSecretYAML(in, existing) + if err != nil { + t.Fatal(err) + } + sec := parseSecretYAML(t, out) + if _, ok := sec.StringData["token"]; ok { + t.Fatal("占位符键应从 stringData 中移除，否则 K8s 合并时会覆盖 data") + } + if string(sec.Data["token"]) != redactFixtureNew { + t.Fatal("data 中用户填写的新值不应被原值覆盖") + } + }) + + t.Run("data 与 stringData 均不存在时拒绝", func(t *testing.T) { + in := "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\n namespace: ops\nstringData:\n missing: " + SecretRedactedPlaceholder + "\n" + if _, err := RestoreRedactedSecretYAML(in, existing); err == nil { + t.Fatal("未知键的占位符应报错") + } + }) +} + +// data 与 stringData 存在同名键时：脱敏 YAML 两处都替换为占位符；原样回写后两处各自恢复原值。 +func TestRedactAndRestoreSameKeyInDataAndStringData(t *testing.T) { + existing := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Secret"}, + ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "ops"}, + Data: map[string][]byte{"shared": []byte(redactFixtureData)}, + StringData: map[string]string{"shared": redactFixtureString}, + } + y, err := SecretToYAML(existing) + if err != nil { + t.Fatal(err) + } + for _, v := range []string{redactFixtureData, redactFixtureString, base64.StdEncoding.EncodeToString([]byte(redactFixtureData))} { + if strings.Contains(y, v) { + t.Fatal("脱敏 YAML 泄露了明文") + } + } + if strings.Count(y, "shared: "+SecretRedactedPlaceholder) != 2 { + t.Fatalf("data 与 stringData 的同名键都应为占位符:\n%s", y) + } + + out, err := RestoreRedactedSecretYAML(y, existing) + if err != nil { + t.Fatal(err) + } + sec := parseSecretYAML(t, out) + if string(sec.Data["shared"]) != redactFixtureData || sec.StringData["shared"] != redactFixtureString { + t.Fatal("同名键原样回写后 data 与 stringData 应各自恢复原值") + } + + if got := RedactSecretData(existing.Data); len(got["shared"]) != 0 { + t.Fatal("data 值应被清空") + } + if got := RedactSecretStringData(existing.StringData); got["shared"] != "" { + t.Fatal("stringData 值应被清空") + } + if k := SecretDataKeyInfos(existing.Data); len(k) != 1 || k[0].Size != len(redactFixtureData) { + t.Fatalf("data_keys 不符: %+v", k) + } + if k := SecretStringDataKeyInfos(existing.StringData); len(k) != 1 || k[0].Size != len(redactFixtureString) { + t.Fatalf("string_data_keys 不符: %+v", k) + } +} + +// secretYAMLDocument 必须与 corev1.Secret 的字段一一对应（名称、JSON 标签、类型；仅 data 的值类型不同）， +// 否则 k8s 升级新增字段时会被静默丢弃。 +func TestSecretYAMLDocumentMatchesCoreV1Secret(t *testing.T) { + want := reflect.TypeOf(corev1.Secret{}) + got := reflect.TypeOf(secretYAMLDocument{}) + if want.NumField() != got.NumField() { + t.Fatalf("字段数不一致: corev1.Secret=%d secretYAMLDocument=%d", want.NumField(), got.NumField()) + } + for i := 0; i < want.NumField(); i++ { + w, g := want.Field(i), got.Field(i) + if w.Name != g.Name || w.Anonymous != g.Anonymous || w.Tag.Get("json") != g.Tag.Get("json") { + t.Fatalf("第 %d 个字段不一致: %s %q vs %s %q", i, w.Name, w.Tag.Get("json"), g.Name, g.Tag.Get("json")) + } + if w.Name == "Data" { + continue + } + if w.Type != g.Type { + t.Fatalf("字段 %s 类型不一致: %s vs %s", w.Name, w.Type, g.Type) + } + } +} + +// 对合法的单文档 YAML，类型化解析结果必须与直接 yaml.Unmarshal 到 corev1.Secret 完全一致。 +func TestYAMLToSecretMatchesDirectUnmarshal(t *testing.T) { + newB64 := base64.StdEncoding.EncodeToString([]byte(redactFixtureNew)) + inputs := []string{ + "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\n namespace: ops\n labels:\n port: 8080\n on: yes\n annotations:\n a: \"1\"\ntype: Opaque\nimmutable: true\ndata:\n k: " + newB64 + "\n n: null\n e: \"\"\nstringData:\n s: " + redactFixtureString + "\n i: 12\n", + "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\nData:\n k: |\n " + newB64[:8] + "\n " + newB64[8:] + "\nStringData:\n s: x\n", + "---\nkind: Secret\nmetadata: {name: app, creationTimestamp: null}\ndata: {}\n", + "kind: Secret\n", + } + for i, y := range inputs { + var direct corev1.Secret + if err := yaml.Unmarshal([]byte(y), &direct); err != nil { + t.Fatalf("输入 %d 直接解析失败", i) + } + typed, err := YAMLToSecret(y) + if err != nil { + t.Fatalf("输入 %d 类型化解析失败: %v", i, err) + } + if !reflect.DeepEqual(&direct, typed) { + t.Fatalf("输入 %d 解析结果不一致", i) + } + } +} + +// 恢复后重新生成的 YAML 必须能被 YAMLToSecret 解析，且不含占位符。 +func TestRestoreRedactedSecretYAMLOutputReparses(t *testing.T) { + existing := &corev1.Secret{Data: map[string][]byte{"k": []byte(redactFixtureData)}} + in := "apiVersion: v1\nkind: Secret\nmetadata:\n name: app\nDATA:\n k: " + SecretRedactedPlaceholder + "\n" + out, err := RestoreRedactedSecretYAML(in, existing) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, SecretRedactedPlaceholder) { + t.Fatal("恢复结果仍含占位符") + } + sec, err := YAMLToSecret(out) + if err != nil { + t.Fatal(err) + } + if string(sec.Data["k"]) != redactFixtureData || sec.Name != "app" { + t.Fatal("恢复结果不对") + } +} + +func TestSanitizeSecretYAMLError(t *testing.T) { + err := sanitizeSecretYAMLError(errString("line 3: cannot decode !!str `" + redactFixtureNew + "` as a !!int; unsupported map key of type: , key: , value: \"" + redactFixtureNew + "\"")) + if strings.Contains(err.Error(), redactFixtureNew) { + t.Fatal("未抹掉回显的值") + } + if !strings.Contains(err.Error(), "line 3") { + t.Fatal("应保留位置信息") + } +} + +type errString string + +func (e errString) Error() string { return string(e) } diff --git a/internal/k8s/utils/serviceaccount.go b/internal/k8s/utils/serviceaccount.go index b6420234..6f723885 100644 --- a/internal/k8s/utils/serviceaccount.go +++ b/internal/k8s/utils/serviceaccount.go @@ -245,3 +245,53 @@ func CreateServiceAccountToken(ctx context.Context, kubeClient *kubernetes.Clien return resp, nil } + +// IsServiceAccountTokenSecretFor 判断 secret 是否是 sa 自己的 token Secret（SEC-29）： +// 类型必须是 kubernetes.io/service-account-token，注解 kubernetes.io/service-account.name 必须等于 SA 名称， +// 若存在注解 kubernetes.io/service-account.uid 则必须等于 SA 的 UID，且命名空间一致、token 非空。 +// sa.secrets 只是引用列表，任何能更新 SA 的人都能把别的 SA 的 token Secret 挂进来，因此不能仅凭引用信任。 +func IsServiceAccountTokenSecretFor(secret *corev1.Secret, sa *corev1.ServiceAccount) bool { + if secret == nil || sa == nil || sa.Name == "" { + return false + } + if secret.Type != corev1.SecretTypeServiceAccountToken { + return false + } + if secret.Namespace != "" && sa.Namespace != "" && secret.Namespace != sa.Namespace { + return false + } + if secret.Annotations[corev1.ServiceAccountNameKey] != sa.Name { + return false + } + if uid, ok := secret.Annotations[corev1.ServiceAccountUIDKey]; ok && uid != string(sa.UID) { + return false + } + return len(secret.Data[corev1.ServiceAccountTokenKey]) > 0 +} + +// SelectServiceAccountTokenSecret 从 SA 引用的 Secrets 中挑出第一个属于该 SA 的 token Secret； +// rejected 返回被拒绝的 service-account-token 类型 Secret 名称（只含名称，便于记录日志）。 +func SelectServiceAccountTokenSecret(sa *corev1.ServiceAccount, secrets []corev1.Secret) (selected *corev1.Secret, rejected []string) { + for i := range secrets { + secret := &secrets[i] + if IsServiceAccountTokenSecretFor(secret, sa) { + if selected == nil { + selected = secret + } + continue + } + if secret.Type == corev1.SecretTypeServiceAccountToken { + rejected = append(rejected, secret.Name) + } + } + return selected, rejected +} + +// ServiceAccountSecretRefNames 返回 SA secrets 引用中的名称列表 +func ServiceAccountSecretRefNames(refs []corev1.ObjectReference) []string { + names := make([]string, 0, len(refs)) + for _, ref := range refs { + names = append(names, ref.Name) + } + return names +} diff --git a/internal/k8s/utils/serviceaccount_token_test.go b/internal/k8s/utils/serviceaccount_token_test.go new file mode 100644 index 00000000..c0e5a257 --- /dev/null +++ b/internal/k8s/utils/serviceaccount_token_test.go @@ -0,0 +1,76 @@ +package utils + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func saTokenSecret(name string, ann map[string]string) corev1.Secret { + return corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ops", Annotations: ann}, + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{corev1.ServiceAccountTokenKey: []byte("fake-utils-token")}, + } +} + +func TestIsServiceAccountTokenSecretFor(t *testing.T) { + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "ops", UID: types.UID("uid-app")}} + own := map[string]string{corev1.ServiceAccountNameKey: "app", corev1.ServiceAccountUIDKey: "uid-app"} + for _, tt := range []struct { + name string + mutate func(*corev1.Secret) + want bool + }{ + {"own secret", func(*corev1.Secret) {}, true}, + {"no uid annotation", func(s *corev1.Secret) { delete(s.Annotations, corev1.ServiceAccountUIDKey) }, true}, + {"name mismatch", func(s *corev1.Secret) { s.Annotations[corev1.ServiceAccountNameKey] = "admin" }, false}, + {"uid mismatch", func(s *corev1.Secret) { s.Annotations[corev1.ServiceAccountUIDKey] = "uid-old" }, false}, + {"empty uid annotation", func(s *corev1.Secret) { s.Annotations[corev1.ServiceAccountUIDKey] = "" }, false}, + {"no annotations", func(s *corev1.Secret) { s.Annotations = nil }, false}, + {"opaque type", func(s *corev1.Secret) { s.Type = corev1.SecretTypeOpaque }, false}, + {"other namespace", func(s *corev1.Secret) { s.Namespace = "kube-system" }, false}, + {"empty token", func(s *corev1.Secret) { s.Data = map[string][]byte{corev1.ServiceAccountTokenKey: nil} }, false}, + } { + t.Run(tt.name, func(t *testing.T) { + ann := map[string]string{} + for k, v := range own { + ann[k] = v + } + s := saTokenSecret("s", ann) + tt.mutate(&s) + if got := IsServiceAccountTokenSecretFor(&s, sa); got != tt.want { + t.Fatalf("got %v want %v", got, tt.want) + } + }) + } + if IsServiceAccountTokenSecretFor(nil, sa) || IsServiceAccountTokenSecretFor(&corev1.Secret{}, nil) { + t.Fatal("nil 输入应返回 false") + } +} + +func TestSelectServiceAccountTokenSecret(t *testing.T) { + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "ops", UID: types.UID("uid-app")}} + opaque := saTokenSecret("cfg", nil) + opaque.Type = corev1.SecretTypeOpaque + secrets := []corev1.Secret{ + saTokenSecret("admin-token", map[string]string{corev1.ServiceAccountNameKey: "admin"}), + opaque, + saTokenSecret("app-token", map[string]string{corev1.ServiceAccountNameKey: "app", corev1.ServiceAccountUIDKey: "uid-app"}), + } + selected, rejected := SelectServiceAccountTokenSecret(sa, secrets) + if selected == nil || selected.Name != "app-token" { + t.Fatalf("应选中 SA 自己的 token Secret: %+v", selected) + } + if len(rejected) != 1 || rejected[0] != "admin-token" { + t.Fatalf("rejected 应只含借挂的 token Secret: %v", rejected) + } + if s, _ := SelectServiceAccountTokenSecret(sa, secrets[:2]); s != nil { + t.Fatal("没有属于自己的 token Secret 时应返回 nil") + } + if names := ServiceAccountSecretRefNames([]corev1.ObjectReference{{Name: "a"}, {Name: ""}, {Name: "b"}}); len(names) < 2 { + t.Fatalf("names=%v", names) + } +} diff --git a/internal/k8s/utils/serviceaccount_update.go b/internal/k8s/utils/serviceaccount_update.go new file mode 100644 index 00000000..a9a8c8d2 --- /dev/null +++ b/internal/k8s/utils/serviceaccount_update.go @@ -0,0 +1,103 @@ +/* + * MIT License + * + * Copyright (c) 2024 Bamboo + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +package utils + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/util/retry" +) + +// ServiceAccountMutateFunc 把请求中允许修改的字段应用到从集群读取的最新 SA 上。 +// 每次冲突重试都会以新读取的对象重新调用，实现必须是幂等的。 +type ServiceAccountMutateFunc func(latest *corev1.ServiceAccount) error + +// UpdateServiceAccountWithRetry 以 retry.RetryOnConflict(retry.DefaultRetry) 更新 SA： +// 每次尝试先 Get 最新版本（携带最新 resourceVersion），调用 mutate 应用允许修改的字段，再 Update。 +// 无论 mutate 做了什么，name 与 secrets 引用都以集群中最新版本为准（SEC-29：secrets 由集群维护）。 +// Get 失败（如 NotFound）直接返回且不调用 Update；连续冲突超过重试次数时返回最后一次的 Conflict 错误。 +func UpdateServiceAccountWithRetry(ctx context.Context, saClient corev1client.ServiceAccountInterface, name string, mutate ServiceAccountMutateFunc) (*corev1.ServiceAccount, error) { + var updated *corev1.ServiceAccount + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest, err := saClient.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + keptName, keptNamespace := latest.Name, latest.Namespace + keptSecrets := append([]corev1.ObjectReference(nil), latest.Secrets...) + if mutate != nil { + if err := mutate(latest); err != nil { + return err + } + } + latest.Name, latest.Namespace = keptName, keptNamespace + latest.Secrets = keptSecrets + updated, err = saClient.Update(ctx, latest, metav1.UpdateOptions{}) + return err + }) + if err != nil { + return nil, err + } + return updated, nil +} + +// UpdateServiceAccountWithResourceVersion 严格乐观锁更新：调用方提交了 resourceVersion（如 YAML 中的 metadata.resourceVersion）时使用。 +// 先 Get 当前对象作为基础（保留不可编辑字段与集群维护的 secrets 引用），调用 mutate 应用允许修改的字段， +// 再以调用方提交的 resourceVersion 只做一次 Update：版本过期时由 API Server 返回 409 Conflict，原样返回，不重试。 +// Get 失败（如 NotFound）直接返回且不调用 Update。 +func UpdateServiceAccountWithResourceVersion(ctx context.Context, saClient corev1client.ServiceAccountInterface, name, resourceVersion string, mutate ServiceAccountMutateFunc) (*corev1.ServiceAccount, error) { + latest, err := saClient.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + keptName, keptNamespace := latest.Name, latest.Namespace + keptSecrets := append([]corev1.ObjectReference(nil), latest.Secrets...) + if mutate != nil { + if err := mutate(latest); err != nil { + return nil, err + } + } + latest.Name, latest.Namespace = keptName, keptNamespace + latest.Secrets = keptSecrets + latest.ResourceVersion = resourceVersion + return saClient.Update(ctx, latest, metav1.UpdateOptions{}) +} + +// ApplyServiceAccountEditableFields 把 desired 中允许通过更新接口修改的字段整体覆盖到 latest： +// labels、annotations、automountServiceAccountToken、imagePullSecrets。 +// 其余字段（name/namespace、secrets、resourceVersion、uid、ownerReferences、finalizers 等）保持 latest 的值。 +func ApplyServiceAccountEditableFields(latest, desired *corev1.ServiceAccount) { + if latest == nil || desired == nil { + return + } + latest.Labels = desired.Labels + latest.Annotations = desired.Annotations + latest.AutomountServiceAccountToken = desired.AutomountServiceAccountToken + latest.ImagePullSecrets = desired.ImagePullSecrets +} diff --git a/internal/k8s/utils/serviceaccount_update_test.go b/internal/k8s/utils/serviceaccount_update_test.go new file mode 100644 index 00000000..5bdb387f --- /dev/null +++ b/internal/k8s/utils/serviceaccount_update_test.go @@ -0,0 +1,141 @@ +package utils + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/retry" +) + +var saGVR = corev1.SchemeGroupVersion.WithResource("serviceaccounts") + +func newRetrySA() *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "app", Namespace: "ops", ResourceVersion: "1", Labels: map[string]string{"v": "1"}}, + Secrets: []corev1.ObjectReference{{Name: "app-token"}}, + } +} + +func saUpdateActions(c *fake.Clientset) []k8stesting.UpdateAction { + var out []k8stesting.UpdateAction + for _, a := range c.Actions() { + if a.GetVerb() == "update" && a.GetResource().Resource == "serviceaccounts" { + out = append(out, a.(k8stesting.UpdateAction)) + } + } + return out +} + +func conflictErr() error { + return apierrors.NewConflict(schema.GroupResource{Resource: "serviceaccounts"}, "app", nil) +} + +// 第一次 Update 前集群里的 SA 被并发修改（resourceVersion 变为 2、多了一个 secrets 引用），Update 返回 Conflict； +// 重试时应重新 Get 最新版本，在其上应用字段后再 Update，且保留最新的 secrets 引用。 +func TestUpdateServiceAccountWithRetryConflictThenSuccess(t *testing.T) { + client := fake.NewSimpleClientset(newRetrySA()) + calls := 0 + client.PrependReactor("update", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + calls++ + if calls > 1 { + return false, nil, nil // 交给默认 tracker 处理 + } + concurrent := newRetrySA() + concurrent.ResourceVersion = "2" + concurrent.Labels = map[string]string{"v": "2"} + concurrent.Secrets = append(concurrent.Secrets, corev1.ObjectReference{Name: "app-token-rotated"}) + if err := client.Tracker().Update(saGVR, concurrent, "ops"); err != nil { + t.Errorf("模拟并发修改失败: %v", err) + } + return true, nil, conflictErr() + }) + + mutations := 0 + updated, err := UpdateServiceAccountWithRetry(context.Background(), client.CoreV1().ServiceAccounts("ops"), "app", func(latest *corev1.ServiceAccount) error { + mutations++ + latest.Labels = map[string]string{"team": "ops"} + latest.Secrets = []corev1.ObjectReference{{Name: "admin-token"}} // 企图篡改 secrets，必须被还原 + latest.Name = "admin" // 企图改名，必须被还原 + return nil + }) + if err != nil { + t.Fatal(err) + } + actions := saUpdateActions(client) + if calls != 2 || len(actions) != 2 || mutations != 2 { + t.Fatalf("应冲突一次后重试成功: updateCalls=%d actions=%d mutations=%d", calls, len(actions), mutations) + } + last := actions[1].GetObject().(*corev1.ServiceAccount) + if last.ResourceVersion != "2" { + t.Fatalf("重试应基于最新版本（resourceVersion=2），got %q", last.ResourceVersion) + } + if last.Name != "app" || last.Labels["team"] != "ops" { + t.Fatalf("name 应保持 URL 值、字段应被应用: %+v", last.ObjectMeta) + } + if len(last.Secrets) != 2 || last.Secrets[0].Name != "app-token" || last.Secrets[1].Name != "app-token-rotated" { + t.Fatalf("应保留集群最新的 secrets 引用: %+v", last.Secrets) + } + if updated == nil || len(updated.Secrets) != 2 { + t.Fatalf("返回值应为更新后的对象: %+v", updated) + } +} + +func TestUpdateServiceAccountWithRetryExhausted(t *testing.T) { + client := fake.NewSimpleClientset(newRetrySA()) + client.PrependReactor("update", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, conflictErr() + }) + _, err := UpdateServiceAccountWithRetry(context.Background(), client.CoreV1().ServiceAccounts("ops"), "app", func(*corev1.ServiceAccount) error { return nil }) + if !apierrors.IsConflict(err) { + t.Fatalf("连续冲突超过重试次数应返回 Conflict 错误, got %v", err) + } + if n := len(saUpdateActions(client)); n != retry.DefaultRetry.Steps { + t.Fatalf("应尝试 %d 次 Update, got %d", retry.DefaultRetry.Steps, n) + } +} + +func TestUpdateServiceAccountWithRetryNotFound(t *testing.T) { + client := fake.NewSimpleClientset() + called := false + _, err := UpdateServiceAccountWithRetry(context.Background(), client.CoreV1().ServiceAccounts("ops"), "app", func(*corev1.ServiceAccount) error { + called = true + return nil + }) + if !apierrors.IsNotFound(err) || called { + t.Fatalf("SA 不存在应返回 NotFound 且不调用 mutate: %v called=%v", err, called) + } + if n := len(saUpdateActions(client)); n != 0 { + t.Fatalf("SA 不存在时不应调用 Update, got %d", n) + } +} + +// 严格乐观锁：以调用方提交的 resourceVersion 只 Update 一次；Conflict 原样返回、不重试；保留集群 secrets 与 name/namespace +func TestUpdateServiceAccountWithResourceVersionSingleAttempt(t *testing.T) { + client := fake.NewSimpleClientset(newRetrySA()) + client.PrependReactor("update", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, conflictErr() + }) + _, err := UpdateServiceAccountWithResourceVersion(context.Background(), client.CoreV1().ServiceAccounts("ops"), "app", "7", func(sa *corev1.ServiceAccount) error { + sa.Name, sa.Namespace = "evil", "kube-system" + sa.Secrets = nil + return nil + }) + if !apierrors.IsConflict(err) { + t.Fatalf("应原样返回 Conflict: %v", err) + } + actions := saUpdateActions(client) + if len(actions) != 1 { + t.Fatalf("只应 Update 一次, got %d", len(actions)) + } + sent := actions[0].GetObject().(*corev1.ServiceAccount) + if sent.ResourceVersion != "7" || sent.Name != "app" || sent.Namespace != "ops" || len(sent.Secrets) != len(newRetrySA().Secrets) { + t.Fatalf("提交对象应携带调用方 resourceVersion 并保留不可编辑字段: %+v", sent) + } +} diff --git a/internal/middleware/auth.go b/internal/middleware/auth.go index 1b8ab33a..a02d787f 100644 --- a/internal/middleware/auth.go +++ b/internal/middleware/auth.go @@ -26,33 +26,14 @@ package middleware import ( - "strings" - + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" "github.com/GoSimplicity/AI-CloudOps/internal/system/service" "github.com/GoSimplicity/AI-CloudOps/pkg/base" "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" "github.com/gin-gonic/gin" ) -var skipAuthPaths = map[string]bool{ - "/api/user/login": true, - "/api/user/logout": true, - "/api/user/refresh_token": true, - "/api/user/profile": true, - "/api/user/codes": true, - "/api/not_auth/getBindIps": true, - "/api/not_auth/getTreeNodeBindIps": true, - "/favicon.ico": true, -} - -// HTTP方法映射 -var methodMapping = map[string]int8{ - "GET": 1, - "POST": 2, - "PUT": 3, - "DELETE": 4, -} - type AuthMiddleware struct { roleService service.RoleService } @@ -63,66 +44,14 @@ func NewAuthMiddleware(roleService service.RoleService) *AuthMiddleware { } } -func matchWildcardPath(apiPath, requestPath string, methodCode int8, apiMethod int8) bool { - // 方法不匹配则返回false - if apiMethod != methodCode { - return false - } - - // 完全匹配 - if apiPath == requestPath { - return true - } - - // 全局通配符匹配所有路径 - if apiPath == "/*" { - return true - } - - if !strings.Contains(apiPath, "*") { - // 代理入口只有通配路由，目录中的参数必须严格占用一个非空路径段 - granted, requested := strings.Split(apiPath, "/"), strings.Split(requestPath, "/") - if len(granted) != len(requested) { - return false - } - for index, part := range granted { - if strings.HasPrefix(part, ":") && len(part) > 1 { - if requested[index] == "" || requested[index] == "." || requested[index] == ".." { - return false - } - } else if part != requested[index] { - return false - } - } - return true - } - - // 末尾通配符：/api/user/* - if strings.HasSuffix(apiPath, "*") { - prefix := strings.TrimSuffix(apiPath, "*") - return strings.HasPrefix(requestPath, prefix) - } - - // 开头通配符：*/logs - if strings.HasPrefix(apiPath, "*") { - suffix := strings.TrimPrefix(apiPath, "*") - return strings.HasSuffix(requestPath, suffix) - } - - // 中间通配符：/api/*/logs - if strings.Count(apiPath, "*") == 1 { - parts := strings.Split(apiPath, "*") - return strings.HasPrefix(requestPath, parts[0]) && strings.HasSuffix(requestPath, parts[1]) - } - - return false -} - +// CheckAuth 按“HTTP 方法 + gin 路由模板（c.FullPath()）”对角色授予做精确比较。 +// 授予中的 /* 与末尾 * 只对模板做前缀匹配，:param 不再被当作通配符； +// 匹配规则见 internal/permission 与 docs/api/user-permissions-v2.md。 func (am *AuthMiddleware) CheckAuth() gin.HandlerFunc { return func(c *gin.Context) { path := c.Request.URL.Path - if skipAuthPaths[path] { + if permission.SkipsAuthorization(path) { c.Next() return } @@ -136,11 +65,6 @@ func (am *AuthMiddleware) CheckAuth() gin.HandlerFunc { // 获取用户信息（兼容nginx代理） userVal, exists := c.Get("user") if !exists { - // 兼容未登录时放行登录接口 - if skipAuthPaths[path] { - c.Next() - return - } base.ForbiddenError(c, "未登录或登录已过期") c.Abort() return @@ -152,47 +76,54 @@ func (am *AuthMiddleware) CheckAuth() gin.HandlerFunc { return } - if path == "/api/user/signup" && user.Username != "admin" { + // 特权只来自内置超管角色（code=super_admin 且 is_system=1 且启用），不再按用户名判断 + roles, err := am.roleService.GetUserRoles(c, user.Uid) + if err != nil { + base.ErrorWithMessage(c, "获取用户角色失败") + c.Abort() + return + } + if roles == nil { + roles = &model.ListResp[*model.Role]{} + } + super := permission.HasSuperAdminRole(roles.Items) + permission.MarkSuperAdmin(c, super) + + if path == "/api/user/signup" && !super { base.ForbiddenError(c, "仅管理员可创建用户") c.Abort() return } - // 管理员放行 - if user.Username == "admin" { + // 超级管理员放行（未命中路由时交给 gin 返回 404） + if super { c.Next() return } // 获取HTTP方法代码 method := c.Request.Method - methodCode, exists := methodMapping[method] + methodCode, exists := permission.MethodCode(method) if !exists { base.ErrorWithMessage(c, "不支持的HTTP方法") c.Abort() return } - roles, err := am.roleService.GetUserRoles(c, user.Uid) - if err != nil { - base.ErrorWithMessage(c, "获取用户角色失败") + // 未命中任何路由（404）时 FullPath 为空：对非超管统一返回 403， + // 既不放行也不暴露路由是否存在。 + fullPath := c.FullPath() + if fullPath == "" { + base.ForbiddenError(c, "无权限访问该接口") c.Abort() return } - for _, role := range roles.Items { - // 跳过禁用角色 - if role.Status != 1 { - continue - } - - // 检查API权限 - for _, api := range role.Apis { - if (api.Path == c.FullPath() && api.Method == methodCode) || matchWildcardPath(api.Path, path, methodCode, api.Method) { - c.Next() - return - } - } + // 代理通配入口（AIOps）借助接口目录把真实路径还原为具体模板 + resolved := permission.ResolveProxyTemplate(method, fullPath, path) + if permission.Allowed(permission.GrantsFromRoles(roles.Items), methodCode, fullPath, resolved) { + c.Next() + return } // 无权限访问 diff --git a/internal/middleware/auth_regression_test.go b/internal/middleware/auth_regression_test.go new file mode 100644 index 00000000..113c5e47 --- /dev/null +++ b/internal/middleware/auth_regression_test.go @@ -0,0 +1,217 @@ +package middleware + +import ( + "net/http/httptest" + "testing" + + k8sapi "github.com/GoSimplicity/AI-CloudOps/internal/k8s/api" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" + treeapi "github.com/GoSimplicity/AI-CloudOps/internal/tree/api" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" +) + +// productionRouter 用真实的 handler RegisterRouters 注册路由（与线上同一份路由表）， +// 在鉴权之后用一个终止中间件代替业务 handler：放行返回 204 并记录命中的模板， +// 拒绝则由鉴权中间件返回 403。 +func productionRouter(t *testing.T, grants []*model.Api) (*gin.Engine, *string) { + t.Helper() + gin.SetMode(gin.TestMode) + engine := gin.New() + matched := new(string) + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 42, Username: "operator"}); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{roles: []*model.Role{{Status: 1, Apis: grants}}}).CheckAuth()) + engine.Use(func(c *gin.Context) { *matched = c.FullPath(); c.AbortWithStatus(204) }) + k8sapi.NewK8sEventHandler(nil).RegisterRouters(engine) + treeapi.NewCloudAccountHandler(nil).RegisterRouters(engine) + treeapi.NewTreeCloudHandler(nil, nil, nil, nil).RegisterRouters(engine) + return engine, matched +} + +func TestParamGrantsDoNotLeakToSiblingStaticRoutes(t *testing.T) { + for _, tt := range []struct { + name string + grant model.Api + method string + allowed string // 授予本身覆盖的请求 + allowedTmpl string + leaked string // 改动前会被授予外溢放行的请求 + leakedTmpl string + }{ + { + name: "k8s event detail vs node events (event.go:49 vs :54)", + grant: model.Api{Method: 1, Path: "/api/k8s/clusters/:cluster_id/events/:namespace/:name"}, + method: "GET", + allowed: "/api/k8s/clusters/1/events/default/pod-oom.17a", + allowedTmpl: "/api/k8s/clusters/:cluster_id/events/:namespace/:name", + leaked: "/api/k8s/clusters/1/events/nodes/worker-1", + leakedTmpl: "/api/k8s/clusters/:cluster_id/events/nodes/:node_name", + }, + { + name: "cloud account status vs batch status (cloud_account_handler.go:54 vs :57)", + grant: model.Api{Method: 3, Path: "/api/tree/cloud/account/:id/status"}, + method: "PUT", + allowed: "/api/tree/cloud/account/7/status", + allowedTmpl: "/api/tree/cloud/account/:id/status", + leaked: "/api/tree/cloud/account/batch/status", + leakedTmpl: "/api/tree/cloud/account/batch/status", + }, + { + name: "cloud resource status vs batch status (tree_cloud_handler.go:69 vs :74)", + grant: model.Api{Method: 3, Path: "/api/tree/cloud/:id/status"}, + method: "PUT", + allowed: "/api/tree/cloud/9/status", + allowedTmpl: "/api/tree/cloud/:id/status", + leaked: "/api/tree/cloud/batch/status", + leakedTmpl: "/api/tree/cloud/batch/status", + }, + } { + t.Run(tt.name, func(t *testing.T) { + grant := tt.grant + engine, matched := productionRouter(t, []*model.Api{&grant}) + + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest(tt.method, tt.allowed, nil)) + if rec.Code != 204 || *matched != tt.allowedTmpl { + t.Fatalf("granted request: status=%d template=%q", rec.Code, *matched) + } + + // 确认泄露请求确实命中的是兄弟静态路由，而不是 404 + *matched = "" + rec = httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest(tt.method, tt.leaked, nil)) + if rec.Code != 403 { + t.Fatalf("sibling route %s leaked: status=%d", tt.leakedTmpl, rec.Code) + } + if !routeRegistered(engine, tt.method, tt.leakedTmpl) { + t.Fatalf("test setup: %s %s is not registered", tt.method, tt.leakedTmpl) + } + + // 显式授予兄弟路由后才放行，说明拒绝来自授权而非路由 + sibling := model.Api{Method: grant.Method, Path: tt.leakedTmpl} + engine, matched = productionRouter(t, []*model.Api{&grant, &sibling}) + rec = httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest(tt.method, tt.leaked, nil)) + if rec.Code != 204 || *matched != tt.leakedTmpl { + t.Fatalf("explicit sibling grant: status=%d template=%q", rec.Code, *matched) + } + }) + } +} + +func routeRegistered(engine *gin.Engine, method, path string) bool { + for _, route := range engine.Routes() { + if route.Method == method && route.Path == path { + return true + } + } + return false +} + +func TestWildcardGrantsMatchTemplatesOnly(t *testing.T) { + for _, tt := range []struct { + name string + grant model.Api + method string + path string + want int + }{ + {"global wildcard", model.Api{Method: 1, Path: "/*"}, "GET", "/api/k8s/clusters/1/events/nodes/worker-1", 204}, + {"global wildcard keeps method", model.Api{Method: 1, Path: "/*"}, "PUT", "/api/tree/cloud/batch/status", 403}, + {"module prefix", model.Api{Method: 3, Path: "/api/tree/cloud/account/*"}, "PUT", "/api/tree/cloud/account/batch/status", 204}, + {"module prefix does not reach parent module", model.Api{Method: 3, Path: "/api/tree/cloud/account/*"}, "PUT", "/api/tree/cloud/batch/status", 403}, + // 前缀只作用于模板：真实路径里的值不能“拼”出被授予的前缀 + {"prefix on real id is not honoured", model.Api{Method: 3, Path: "/api/tree/cloud/9/*"}, "PUT", "/api/tree/cloud/9/status", 403}, + {"prefix written with template param", model.Api{Method: 3, Path: "/api/tree/cloud/:id/*"}, "PUT", "/api/tree/cloud/9/status", 204}, + {"prefix with param does not cover static sibling", model.Api{Method: 3, Path: "/api/tree/cloud/:id/*"}, "PUT", "/api/tree/cloud/batch/status", 403}, + {"exact real path is not a template", model.Api{Method: 3, Path: "/api/tree/cloud/9/status"}, "PUT", "/api/tree/cloud/9/status", 403}, + {"leading wildcard dropped", model.Api{Method: 3, Path: "*/status"}, "PUT", "/api/tree/cloud/9/status", 403}, + {"middle wildcard dropped", model.Api{Method: 3, Path: "/api/tree/*/status"}, "PUT", "/api/tree/cloud/:id/status", 403}, + } { + t.Run(tt.name, func(t *testing.T) { + grant := tt.grant + engine, _ := productionRouter(t, []*model.Api{&grant}) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest(tt.method, tt.path, nil)) + if rec.Code != tt.want { + t.Fatalf("status=%d want=%d", rec.Code, tt.want) + } + }) + } +} + +func TestUnmatchedRouteIsForbiddenForOrdinaryUsersAndNotFoundForSuperAdmin(t *testing.T) { + for _, tt := range []struct { + name string + roles []*model.Role + want int + }{ + {"global wildcard grant", []*model.Role{{Status: 1, Apis: []*model.Api{{Method: 1, Path: "/*"}}}}, 403}, + {"no roles", nil, 403}, + {"super admin", []*model.Role{{Code: permission.SuperAdminRoleCode, IsSystem: 1, Status: 1}}, 404}, + } { + t.Run(tt.name, func(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 1}); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{roles: tt.roles}).CheckAuth()) + engine.GET("/api/item/:id", func(c *gin.Context) { c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", "/api/no-such-route", nil)) + if rec.Code != tt.want { + t.Fatalf("status=%d want=%d", rec.Code, tt.want) + } + }) + } +} + +func TestProxyGrantsResolveToMostSpecificCatalogTemplate(t *testing.T) { + useProxyCatalog(t, []permission.Route{ + {Method: "GET", Path: "/api/v1/inspection/report/:report_id"}, + {Method: "GET", Path: "/api/v1/inspection/report/latest"}, + {Method: "GET", Path: "/api/v1/inspection/report/:report_id/markdown"}, + {Method: "POST", Path: "/api/v1/autofix"}, + {Method: "POST", Path: "/api/v1/autofix/"}, + }) + for _, tt := range []struct { + name string + grants []*model.Api + method string + path string + want int + }{ + {"param grant", []*model.Api{{Method: 1, Path: "/api/v1/inspection/report/:report_id"}}, "GET", "/api/v1/inspection/report/r-1", 204}, + {"param grant does not reach static sibling", []*model.Api{{Method: 1, Path: "/api/v1/inspection/report/:report_id"}}, "GET", "/api/v1/inspection/report/latest", 403}, + {"param grant does not reach deeper template", []*model.Api{{Method: 1, Path: "/api/v1/inspection/report/:report_id"}}, "GET", "/api/v1/inspection/report/r-1/markdown", 403}, + {"static grant", []*model.Api{{Method: 1, Path: "/api/v1/inspection/report/latest"}}, "GET", "/api/v1/inspection/report/latest", 204}, + {"prefix grant", []*model.Api{{Method: 1, Path: "/api/v1/inspection/*"}}, "GET", "/api/v1/inspection/report/r-1/markdown", 204}, + {"prefix grant ignores uncatalogued endpoints", []*model.Api{{Method: 1, Path: "/api/v1/inspection/*"}}, "GET", "/api/v1/inspection/internal/debug", 403}, + {"module grant covers uncatalogued endpoints", []*model.Api{{Method: 1, Path: "/api/v1/*path"}}, "GET", "/api/v1/inspection/internal/debug", 204}, + {"trailing slash alias is distinct", []*model.Api{{Method: 2, Path: "/api/v1/autofix"}}, "POST", "/api/v1/autofix/", 403}, + {"trailing slash alias granted", []*model.Api{{Method: 2, Path: "/api/v1/autofix/"}}, "POST", "/api/v1/autofix/", 204}, + } { + t.Run(tt.name, func(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 7}); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{roles: []*model.Role{{Status: 1, Apis: tt.grants}}}).CheckAuth()) + engine.Any(permission.AIOpsProxyTemplate, func(c *gin.Context) { c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest(tt.method, tt.path, nil)) + if rec.Code != tt.want { + t.Fatalf("status=%d want=%d", rec.Code, tt.want) + } + }) + } +} + +func TestDisabledRoleGrantsNothing(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 7}); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{roles: []*model.Role{{Status: 0, Apis: []*model.Api{{Method: 1, Path: "/*"}}}}}).CheckAuth()) + engine.GET("/api/item/:id", func(c *gin.Context) { c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", "/api/item/1", nil)) + if rec.Code != 403 { + t.Fatalf("status=%d", rec.Code) + } +} diff --git a/internal/middleware/auth_sa_token_test.go b/internal/middleware/auth_sa_token_test.go new file mode 100644 index 00000000..0ea0ddba --- /dev/null +++ b/internal/middleware/auth_sa_token_test.go @@ -0,0 +1,73 @@ +package middleware + +import ( + "net/http/httptest" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" +) + +// SEC-29：GET（token Secret）与 POST（TokenRequest）/token 只接受按路由模板 + 方法的显式授权 +func TestServiceAccountTokenRequiresExplicitGrant(t *testing.T) { + const ( + detailRoute = "/api/k8s/serviceaccount/:cluster_id/:namespace/:name/detail" + tokenPath = "/api/k8s/serviceaccount/1/ops/app/token" + detailPath = "/api/k8s/serviceaccount/1/ops/app/detail" + ) + for _, tt := range []struct { + name string + grants []*model.Api + method string + path string + want int + }{ + {"explicit GET grant", []*model.Api{{Path: model.K8sServiceAccountTokenRoute, Method: 1}}, "GET", tokenPath, 204}, + {"explicit POST grant", []*model.Api{{Path: model.K8sServiceAccountTokenRoute, Method: 2}}, "POST", tokenPath, 204}, + {"GET grant does not cover POST TokenRequest", []*model.Api{{Path: model.K8sServiceAccountTokenRoute, Method: 1}}, "POST", tokenPath, 403}, + {"POST grant does not cover GET", []*model.Api{{Path: model.K8sServiceAccountTokenRoute, Method: 2}}, "GET", tokenPath, 403}, + {"module wildcard does not cover GET token", []*model.Api{{Path: "/api/k8s/*", Method: 1}}, "GET", tokenPath, 403}, + {"module wildcard does not cover POST token", []*model.Api{{Path: "/api/k8s/*", Method: 2}}, "POST", tokenPath, 403}, + {"global wildcard does not cover GET token", []*model.Api{{Path: "/*", Method: 1}}, "GET", tokenPath, 403}, + {"global wildcard does not cover POST token", []*model.Api{{Path: "/*", Method: 2}}, "POST", tokenPath, 403}, + {"parameterized action path does not cover token", []*model.Api{{Path: "/api/k8s/serviceaccount/:cluster_id/:namespace/:name/:action", Method: 1}}, "GET", tokenPath, 403}, + {"renamed params do not cover token", []*model.Api{{Path: "/api/k8s/serviceaccount/:a/:b/:c/token", Method: 2}}, "POST", tokenPath, 403}, + {"fully parameterized path does not cover token", []*model.Api{{Path: "/:a/:b/:c/:d/:e/:f/:g", Method: 1}}, "GET", tokenPath, 403}, + {"concrete request path does not cover token", []*model.Api{{Path: tokenPath, Method: 1}}, "GET", tokenPath, 403}, + {"middle wildcard does not cover token", []*model.Api{{Path: "/api/k8s/*/token", Method: 1}}, "GET", tokenPath, 403}, + {"detail grant does not cover token", []*model.Api{{Path: detailRoute, Method: 1}}, "GET", tokenPath, 403}, + {"exact grant among wildcard grants", []*model.Api{{Path: "/api/k8s/*", Method: 1}, {Path: model.K8sServiceAccountTokenRoute, Method: 1}}, "GET", tokenPath, 204}, + {"module wildcard still covers detail", []*model.Api{{Path: "/api/k8s/*", Method: 1}}, "GET", detailPath, 204}, + } { + t.Run(tt.name, func(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 7, Username: "operator"}); c.Next() }) + engine.Use(NewAuthMiddleware(aiProxyRoles{apis: tt.grants}).CheckAuth()) + ok := func(c *gin.Context) { c.Status(204) } + engine.GET(model.K8sServiceAccountTokenRoute, ok) + engine.POST(model.K8sServiceAccountTokenRoute, ok) + engine.GET(detailRoute, ok) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest(tt.method, tt.path, nil)) + if rec.Code != tt.want { + t.Fatalf("status=%d want=%d", rec.Code, tt.want) + } + }) + } +} + +func TestServiceAccountTokenAdminBypass(t *testing.T) { + for _, method := range []string{"GET", "POST"} { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 1, Username: "admin"}); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{roles: []*model.Role{{Code: permission.SuperAdminRoleCode, IsSystem: 1, Status: 1}}}).CheckAuth()) + engine.Handle(method, model.K8sServiceAccountTokenRoute, func(c *gin.Context) { c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest(method, "/api/k8s/serviceaccount/1/ops/app/token", nil)) + if rec.Code != 204 { + t.Fatalf("%s admin 应直接放行, status=%d", method, rec.Code) + } + } +} diff --git a/internal/middleware/auth_secret_reveal_test.go b/internal/middleware/auth_secret_reveal_test.go new file mode 100644 index 00000000..a6cb6dd2 --- /dev/null +++ b/internal/middleware/auth_secret_reveal_test.go @@ -0,0 +1,61 @@ +package middleware + +import ( + "net/http/httptest" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" +) + +func TestSecretRevealRequiresExplicitGrant(t *testing.T) { + const detailRoute = "/api/k8s/secret/:cluster_id/:namespace/:name/detail" + for _, tt := range []struct { + name string + grants []*model.Api + path string + want int + }{ + {"explicit reveal grant", []*model.Api{{Path: model.K8sSecretRevealRoute, Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 204}, + {"module wildcard does not cover reveal", []*model.Api{{Path: "/api/k8s/*", Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 403}, + {"global wildcard does not cover reveal", []*model.Api{{Path: "/*", Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 403}, + {"detail grant does not cover reveal", []*model.Api{{Path: detailRoute, Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 403}, + {"module wildcard still covers detail", []*model.Api{{Path: "/api/k8s/*", Method: 1}}, "/api/k8s/secret/1/ops/app/detail", 204}, + // 参数化路径（不含 *）在 matchWildcardPath 中按段匹配，但敏感路由只认与路由模板逐字相等的授权 + {"parameterized action path does not cover reveal", []*model.Api{{Path: "/api/k8s/secret/:cluster_id/:namespace/:name/:action", Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 403}, + {"renamed params do not cover reveal", []*model.Api{{Path: "/api/k8s/secret/:a/:b/:c/reveal", Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 403}, + {"fully parameterized path does not cover reveal", []*model.Api{{Path: "/:a/:b/:c/:d/:e/:f/:g", Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 403}, + {"concrete request path does not cover reveal", []*model.Api{{Path: "/api/k8s/secret/1/ops/app/reveal", Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 403}, + {"middle wildcard does not cover reveal", []*model.Api{{Path: "/api/k8s/*/reveal", Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 403}, + {"reveal route with wrong method does not cover reveal", []*model.Api{{Path: model.K8sSecretRevealRoute, Method: 2}}, "/api/k8s/secret/1/ops/app/reveal", 403}, + {"exact grant among wildcard grants", []*model.Api{{Path: "/*", Method: 1}, {Path: model.K8sSecretRevealRoute, Method: 1}}, "/api/k8s/secret/1/ops/app/reveal", 204}, + {"renamed parameter grant does not cover detail", []*model.Api{{Path: "/api/k8s/secret/:a/:b/:c/detail", Method: 1}}, "/api/k8s/secret/1/ops/app/detail", 403}, + } { + t.Run(tt.name, func(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 7, Username: "operator"}); c.Next() }) + engine.Use(NewAuthMiddleware(aiProxyRoles{apis: tt.grants}).CheckAuth()) + engine.GET(model.K8sSecretRevealRoute, func(c *gin.Context) { c.Status(204) }) + engine.GET(detailRoute, func(c *gin.Context) { c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", tt.path, nil)) + if rec.Code != tt.want { + t.Fatalf("status=%d want=%d", rec.Code, tt.want) + } + }) + } +} + +func TestSecretRevealAdminBypass(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 1, Username: "admin"}); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{roles: []*model.Role{{Code: permission.SuperAdminRoleCode, IsSystem: 1, Status: 1}}}).CheckAuth()) + engine.GET(model.K8sSecretRevealRoute, func(c *gin.Context) { c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", "/api/k8s/secret/1/ops/app/reveal", nil)) + if rec.Code != 204 { + t.Fatalf("admin 应直接放行, status=%d", rec.Code) + } +} diff --git a/internal/middleware/auth_sensitive_admin_test.go b/internal/middleware/auth_sensitive_admin_test.go new file mode 100644 index 00000000..2fe7a863 --- /dev/null +++ b/internal/middleware/auth_sensitive_admin_test.go @@ -0,0 +1,40 @@ +package middleware + +import ( + "net/http/httptest" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" +) + +func TestAdminUsernameDoesNotBypassSensitiveGrants(t *testing.T) { + for _, endpoint := range []struct { + method, route, path string + }{ + {"GET", model.K8sSecretRevealRoute, "/api/k8s/secret/1/ops/app/reveal"}, + {"GET", model.K8sServiceAccountTokenRoute, "/api/k8s/serviceaccount/1/ops/app/token"}, + {"POST", model.K8sServiceAccountTokenRoute, "/api/k8s/serviceaccount/1/ops/app/token"}, + } { + for _, roles := range []struct { + name string + items []*model.Role + }{ + {name: "no roles"}, + {name: "wildcard grants", items: []*model.Role{{Status: 1, Apis: []*model.Api{{Method: 1, Path: "/*"}, {Method: 2, Path: "/*"}}}}}, + } { + t.Run(endpoint.method+" "+endpoint.route+"/"+roles.name, func(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 1, Username: "admin"}); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{roles: roles.items}).CheckAuth()) + engine.Handle(endpoint.method, endpoint.route, func(c *gin.Context) { c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest(endpoint.method, endpoint.path, nil)) + if rec.Code != 403 { + t.Fatalf("admin without a superadmin role reached sensitive endpoint: status=%d", rec.Code) + } + }) + } + } +} diff --git a/internal/middleware/auth_superadmin_test.go b/internal/middleware/auth_superadmin_test.go new file mode 100644 index 00000000..2d6dc562 --- /dev/null +++ b/internal/middleware/auth_superadmin_test.go @@ -0,0 +1,87 @@ +package middleware + +import ( + "context" + "errors" + "net/http/httptest" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" + "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" +) + +// fixedRoles 返回固定角色集合的 RoleService 假实现；err 非空时模拟数据库故障。 +type fixedRoles struct { + service.RoleService + roles []*model.Role + err error +} + +func (r fixedRoles) GetUserRoles(context.Context, int) (*model.ListResp[*model.Role], error) { + if r.err != nil { + return nil, r.err + } + return &model.ListResp[*model.Role]{Items: r.roles}, nil +} + +func TestPrivilegeComesFromSuperAdminRoleNotUsername(t *testing.T) { + superRole := &model.Role{Code: permission.SuperAdminRoleCode, IsSystem: 1, Status: 1} + for _, tt := range []struct { + name string + claims jwt.UserClaims + roles []*model.Role + status int + super bool + }{ + {"admin username alone is ordinary", jwt.UserClaims{Uid: 1, Username: "admin"}, nil, 403, false}, + {"admin with super role", jwt.UserClaims{Uid: 1, Username: "admin"}, []*model.Role{superRole}, 204, true}, + {"renamed account with super role", jwt.UserClaims{Uid: 9, Username: "ops-lead"}, []*model.Role{superRole}, 204, true}, + {"custom role reusing super code", jwt.UserClaims{Uid: 9, Username: "ops"}, []*model.Role{{Code: permission.SuperAdminRoleCode, IsSystem: 0, Status: 1}}, 403, false}, + {"disabled super role", jwt.UserClaims{Uid: 1, Username: "admin"}, []*model.Role{{Code: permission.SuperAdminRoleCode, IsSystem: 1, Status: 0}}, 403, false}, + } { + t.Run(tt.name, func(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", tt.claims); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{roles: tt.roles}).CheckAuth()) + var sawSuper bool + engine.DELETE("/api/user/:id/delete", func(c *gin.Context) { sawSuper = permission.IsSuperAdmin(c); c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("DELETE", "/api/user/3/delete", nil)) + if rec.Code != tt.status { + t.Fatalf("status=%d want=%d", rec.Code, tt.status) + } + if tt.status == 204 && sawSuper != tt.super { + t.Fatalf("super mark=%v want=%v", sawSuper, tt.super) + } + }) + } +} + +func TestOrdinaryGrantMarksRequestAsNotSuperAdmin(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 7, Username: "reader"}); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{roles: []*model.Role{{Status: 1, Apis: []*model.Api{{Path: "/api/item/:id", Method: 1}}}}}).CheckAuth()) + sawSuper := true + engine.GET("/api/item/:id", func(c *gin.Context) { sawSuper = permission.IsSuperAdmin(c); c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", "/api/item/1", nil)) + if rec.Code != 204 || sawSuper { + t.Fatalf("status=%d super=%v", rec.Code, sawSuper) + } +} + +func TestRoleLookupFailureFailsClosed(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 1, Username: "admin"}); c.Next() }) + engine.Use(NewAuthMiddleware(fixedRoles{err: errors.New("db down")}).CheckAuth()) + called := false + engine.GET("/api/item/:id", func(c *gin.Context) { called = true; c.Status(204) }) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", "/api/item/1", nil)) + if called { + t.Fatalf("handler reached despite role lookup failure (status=%d)", rec.Code) + } +} diff --git a/internal/middleware/auth_test.go b/internal/middleware/auth_test.go index 3ac215ea..0532ad7c 100644 --- a/internal/middleware/auth_test.go +++ b/internal/middleware/auth_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" "github.com/GoSimplicity/AI-CloudOps/internal/system/service" "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" "github.com/gin-gonic/gin" @@ -44,6 +45,13 @@ func (r aiProxyRoles) GetUserRoles(context.Context, int) (*model.ListResp[*model return &model.ListResp[*model.Role]{Items: []*model.Role{{Status: 1, Apis: r.apis}}}, nil } func TestAssistantProxyPermissionsDoNotGrantInfrastructureActions(t *testing.T) { + // 代理入口只有通配模板，鉴权依赖启动时登记的 AIOps 接口目录（真实目录见 internal/app 的测试） + useProxyCatalog(t, []permission.Route{ + {Method: "POST", Path: "/api/v1/assistant/query"}, + {Method: "GET", Path: "/api/v1/assistant/session/:session_id"}, + {Method: "POST", Path: "/api/v1/autofix/workflow/confirm"}, + {Method: "POST", Path: "/api/v1/autofix/notify"}, + }) grants := []*model.Api{{Path: "/api/v1/assistant/query", Method: 2}, {Path: "/api/v1/assistant/session/:session_id", Method: 1}} for _, test := range []struct { method, path string @@ -73,6 +81,13 @@ func TestAssistantProxyPermissionsDoNotGrantInfrastructureActions(t *testing.T) } } +func useProxyCatalog(t *testing.T, routes []permission.Route) { + t.Helper() + previous := permission.ProxyCatalog(permission.AIOpsProxyTemplate) + permission.SetProxyCatalog(permission.AIOpsProxyTemplate, routes) + t.Cleanup(func() { permission.SetProxyCatalog(permission.AIOpsProxyTemplate, previous) }) +} + func TestLegacyModuleGrantRemainsExplicitlyBroad(t *testing.T) { engine := gin.New() engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 7, Username: "operator"}); c.Next() }) diff --git a/internal/middleware/signup_test.go b/internal/middleware/signup_test.go index e70012ce..fdfaf1bc 100644 --- a/internal/middleware/signup_test.go +++ b/internal/middleware/signup_test.go @@ -4,20 +4,30 @@ import ( "net/http/httptest" "testing" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" ijwt "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" "github.com/gin-gonic/gin" ) func TestSignupRequiresAuthenticatedAdministrator(t *testing.T) { + superRole := &model.Role{Code: permission.SuperAdminRoleCode, IsSystem: 1, Status: 1} + signupGrant := &model.Role{Status: 1, Apis: []*model.Api{{Path: "/api/user/signup", Method: 2}, {Path: "/*", Method: 2}}} for _, tt := range []struct { name string claims *ijwt.UserClaims + roles []*model.Role status int }{ - {"anonymous", nil, 401}, - {"ordinary account", &ijwt.UserClaims{Username: "member", AccountType: 1}, 403}, - {"service account", &ijwt.UserClaims{Username: "service", AccountType: 2}, 403}, - {"administrator", &ijwt.UserClaims{Username: "admin"}, 204}, + {"anonymous", nil, nil, 401}, + {"ordinary account", &ijwt.UserClaims{Uid: 2, Username: "member", AccountType: 1}, nil, 403}, + {"service account", &ijwt.UserClaims{Uid: 3, Username: "service", AccountType: 2}, nil, 403}, + // 角色里显式授予 signup 也不够：建号只属于超管 + {"granted but not super admin", &ijwt.UserClaims{Uid: 4, Username: "operator"}, []*model.Role{signupGrant}, 403}, + // 用户名叫 admin 但没有超管角色：不再有任何特权 + {"username admin without role", &ijwt.UserClaims{Uid: 1, Username: "admin"}, nil, 403}, + {"super admin role", &ijwt.UserClaims{Uid: 5, Username: "alice"}, []*model.Role{superRole}, 204}, + {"administrator", &ijwt.UserClaims{Uid: 1, Username: "admin"}, []*model.Role{superRole}, 204}, } { t.Run(tt.name, func(t *testing.T) { engine := gin.New() @@ -30,7 +40,7 @@ func TestSignupRequiresAuthenticatedAdministrator(t *testing.T) { } else { engine.Use(func(c *gin.Context) { c.Set("user", *tt.claims); c.Next() }) } - engine.Use(NewAuthMiddleware(nil).CheckAuth()) + engine.Use(NewAuthMiddleware(fixedRoles{roles: tt.roles}).CheckAuth()) engine.POST("/api/user/signup", func(c *gin.Context) { c.Status(204) }) rec := httptest.NewRecorder() engine.ServeHTTP(rec, httptest.NewRequest("POST", "/api/user/signup", nil)) diff --git a/internal/model/k8s_cluster.go b/internal/model/k8s_cluster.go index 808fa2a4..9a273745 100644 --- a/internal/model/k8s_cluster.go +++ b/internal/model/k8s_cluster.go @@ -25,6 +25,11 @@ package model +import ( + "encoding/json" + "strings" +) + type Env int8 const ( @@ -56,13 +61,22 @@ type K8sCluster struct { Env Env `json:"env,omitempty" gorm:"comment:集群环境 (1:Prod, 2:Dev, 3:Stage, 4:Rc, 5:Press)"` // 集群环境 Version string `json:"version,omitempty" gorm:"comment:集群版本"` // 集群版本 ApiServerAddr string `json:"api_server_addr,omitempty" gorm:"comment:API Server 地址"` // API Server 地址 - KubeConfigContent string `json:"kube_config_content,omitempty" gorm:"type:text;comment:kubeConfig 内容"` // kubeConfig 内容 + KubeConfigContent string `json:"-" gorm:"type:text;comment:kubeConfig 内容"` // kubeConfig 内容，含凭据，仅供服务端内部使用，不对外序列化 ActionTimeoutSeconds int `json:"action_timeout_seconds,omitempty" gorm:"comment:操作超时时间（秒）"` // 操作超时时间（秒） CreateUserName string `json:"create_user_name,omitempty" gorm:"comment:创建者用户名"` // 创建者用户名 CreateUserID int `json:"create_user_id,omitempty" gorm:"comment:创建者用户ID"` // 创建者用户ID Tags KeyValueList `json:"tags,omitempty" gorm:"type:text;serializer:json;comment:标签"` // 标签 } +// MarshalJSON 读取接口仅暴露 kubeconfig 存在标志，数据库读写与内部连接仍使用原值 +func (k8sCluster K8sCluster) MarshalJSON() ([]byte, error) { + type publicCluster K8sCluster + return json.Marshal(struct { + publicCluster + HasKubeConfig bool `json:"has_kube_config"` + }{publicCluster: publicCluster(k8sCluster), HasKubeConfig: strings.TrimSpace(k8sCluster.KubeConfigContent) != ""}) +} + func (k8sCluster *K8sCluster) TableName() string { return "cl_k8s_clusters" } @@ -95,7 +109,7 @@ type UpdateClusterReq struct { Env Env `json:"env,omitempty"` // 集群环境 Version string `json:"version,omitempty"` // 集群版本 ApiServerAddr string `json:"api_server_addr,omitempty"` // API Server 地址 - KubeConfigContent string `json:"kube_config_content,omitempty"` // kubeConfig 内容 + KubeConfigContent string `json:"kube_config_content,omitempty"` // kubeConfig 内容，选填：不传或空字符串保留原值，非空则整段替换并按创建规则校验 ActionTimeoutSeconds int `json:"action_timeout_seconds,omitempty"` // 操作超时时间（秒） Tags KeyValueList `json:"tags,omitempty"` // 标签 } diff --git a/internal/model/k8s_cluster_test.go b/internal/model/k8s_cluster_test.go new file mode 100644 index 00000000..89728bd2 --- /dev/null +++ b/internal/model/k8s_cluster_test.go @@ -0,0 +1,34 @@ +package model + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestK8sClusterJSONHidesKubeconfig(t *testing.T) { + secret := "apiVersion: v1\nusers:\n- user: {client-key-data: FIXTURE-PRIVATE-KEY}\n" + cases := []struct { + content string + want bool + }{{secret, true}, {"", false}, {" \n", false}} + for _, tc := range cases { + c := K8sCluster{Model: Model{ID: 7}, Name: "c", KubeConfigContent: tc.content} + for _, v := range []any{c, &c, ListResp[*K8sCluster]{Items: []*K8sCluster{&c}, Total: 1}} { + out, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + s := string(out) + if strings.Contains(s, "kube_config_content") || strings.Contains(s, "FIXTURE-PRIVATE-KEY") { + t.Fatalf("kubeconfig 被序列化: %s", s) + } + if !strings.Contains(s, `"has_kube_config":`+map[bool]string{true: "true", false: "false"}[tc.want]) { + t.Fatalf("has_kube_config 不正确: %s", s) + } + if !strings.Contains(s, `"id":7`) || !strings.Contains(s, `"name":"c"`) { + t.Fatalf("其他字段应保持不变: %s", s) + } + } + } +} diff --git a/internal/model/k8s_secret.go b/internal/model/k8s_secret.go index 7c281837..472d2e87 100644 --- a/internal/model/k8s_secret.go +++ b/internal/model/k8s_secret.go @@ -47,21 +47,85 @@ const ( // K8sSecret Kubernetes Secret type K8sSecret struct { - Name string `json:"name" binding:"required,min=1,max=200"` // Secret名称 - Namespace string `json:"namespace" binding:"required,min=1,max=200"` // 所属命名空间 - ClusterID int `json:"cluster_id" gorm:"index;not null"` // 所属集群ID - UID string `json:"uid" gorm:"size:100"` // Secret UID - Type K8sSecretType `json:"type"` // Secret类型 - Data BinaryDataMap `json:"data"` // 加密数据 - StringData map[string]string `json:"string_data"` // 明文数据 - Labels map[string]string `json:"labels"` // 标签 - Annotations map[string]string `json:"annotations"` // 注解 - Immutable bool `json:"immutable"` // 是否不可变 - DataCount int `json:"data_count"` // 数据条目数量 - Size string `json:"size"` // 数据大小 - Age string `json:"age"` // 存在时间 - CreatedAt time.Time `json:"created_at"` // 创建时间 - RawSecret *corev1.Secret `json:"-"` // 原始 Secret 对象，不序列化到 JSON + Name string `json:"name" binding:"required,min=1,max=200"` // Secret名称 + Namespace string `json:"namespace" binding:"required,min=1,max=200"` // 所属命名空间 + ClusterID int `json:"cluster_id" gorm:"index;not null"` // 所属集群ID + UID string `json:"uid" gorm:"size:100"` // Secret UID + Type K8sSecretType `json:"type"` // Secret类型 + Data BinaryDataMap `json:"data"` // 默认脱敏：仅保留键，值恒为空字符串；明文需调用 reveal 接口 + StringData map[string]string `json:"string_data"` // 默认脱敏：仅保留键，值恒为空字符串 + // DataKeys / StringDataKeys 脱敏后的键与原始值字节数（按 key 升序） + DataKeys []SecretKeyInfo `json:"data_keys"` + StringDataKeys []SecretKeyInfo `json:"string_data_keys"` + Redacted bool `json:"redacted"` // 为 true 表示 data/string_data 的值已被移除 + Labels map[string]string `json:"labels"` // 标签 + Annotations map[string]string `json:"annotations"` // 注解 + Immutable bool `json:"immutable"` // 是否不可变 + DataCount int `json:"data_count"` // 数据条目数量 + Size string `json:"size"` // 数据大小 + Age string `json:"age"` // 存在时间 + CreatedAt time.Time `json:"created_at"` // 创建时间 + RawSecret *corev1.Secret `json:"-"` // 原始 Secret 对象，不序列化到 JSON +} + +// SecretKeyInfo Secret 中单个键的脱敏描述，只包含键名与值的字节数，不包含值 +type SecretKeyInfo struct { + Key string `json:"key"` + Size int `json:"size"` // 值的字节数（data 为解码后的原始字节数） +} + +// RevealSecretReq 获取Secret明文请求（每次调用都会写审计） +type RevealSecretReq struct { + ClusterID int `json:"cluster_id" form:"cluster_id" binding:"required" comment:"集群ID"` // 集群ID + Namespace string `json:"namespace" form:"namespace" binding:"required" comment:"命名空间"` // 命名空间 + Name string `json:"name" form:"name" binding:"required" comment:"Secret名称"` // Secret名称 + Keys []string `json:"keys" form:"keys" comment:"仅返回指定的键，为空时返回全部"` // 仅返回指定的键 +} + +// K8sSecretReveal Secret 明文响应 +type K8sSecretReveal struct { + ClusterID int `json:"cluster_id"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Type K8sSecretType `json:"type"` + Data BinaryDataMap `json:"data"` // base64 编码的明文，与改造前 detail 的 data 编码一致 + StringData map[string]string `json:"string_data"` // 明文（通常为空，K8s 会合并进 data） + Keys []string `json:"keys"` // 本次返回的键（同时写入审计） +} + +// SecretRevealActor reveal 操作者信息（由 handler 从登录态中提取） +type SecretRevealActor struct { + UserID int + Username string + IP string + UserAgent string + TraceID string + Endpoint string +} + +// K8sSecretRevealRoute Secret 明文接口的完整路由（权限码），仅允许按该路由显式授权，通配符授权不生效 +const K8sSecretRevealRoute = "/api/k8s/secret/:cluster_id/:namespace/:name/reveal" + +// K8sSecretRevealPermissionName Secret 明文接口在接口目录中的权限名 +const K8sSecretRevealPermissionName = "k8s:secret:reveal" + +const ( + SecretRevealAuditAction = "k8s_secret_reveal" + SecretRevealAuditOperation = "REVEAL" + SecretRevealAuditTargetType = "k8s_secret" +) + +// SecretRevealAuditRecord reveal 审计内容，写入 AuditLog.RequestBody。只含元数据与键名，绝不含值。 +type SecretRevealAuditRecord struct { + Action string `json:"action"` + UserID int `json:"user_id"` + Username string `json:"username"` + ClusterID int `json:"cluster_id"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Keys []string `json:"keys"` + Success bool `json:"success"` + RevealedAt string `json:"revealed_at"` // RFC3339 UTC } // GetSecretListReq 获取Secret列表请求 diff --git a/internal/model/k8s_serviceaccount.go b/internal/model/k8s_serviceaccount.go index 2ba949ab..5e0fa967 100644 --- a/internal/model/k8s_serviceaccount.go +++ b/internal/model/k8s_serviceaccount.go @@ -107,7 +107,7 @@ type CreateServiceAccountTokenReq struct { ClusterID int `json:"cluster_id" binding:"required" comment:"集群ID"` Namespace string `json:"namespace" binding:"required" comment:"命名空间"` ServiceAccountName string `json:"service_account_name" binding:"required" comment:"ServiceAccount名称"` - ExpirationSeconds *int64 `json:"expiration_seconds" comment:"令牌过期时间（秒）"` + ExpirationSeconds *int64 `json:"expiration_seconds" comment:"令牌过期时间（秒），不传或0为3600，允许600~3600"` } // K8sServiceAccount ServiceAccount主model @@ -134,3 +134,47 @@ type ServiceAccountTokenInfo struct { CreatedAt string `json:"created_at"` ExpirationTime string `json:"expiration_time"` } + +// POST /token（TokenRequest）的有效期约束（秒）。不传或传 0 使用默认值； +// 下限为 K8s TokenRequest 允许的最小值 600，上限 3600，越界返回参数错误且不调用 K8s。 +const ( + ServiceAccountTokenDefaultExpirationSeconds int64 = 3600 + ServiceAccountTokenMinExpirationSeconds int64 = 600 + ServiceAccountTokenMaxExpirationSeconds int64 = 3600 +) + +// K8sServiceAccountTokenRoute ServiceAccount token 接口的完整路由（权限码对应的 path）。 +// GET 读取 SA 名下的 token Secret，POST 通过 TokenRequest 签发新 token；两者都只允许按该路由 + 方法显式授权， +// /api/k8s/*、/* 及带 :param 的通配授权都不生效。 +const K8sServiceAccountTokenRoute = "/api/k8s/serviceaccount/:cluster_id/:namespace/:name/token" + +const ( + // K8sServiceAccountTokenPermissionName GET token 在接口目录（cl_system_apis.name）中的权限名 + K8sServiceAccountTokenPermissionName = "k8s:serviceaccount:token" + // K8sServiceAccountTokenCreatePermissionName POST token（TokenRequest）在接口目录中的权限名； + // cl_system_apis.name 有唯一索引，GET/POST 不能共用同一个名字 + K8sServiceAccountTokenCreatePermissionName = "k8s:serviceaccount:token:create" +) + +const ( + ServiceAccountTokenRevealAuditAction = "k8s_serviceaccount_token_reveal" + ServiceAccountTokenCreateAuditAction = "k8s_serviceaccount_token_create" + ServiceAccountTokenRevealAuditOperation = "TOKEN_REVEAL" + ServiceAccountTokenCreateAuditOperation = "TOKEN_CREATE" + ServiceAccountTokenAuditTargetType = "k8s_serviceaccount_token" +) + +// ServiceAccountTokenAuditRecord token 接口审计内容，写入 AuditLog.RequestBody。只含元数据，绝不含 token。 +type ServiceAccountTokenAuditRecord struct { + Action string `json:"action"` + UserID int `json:"user_id"` + Username string `json:"username"` + ClusterID int `json:"cluster_id"` + Namespace string `json:"namespace"` + ServiceAccount string `json:"service_account"` + ServiceAccountUID string `json:"service_account_uid,omitempty"` + SecretName string `json:"secret_name,omitempty"` // GET：返回的 token 所在 Secret 名称 + ExpirationSeconds *int64 `json:"expiration_seconds,omitempty"` // POST：生效的有效期（越界被拒时为请求值） + Success bool `json:"success"` + IssuedAt string `json:"issued_at"` // RFC3339 UTC +} diff --git a/internal/model/system_permission.go b/internal/model/system_permission.go new file mode 100644 index 00000000..906ca8dd --- /dev/null +++ b/internal/model/system_permission.go @@ -0,0 +1,12 @@ +package model + +// UserPermissionsVersion 是 GET /api/user/permissions 响应格式的版本号。 +const UserPermissionsVersion = 2 + +// UserPermissionsResp 是 GET /api/user/permissions 的响应数据（契约见 docs/api/user-permissions-v2.md）。 +// 仅用于接口响应，不是数据库模型，不参与 AutoMigrate。 +type UserPermissionsResp struct { + Version int `json:"version"` + SuperAdmin bool `json:"super_admin"` + Permissions []string `json:"permissions"` +} diff --git a/internal/model/system_user.go b/internal/model/system_user.go index 4d336745..4bf6d690 100644 --- a/internal/model/system_user.go +++ b/internal/model/system_user.go @@ -89,8 +89,8 @@ type GetPermCodeReq struct { } type ChangePasswordReq struct { - UserID int `json:"user_id" form:"user_id" binding:"required"` // 用户ID - Username string `json:"username" form:"username" binding:"required"` // 用户名 + UserID int `json:"user_id" form:"user_id"` // 可选，已忽略：服务端以 token 中的 uid 为准（SEC-28） + Username string `json:"username" form:"username"` // 可选，已忽略：服务端以 token 为准，不参与改密（SEC-28） Password string `json:"password" form:"password" binding:"required"` // 原密码 NewPassword string `json:"new_password" form:"new_password" binding:"required,min=6"` // 新密码，至少6位 ConfirmPassword string `json:"confirm_password" form:"confirm_password" binding:"required,eqfield=NewPassword"` // 确认密码，必须与新密码相同 diff --git a/internal/permission/catalog.go b/internal/permission/catalog.go new file mode 100644 index 00000000..7d226ac1 --- /dev/null +++ b/internal/permission/catalog.go @@ -0,0 +1,112 @@ +package permission + +import ( + "sort" + "strings" + + "github.com/gin-gonic/gin" +) + +var skipAuthorizationPaths = map[string]bool{ + "/api/user/login": true, + "/api/user/logout": true, + "/api/user/refresh_token": true, + "/api/user/profile": true, + "/api/user/codes": true, + "/api/user/permissions": true, + "/api/not_auth/getBindIps": true, + "/api/not_auth/getTreeNodeBindIps": true, + "/favicon.ico": true, +} + +// SkipsAuthorization 判断路径是否不经角色授权（登录态仍由登录中间件负责）。 +func SkipsAuthorization(path string) bool { + return skipAuthorizationPaths[path] +} + +// RequiresSuperAdmin 判断路由模板是否只允许超管访问（角色授予对其无效）。 +func RequiresSuperAdmin(method, template string) bool { + if method == "POST" && template == "/api/user/signup" { + return true + } + return template == "/api/department" || strings.HasPrefix(template, "/api/department/") +} + +// CatalogEntry 是可授权接口目录中的一项。Via 非空表示该接口经由 gin 通配模板 +// （代理）提供，授予覆盖 Via 本身也视为覆盖该接口。 +type CatalogEntry struct { + Method int8 + Path string + Via string +} + +// Code 返回 "METHOD /path/:param" 形式的权限码。 +func (e CatalogEntry) Code() string { + return MethodName(e.Method) + " " + e.Path +} + +// BuildCatalog 根据 gin 已注册路由与已登记的代理目录构建可授权接口目录： +// 只保留 GET/POST/PUT/DELETE、/api/ 前缀、需要角色授权且非超管专属的接口。 +func BuildCatalog(routes gin.RoutesInfo) []CatalogEntry { + seen := map[string]bool{} + var entries []CatalogEntry + add := func(method, path, via string) { + code, ok := MethodCode(method) + if !ok || !strings.HasPrefix(path, "/api/") || SkipsAuthorization(path) || RequiresSuperAdmin(method, path) { + return + } + key := method + " " + path + if seen[key] { + return + } + seen[key] = true + entries = append(entries, CatalogEntry{Method: code, Path: path, Via: via}) + } + var proxies []string + for _, route := range routes { + add(route.Method, route.Path, "") + if !contains(proxies, route.Path) && len(ProxyCatalog(route.Path)) > 0 { + proxies = append(proxies, route.Path) + } + } + for _, template := range proxies { + registered := map[string]bool{} + for _, route := range routes { + if route.Path == template { + registered[route.Method] = true + } + } + for _, route := range ProxyCatalog(template) { + if registered[route.Method] { + add(route.Method, route.Path, template) + } + } + } + return entries +} + +// Expand 把授予展开为目录中被覆盖的权限码，结果已排序去重，且不为 nil。 +// 判定与鉴权中间件一致：方法相同，且授予覆盖接口模板或其代理通配模板。 +func Expand(grants []Grant, catalog []CatalogEntry) []string { + codes := map[string]bool{} + for _, entry := range catalog { + if Allowed(grants, entry.Method, entry.Path) || (entry.Via != "" && Allowed(grants, entry.Method, entry.Via)) { + codes[entry.Code()] = true + } + } + result := make([]string, 0, len(codes)) + for code := range codes { + result = append(result, code) + } + sort.Strings(result) + return result +} + +func contains(items []string, item string) bool { + for _, existing := range items { + if existing == item { + return true + } + } + return false +} diff --git a/internal/permission/matcher.go b/internal/permission/matcher.go new file mode 100644 index 00000000..3eea871e --- /dev/null +++ b/internal/permission/matcher.go @@ -0,0 +1,87 @@ +package permission + +import ( + "strings" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" +) + +// SuperCode 是 /api/user/permissions 返回给超管的唯一权限码，表示全部放行。 +const SuperCode = "*" + +var methodCodes = map[string]int8{"GET": 1, "POST": 2, "PUT": 3, "DELETE": 4} +var methodNames = map[int8]string{1: "GET", 2: "POST", 3: "PUT", 4: "DELETE"} + +// MethodCode 把 HTTP 方法转换为 cl_system_apis.method 的取值。 +func MethodCode(method string) (int8, bool) { + code, ok := methodCodes[method] + return code, ok +} + +// MethodName 把 cl_system_apis.method 的取值转换为 HTTP 方法；未知取值返回 ""。 +func MethodName(code int8) string { + return methodNames[code] +} + +// Grant 是角色授予的一条接口权限（方法 + 授予路径）。 +type Grant struct { + Method int8 + Path string +} + +// GrantsFromRoles 收集启用中角色的接口授予。 +func GrantsFromRoles(roles []*model.Role) []Grant { + var grants []Grant + for _, role := range roles { + if role == nil || role.Status != 1 { + continue + } + for _, api := range role.Apis { + if api == nil { + continue + } + grants = append(grants, Grant{Method: api.Method, Path: api.Path}) + } + } + return grants +} + +// Covers 判断授予路径是否覆盖某个 gin 路由模板。 +// +// - 模板为空（请求没有命中路由）时永远不覆盖； +// - 与模板完全相同则覆盖（包括 /api/v1/*path 这类 gin 通配模板本身）； +// - 只含一个 * 且位于末尾的授予（如 /*、/api/k8s/*）对模板字符串做前缀匹配； +// - 其他写法（开头或中间的 *、多个 *）不覆盖任何模板； +// - :param 只是模板里的字面量，不会被当作通配符。 +func Covers(granted, template string) bool { + if template == "" || granted == "" { + return false + } + if granted == template { + return true + } + // Secret 明文读取和 ServiceAccount 令牌操作必须显式授予。 + if template == model.K8sSecretRevealRoute || template == model.K8sServiceAccountTokenRoute { + return false + } + if strings.Count(granted, "*") == 1 && strings.HasSuffix(granted, "*") { + return strings.HasPrefix(template, strings.TrimSuffix(granted, "*")) + } + return false +} + +// Allowed 判断授予集合中是否存在方法相同且覆盖任一候选模板的授予。 +// 空模板会被忽略。 +func Allowed(grants []Grant, method int8, templates ...string) bool { + for _, grant := range grants { + if grant.Method != method { + continue + } + for _, template := range templates { + if Covers(grant.Path, template) { + return true + } + } + } + return false +} diff --git a/internal/permission/matcher_test.go b/internal/permission/matcher_test.go new file mode 100644 index 00000000..eb4f71e0 --- /dev/null +++ b/internal/permission/matcher_test.go @@ -0,0 +1,174 @@ +package permission + +import ( + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/gin-gonic/gin" +) + +func TestCoversComparesTemplatesNotRequestPaths(t *testing.T) { + for _, tt := range []struct { + granted, template string + want bool + }{ + {"/api/tree/cloud/:id/status", "/api/tree/cloud/:id/status", true}, + {"/api/tree/cloud/:id/status", "/api/tree/cloud/batch/status", false}, + {"/api/tree/cloud/:id/status", "/api/tree/cloud/9/status", false}, + {"/api/k8s/clusters/:cluster_id/events/:namespace/:name", "/api/k8s/clusters/:cluster_id/events/nodes/:node_name", false}, + {"/*", "/api/anything/:id", true}, + {"*", "/api/anything/:id", true}, // 与改动前一致：单个 * 视为全局前缀 + {"/api/tree/*", "/api/tree/cloud/batch/status", true}, + {"/api/tree/*", "/api/treex/list", false}, + {"/api/tree*", "/api/treex/list", true}, // 前缀按字符串比较，与改动前一致 + {"/api/tree/cloud/:id/*", "/api/tree/cloud/:id/status", true}, + {"/api/tree/cloud/:id/*", "/api/tree/cloud/batch/status", false}, + {"/api/v1/*path", "/api/v1/*path", true}, + {"/api/v1/*", "/api/v1/*path", true}, + {"*/status", "/api/tree/cloud/:id/status", false}, + {"/api/*/status", "/api/tree/cloud/:id/status", false}, + {"/api/**", "/api/tree/list", false}, + {"/api/*/*", "/api/tree/list", false}, + {"", "/api/tree/list", false}, + {"/*", "", false}, + } { + if got := Covers(tt.granted, tt.template); got != tt.want { + t.Errorf("Covers(%q, %q)=%v want %v", tt.granted, tt.template, got, tt.want) + } + } +} + +func TestAllowedRequiresSameMethodAndIgnoresDisabledRoles(t *testing.T) { + grants := GrantsFromRoles([]*model.Role{ + nil, + {Status: 1, Apis: []*model.Api{nil, {Method: 1, Path: "/api/x/:id"}}}, + {Status: 0, Apis: []*model.Api{{Method: 3, Path: "/*"}}}, + }) + if len(grants) != 1 { + t.Fatalf("grants=%v", grants) + } + if !Allowed(grants, 1, "/api/x/:id") || Allowed(grants, 3, "/api/x/:id") || Allowed(grants, 1, "", "") { + t.Fatal("method/empty-template handling broken") + } + if !Allowed(grants, 1, "/api/v1/*path", "/api/x/:id") { + t.Fatal("any candidate template should be enough") + } +} + +func TestMethodCodes(t *testing.T) { + for name, code := range map[string]int8{"GET": 1, "POST": 2, "PUT": 3, "DELETE": 4} { + if got, ok := MethodCode(name); !ok || got != code || MethodName(code) != name { + t.Fatalf("%s", name) + } + } + if _, ok := MethodCode("PATCH"); ok { + t.Fatal("PATCH has no grant code") + } + if MethodName(9) != "" { + t.Fatal("unknown code") + } +} + +func withProxyCatalog(t *testing.T, routes []Route) { + t.Helper() + previous := ProxyCatalog(AIOpsProxyTemplate) + SetProxyCatalog(AIOpsProxyTemplate, routes) + t.Cleanup(func() { SetProxyCatalog(AIOpsProxyTemplate, previous) }) +} + +func TestResolveProxyTemplatePicksMostSpecific(t *testing.T) { + withProxyCatalog(t, []Route{ + {"GET", "/api/v1/report/:id"}, + {"GET", "/api/v1/report/latest"}, + {"GET", "/api/v1/:kind/latest"}, + {"GET", "/api/v1/report/:id/markdown"}, + {"POST", "/api/v1/autofix"}, + {"POST", "/api/v1/autofix/"}, + }) + for _, tt := range []struct { + method, fullPath, path, want string + }{ + {"GET", AIOpsProxyTemplate, "/api/v1/report/r1", "/api/v1/report/:id"}, + {"GET", AIOpsProxyTemplate, "/api/v1/report/latest", "/api/v1/report/latest"}, + {"GET", AIOpsProxyTemplate, "/api/v1/task/latest", "/api/v1/:kind/latest"}, + {"GET", AIOpsProxyTemplate, "/api/v1/report/r1/markdown", "/api/v1/report/:id/markdown"}, + {"GET", AIOpsProxyTemplate, "/api/v1/report/", ""}, + {"GET", AIOpsProxyTemplate, "/api/v1/report/..", ""}, + {"GET", AIOpsProxyTemplate, "/api/v1/report/.", ""}, + {"GET", AIOpsProxyTemplate, "/api/v1/report/r1/extra", ""}, + {"POST", AIOpsProxyTemplate, "/api/v1/report/r1", ""}, + {"POST", AIOpsProxyTemplate, "/api/v1/autofix", "/api/v1/autofix"}, + {"POST", AIOpsProxyTemplate, "/api/v1/autofix/", "/api/v1/autofix/"}, + {"GET", "/api/other/*path", "/api/v1/report/r1", ""}, + {"GET", "", "/api/v1/report/r1", ""}, + } { + if got := ResolveProxyTemplate(tt.method, tt.fullPath, tt.path); got != tt.want { + t.Errorf("Resolve(%s %s)=%q want %q", tt.method, tt.path, got, tt.want) + } + } +} + +func TestProxyCatalogIsCopiedAndClearable(t *testing.T) { + routes := []Route{{"GET", "/api/v1/a"}} + withProxyCatalog(t, routes) + routes[0].Path = "/api/v1/mutated" + got := ProxyCatalog(AIOpsProxyTemplate) + got[0].Path = "/api/v1/mutated-again" + if ProxyCatalog(AIOpsProxyTemplate)[0].Path != "/api/v1/a" { + t.Fatal("catalog must be copied on set and get") + } + SetProxyCatalog(AIOpsProxyTemplate, nil) + if len(ProxyCatalog(AIOpsProxyTemplate)) != 0 || ResolveProxyTemplate("GET", AIOpsProxyTemplate, "/api/v1/a") != "" { + t.Fatal("empty catalog should clear registration") + } +} + +func TestBuildCatalogAndExpand(t *testing.T) { + withProxyCatalog(t, []Route{{"GET", "/api/v1/a"}, {"POST", "/api/v1/b"}, {"PATCH", "/api/v1/c"}}) + routes := gin.RoutesInfo{ + {Method: "GET", Path: "/api/item/:id"}, + {Method: "GET", Path: "/api/item/:id"}, // 重复 + {Method: "PATCH", Path: "/api/item/:id"}, + {Method: "POST", Path: "/api/user/signup"}, + {Method: "GET", Path: "/api/department/list"}, + {Method: "GET", Path: "/api/user/codes"}, + {Method: "GET", Path: "/api/user/permissions"}, + {Method: "GET", Path: "/health"}, + {Method: "GET", Path: AIOpsProxyTemplate}, // 代理只注册了 GET + } + var codes []string + for _, entry := range BuildCatalog(routes) { + codes = append(codes, entry.Code()+"|"+entry.Via) + } + want := "GET /api/item/:id|,GET /api/v1/*path|,GET /api/v1/a|/api/v1/*path" + if strings.Join(codes, ",") != want { + t.Fatalf("catalog=%v", codes) + } + + catalog := BuildCatalog(routes) + if got := Expand(nil, catalog); got == nil || len(got) != 0 { + t.Fatalf("empty grants must expand to [], got %#v", got) + } + if got := strings.Join(Expand([]Grant{{1, AIOpsProxyTemplate}}, catalog), ","); got != "GET /api/v1/*path,GET /api/v1/a" { + t.Fatalf("whole proxy: %s", got) + } + if got := strings.Join(Expand([]Grant{{1, "/api/v1/a"}, {1, "/api/v1/a"}, {1, "/api/item/:id"}}, catalog), ","); got != "GET /api/item/:id,GET /api/v1/a" { + t.Fatalf("exact: %s", got) + } + if got := Expand([]Grant{{2, "/*"}}, catalog); len(got) != 0 { + t.Fatalf("super-only and unregistered-method entries must not appear: %v", got) + } +} + +func TestRequiresSuperAdminAndSkips(t *testing.T) { + if !RequiresSuperAdmin("POST", "/api/user/signup") || RequiresSuperAdmin("GET", "/api/user/signup") { + t.Fatal("signup") + } + if !RequiresSuperAdmin("GET", "/api/department") || !RequiresSuperAdmin("DELETE", "/api/department/:id") || RequiresSuperAdmin("GET", "/api/departments") { + t.Fatal("department") + } + if !SkipsAuthorization("/api/user/permissions") || SkipsAuthorization("/api/user/list") { + t.Fatal("skip") + } +} diff --git a/internal/permission/proxy.go b/internal/permission/proxy.go new file mode 100644 index 00000000..dcf62c04 --- /dev/null +++ b/internal/permission/proxy.go @@ -0,0 +1,100 @@ +package permission + +import ( + "strings" + "sync" +) + +// AIOpsProxyTemplate 是 AIOps 反向代理在 gin 中注册的通配模板。 +const AIOpsProxyTemplate = "/api/v1/*path" + +// Route 是代理目录中的一个接口模板。 +type Route struct { + Method string `json:"method"` + Path string `json:"path"` +} + +var ( + proxyMu sync.RWMutex + proxyCatalogs = map[string][]Route{} +) + +// SetProxyCatalog 为某个 gin 通配模板登记下游接口目录（覆盖原有登记）。 +// 代理请求的 c.FullPath() 只有通配模板，鉴权需要借助目录还原出具体接口模板。 +func SetProxyCatalog(template string, routes []Route) { + copied := append([]Route(nil), routes...) + proxyMu.Lock() + defer proxyMu.Unlock() + if len(copied) == 0 { + delete(proxyCatalogs, template) + return + } + proxyCatalogs[template] = copied +} + +// ProxyCatalog 返回某个通配模板登记的下游接口目录副本。 +func ProxyCatalog(template string) []Route { + proxyMu.RLock() + defer proxyMu.RUnlock() + return append([]Route(nil), proxyCatalogs[template]...) +} + +// ResolveProxyTemplate 把命中代理通配模板的真实请求路径解析为目录中最具体的接口模板。 +// fullPath 不是已登记的代理模板、或目录中没有匹配项时返回 ""。 +// +// 匹配逐段进行：段数必须相同，静态段必须完全相等，:param 段只匹配一个非空且不为 +// "." / ".." 的段。多个模板同时匹配时，最先出现静态段的模板胜出（静态优先于参数， +// 与 gin 路由优先级一致）；完全并列时取目录中靠前的一条。 +func ResolveProxyTemplate(method, fullPath, requestPath string) string { + proxyMu.RLock() + routes := proxyCatalogs[fullPath] + proxyMu.RUnlock() + if len(routes) == 0 { + return "" + } + requested := strings.Split(requestPath, "/") + best, bestShape := "", []bool(nil) + for _, route := range routes { + if route.Method != method { + continue + } + shape, ok := matchSegments(strings.Split(route.Path, "/"), requested) + if !ok { + continue + } + if best == "" || moreSpecific(shape, bestShape) { + best, bestShape = route.Path, shape + } + } + return best +} + +// matchSegments 返回每一段是否为静态段。 +func matchSegments(template, requested []string) ([]bool, bool) { + if len(template) != len(requested) { + return nil, false + } + shape := make([]bool, len(template)) + for i, part := range template { + if strings.HasPrefix(part, ":") && len(part) > 1 { + if requested[i] == "" || requested[i] == "." || requested[i] == ".." { + return nil, false + } + continue + } + if part != requested[i] { + return nil, false + } + shape[i] = true + } + return shape, true +} + +func moreSpecific(a, b []bool) bool { + for i := range a { + if a[i] != b[i] { + return a[i] + } + } + return false +} diff --git a/internal/permission/sensitive_grants_test.go b/internal/permission/sensitive_grants_test.go new file mode 100644 index 00000000..6a5b1199 --- /dev/null +++ b/internal/permission/sensitive_grants_test.go @@ -0,0 +1,65 @@ +package permission + +import ( + "reflect" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/gin-gonic/gin" +) + +func TestSensitivePermissionsRequireExactMethodAndTemplate(t *testing.T) { + for _, endpoint := range []CatalogEntry{ + {Method: 1, Path: model.K8sSecretRevealRoute}, + {Method: 1, Path: model.K8sServiceAccountTokenRoute}, + {Method: 2, Path: model.K8sServiceAccountTokenRoute}, + } { + t.Run(endpoint.Code(), func(t *testing.T) { + if !Allowed([]Grant{{Method: endpoint.Method, Path: endpoint.Path}}, endpoint.Method, endpoint.Path) { + t.Fatal("exact method and template must authorize the sensitive endpoint") + } + for _, path := range []string{"*", "/*", "/api/*", "/api/k8s/*"} { + if Covers(path, endpoint.Path) || Allowed([]Grant{{Method: endpoint.Method, Path: path}}, endpoint.Method, endpoint.Path) { + t.Errorf("wildcard %q authorized sensitive endpoint", path) + } + } + otherMethod := int8(1) + if endpoint.Method == otherMethod { + otherMethod = 2 + } + if Allowed([]Grant{{Method: otherMethod, Path: endpoint.Path}}, endpoint.Method, endpoint.Path) { + t.Fatal("exact template with a different method must not authorize the endpoint") + } + }) + } +} + +func TestExpandKeepsSensitivePermissionsExplicit(t *testing.T) { + catalog := BuildCatalog(gin.RoutesInfo{ + {Method: "GET", Path: model.K8sSecretRevealRoute}, + {Method: "GET", Path: model.K8sServiceAccountTokenRoute}, + {Method: "POST", Path: model.K8sServiceAccountTokenRoute}, + {Method: "GET", Path: "/api/k8s/secret/:cluster_id/:namespace/:name/detail"}, + }) + for _, wildcard := range []string{"*", "/*", "/api/*", "/api/k8s/*"} { + t.Run(wildcard, func(t *testing.T) { + grants := []Grant{{Method: 1, Path: wildcard}, {Method: 2, Path: wildcard}} + want := []string{"GET /api/k8s/secret/:cluster_id/:namespace/:name/detail"} + if got := Expand(grants, catalog); !reflect.DeepEqual(got, want) { + t.Fatalf("wildcard expansion=%v want=%v", got, want) + } + }) + } + for _, endpoint := range catalog { + if endpoint.Path != model.K8sSecretRevealRoute && endpoint.Path != model.K8sServiceAccountTokenRoute { + continue + } + t.Run(endpoint.Code(), func(t *testing.T) { + grants := []Grant{{Method: endpoint.Method, Path: endpoint.Path}} + want := []string{endpoint.Code()} + if got := Expand(grants, catalog); !reflect.DeepEqual(got, want) { + t.Fatalf("exact grant expansion=%v want=%v", got, want) + } + }) + } +} diff --git a/internal/permission/superadmin.go b/internal/permission/superadmin.go new file mode 100644 index 00000000..6ae1b50d --- /dev/null +++ b/internal/permission/superadmin.go @@ -0,0 +1,62 @@ +// Package permission 集中定义 RBAC 的身份判定与授权匹配规则，供鉴权中间件、 +// 权限码接口与启动引导共用，避免各处各写一套导致语义漂移。 +package permission + +import ( + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/gin-gonic/gin" +) + +const ( + // SuperAdminRoleCode 内置超级管理员角色编码。身份判定同时要求 is_system=1， + // 普通角色即使使用同样的编码也不会获得超管身份（CreateRole 强制 is_system=0）。 + SuperAdminRoleCode = "super_admin" + // SuperAdminRoleName 内置超级管理员角色名称。 + SuperAdminRoleName = "超级管理员" + // SuperAdminRoleDescription 内置超级管理员角色描述。 + SuperAdminRoleDescription = "内置超级管理员角色，由启动引导维护，不可修改或删除" + // BootstrapAdminUsername 启动引导会绑定到超管角色的内置账号。它只用于引导， + // 运行期授权从不按用户名判断。 + BootstrapAdminUsername = "admin" + + superAdminContextKey = "cloudops.rbac.super_admin" +) + +// IsSuperAdminIdentity 判断角色是否是内置超管角色本身（不看启用状态）。 +func IsSuperAdminIdentity(role *model.Role) bool { + return role != nil && role.IsSystem == 1 && role.Code == SuperAdminRoleCode +} + +// IsSuperAdminRole 判断角色是否是启用中的内置超管角色。 +func IsSuperAdminRole(role *model.Role) bool { + return IsSuperAdminIdentity(role) && role.Status == 1 +} + +// HasSuperAdminRole 判断角色集合中是否包含启用中的内置超管角色。 +func HasSuperAdminRole(roles []*model.Role) bool { + for _, role := range roles { + if IsSuperAdminRole(role) { + return true + } + } + return false +} + +// MarkSuperAdmin 由鉴权中间件在读取用户角色后写入请求上下文。 +func MarkSuperAdmin(c *gin.Context, super bool) { + c.Set(superAdminContextKey, super) +} + +// IsSuperAdmin 读取鉴权中间件写入的超管标记。未经过鉴权中间件（没有标记）时 +// 一律视为非超管，保证 fail closed。 +func IsSuperAdmin(c *gin.Context) bool { + if c == nil { + return false + } + value, ok := c.Get(superAdminContextKey) + if !ok { + return false + } + super, ok := value.(bool) + return ok && super +} diff --git a/internal/permission/superadmin_test.go b/internal/permission/superadmin_test.go new file mode 100644 index 00000000..4767bac7 --- /dev/null +++ b/internal/permission/superadmin_test.go @@ -0,0 +1,50 @@ +package permission + +import ( + "net/http/httptest" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/gin-gonic/gin" +) + +func TestSuperAdminIdentityRequiresSystemRoleAndStatus(t *testing.T) { + for _, tt := range []struct { + name string + role *model.Role + super bool + }{ + {"nil", nil, false}, + {"system enabled", &model.Role{Code: SuperAdminRoleCode, IsSystem: 1, Status: 1}, true}, + {"system disabled", &model.Role{Code: SuperAdminRoleCode, IsSystem: 1, Status: 0}, false}, + {"custom role reusing code", &model.Role{Code: SuperAdminRoleCode, IsSystem: 0, Status: 1}, false}, + {"other system role", &model.Role{Code: "auditor", IsSystem: 1, Status: 1}, false}, + {"name only", &model.Role{Name: SuperAdminRoleName, Code: "x", IsSystem: 1, Status: 1}, false}, + } { + if got := IsSuperAdminRole(tt.role); got != tt.super { + t.Fatalf("%s: got %v want %v", tt.name, got, tt.super) + } + if got := HasSuperAdminRole([]*model.Role{{Code: "member", Status: 1}, tt.role}); got != tt.super { + t.Fatalf("%s (set): got %v want %v", tt.name, got, tt.super) + } + } +} + +func TestSuperAdminContextFailsClosed(t *testing.T) { + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + if IsSuperAdmin(c) || IsSuperAdmin(nil) { + t.Fatal("missing mark must not be super admin") + } + c.Set(superAdminContextKey, "true") + if IsSuperAdmin(c) { + t.Fatal("non-bool mark must not be super admin") + } + MarkSuperAdmin(c, true) + if !IsSuperAdmin(c) { + t.Fatal("mark not read back") + } + MarkSuperAdmin(c, false) + if IsSuperAdmin(c) { + t.Fatal("cleared mark still super admin") + } +} diff --git a/internal/system/api/change_password_regression_test.go b/internal/system/api/change_password_regression_test.go new file mode 100644 index 00000000..00309a79 --- /dev/null +++ b/internal/system/api/change_password_regression_test.go @@ -0,0 +1,122 @@ +package api + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + ijwt "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" +) + +// changePasswordRecorder records which identity reaches the service layer (SEC-28 regression). +type changePasswordRecorder struct { + service.UserService + pwd *model.ChangePasswordReq + woUID int + profile *model.UpdateProfileReq +} + +func (s *changePasswordRecorder) ChangePassword(_ context.Context, r *model.ChangePasswordReq) error { + cp := *r + s.pwd = &cp + return nil +} +func (s *changePasswordRecorder) WriteOff(_ context.Context, uid int, _ string) error { + s.woUID = uid + return nil +} +func (s *changePasswordRecorder) UpdateProfile(_ context.Context, r *model.UpdateProfileReq, _ bool) error { + cp := *r + s.profile = &cp + return nil +} +func (s *changePasswordRecorder) GetProfile(_ context.Context, uid int) (*model.User, error) { + return &model.User{Model: model.Model{ID: uid}}, nil +} + +func newChangePasswordTestEngine(rec *changePasswordRecorder, uid int, name string) *gin.Engine { + h := NewUserHandler(rec, nil) + e := gin.New() + e.Use(func(c *gin.Context) { c.Set("user", ijwt.UserClaims{Uid: uid, Username: name}); c.Next() }) + h.RegisterRoutes(e) + return e +} + +func TestChangePasswordIgnoresClientUserID(t *testing.T) { + const self, victim = 7, 99 + cases := []struct{ name, ct, path, body string }{ + {"json", "application/json", "/api/user/change_password", + `{"user_id":99,"username":"victim","password":"old-pass-x","new_password":"new-pass-x","confirm_password":"new-pass-x"}`}, + {"form", "application/x-www-form-urlencoded", "/api/user/change_password", + url.Values{"user_id": {"99"}, "username": {"victim"}, "password": {"old-pass-x"}, "new_password": {"new-pass-x"}, "confirm_password": {"new-pass-x"}}.Encode()}, + {"query+json", "application/json", "/api/user/change_password?user_id=99", + `{"username":"victim","password":"old-pass-x","new_password":"new-pass-x","confirm_password":"new-pass-x"}`}, + {"query+form", "application/x-www-form-urlencoded", "/api/user/change_password?user_id=99", + url.Values{"password": {"old-pass-x"}, "new_password": {"new-pass-x"}, "confirm_password": {"new-pass-x"}}.Encode()}, + {"json user_id=0", "application/json", "/api/user/change_password", + `{"user_id":0,"username":"victim","password":"old-pass-x","new_password":"new-pass-x","confirm_password":"new-pass-x"}`}, + {"user_id and username omitted", "application/json", "/api/user/change_password", + `{"password":"old-pass-x","new_password":"new-pass-x","confirm_password":"new-pass-x"}`}, + {"username omitted", "application/json", "/api/user/change_password", + `{"user_id":7,"password":"old-pass-x","new_password":"new-pass-x","confirm_password":"new-pass-x"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := &changePasswordRecorder{} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, tc.path, strings.NewReader(tc.body)) + r.Header.Set("Content-Type", tc.ct) + newChangePasswordTestEngine(rec, self, "alice").ServeHTTP(w, r) + if rec.pwd == nil { + t.Fatalf("service not reached: %d %s", w.Code, w.Body.String()) + } + t.Logf("service received UserID=%d (token uid=%d, body/query user_id=%d)", rec.pwd.UserID, self, victim) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status %d: %s", w.Code, w.Body.String()) + } + if rec.pwd.UserID != self { + t.Fatalf("client user_id overrode token uid -> %d", rec.pwd.UserID) + } + }) + } +} + +func TestWriteOffUsesTokenUID(t *testing.T) { + rec := &changePasswordRecorder{} + r := httptest.NewRequest(http.MethodPost, "/api/user/write_off", strings.NewReader(`{"password":"pw-x","user_id":99,"id":99}`)) + r.Header.Set("Content-Type", "application/json") + newChangePasswordTestEngine(rec, 7, "alice").ServeHTTP(httptest.NewRecorder(), r) + t.Logf("write_off uid=%d", rec.woUID) + if rec.woUID != 7 { + t.Fatalf("write_off target=%d", rec.woUID) + } +} + +func TestUpdateProfileOwnership(t *testing.T) { + body := `{"id":99,"real_name":"x","mobile":"1","account_type":1,"home_path":"/","avatar":"data:x"}` + // path id = other user, non-admin -> 403 + rec := &changePasswordRecorder{} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPut, "/api/user/profile/update/99", strings.NewReader(body)) + r.Header.Set("Content-Type", "application/json") + newChangePasswordTestEngine(rec, 7, "alice").ServeHTTP(w, r) + t.Logf("other path id: status=%d reached=%v", w.Code, rec.profile != nil) + if w.Code != http.StatusForbidden || rec.profile != nil { + t.Fatalf("non-admin could update other profile") + } + // path id = self, body id = other -> service gets self + rec = &changePasswordRecorder{} + r = httptest.NewRequest(http.MethodPut, "/api/user/profile/update/7", strings.NewReader(body)) + r.Header.Set("Content-Type", "application/json") + newChangePasswordTestEngine(rec, 7, "alice").ServeHTTP(httptest.NewRecorder(), r) + if rec.profile == nil || rec.profile.ID != 7 { + t.Fatalf("body id overrode path id: %+v", rec.profile) + } + t.Logf("self path id + body id=99: service ID=%d", rec.profile.ID) +} diff --git a/internal/system/api/department_handler.go b/internal/system/api/department_handler.go index 677207db..61aa773e 100644 --- a/internal/system/api/department_handler.go +++ b/internal/system/api/department_handler.go @@ -2,8 +2,8 @@ package api import ( "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" "github.com/GoSimplicity/AI-CloudOps/internal/system/service" - userutils "github.com/GoSimplicity/AI-CloudOps/internal/system/utils" "github.com/GoSimplicity/AI-CloudOps/pkg/base" "github.com/gin-gonic/gin" ) @@ -17,8 +17,8 @@ func (h *DepartmentHandler) RegisterRouters(server *gin.Engine) { group := server.Group("/api/department") // Organization assignments are administrative state, never self-service profile data. group.Use(func(c *gin.Context) { - claims, err := userutils.ExtractClaims(c) - if err != nil || claims.Username != "admin" { + // 超管身份由鉴权中间件根据内置超管角色写入上下文，未写入时视为非超管 + if !permission.IsSuperAdmin(c) { base.ForbiddenError(c, "仅管理员可管理部门及用户归属") c.Abort() return diff --git a/internal/system/api/department_handler_test.go b/internal/system/api/department_handler_test.go index e5f70d9c..09d987c0 100644 --- a/internal/system/api/department_handler_test.go +++ b/internal/system/api/department_handler_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" "github.com/GoSimplicity/AI-CloudOps/internal/system/dao" "github.com/GoSimplicity/AI-CloudOps/internal/system/service" "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" @@ -33,7 +34,11 @@ func TestDepartmentHTTPPersistenceAndPermissions(t *testing.T) { } router := gin.New() router.Use(func(c *gin.Context) { - c.Set("user", jwt.UserClaims{Uid: user.ID, Username: c.GetHeader("X-Test-Identity")}) + // 模拟鉴权中间件：只有 "super" 身份被标记为持有内置超管角色； + // "admin" 身份只有用户名叫 admin，没有超管角色。 + identity := c.GetHeader("X-Test-Identity") + c.Set("user", jwt.UserClaims{Uid: user.ID, Username: identity}) + permission.MarkSuperAdmin(c, identity == "super") c.Next() }) NewDepartmentHandler(service.NewDepartmentService(dao.NewDepartmentDAO(db))).RegisterRouters(router) @@ -59,28 +64,31 @@ func TestDepartmentHTTPPersistenceAndPermissions(t *testing.T) { if status, _ := call("POST", "/api/department/create", "member", map[string]any{"name": "Forbidden"}); status != 403 { t.Fatalf("non-admin status %d", status) } - _, created := call("POST", "/api/department/create", "admin", map[string]any{"name": "Engineering"}) + if status, _ := call("POST", "/api/department/create", "admin", map[string]any{"name": "Forbidden"}); status != 403 { + t.Fatalf("username admin without super admin role status %d", status) + } + _, created := call("POST", "/api/department/create", "super", map[string]any{"name": "Engineering"}) if created["code"] != float64(0) { t.Fatal(created) } id := int(created["data"].(map[string]any)["id"].(float64)) - _, child := call("POST", "/api/department/create", "admin", map[string]any{"name": "Operations", "parent_id": id}) + _, child := call("POST", "/api/department/create", "super", map[string]any{"name": "Operations", "parent_id": id}) if child["code"] != float64(0) { t.Fatal(child) } childID := int(child["data"].(map[string]any)["id"].(float64)) - _, cycle := call("PUT", fmt.Sprintf("/api/department/%d/update", id), "admin", map[string]any{"name": "Engineering", "parent_id": childID}) + _, cycle := call("PUT", fmt.Sprintf("/api/department/%d/update", id), "super", map[string]any{"name": "Engineering", "parent_id": childID}) if cycle["code"] == float64(0) { t.Fatal("cycle accepted") } - _, nonempty := call("DELETE", fmt.Sprintf("/api/department/%d/delete", id), "admin", nil) + _, nonempty := call("DELETE", fmt.Sprintf("/api/department/%d/delete", id), "super", nil) if nonempty["code"] == float64(0) { t.Fatal("deleted parent with child") } if status, _ := call("PUT", fmt.Sprintf("/api/department/user/%d", user.ID), "member", map[string]int{"department_id": childID}); status != 403 { t.Fatal("non-admin assigned department") } - _, assigned := call("PUT", fmt.Sprintf("/api/department/user/%d", user.ID), "admin", map[string]int{"department_id": childID}) + _, assigned := call("PUT", fmt.Sprintf("/api/department/user/%d", user.ID), "super", map[string]int{"department_id": childID}) if assigned["code"] != float64(0) { t.Fatal(assigned) } @@ -89,32 +97,32 @@ func TestDepartmentHTTPPersistenceAndPermissions(t *testing.T) { if persisted.DepartmentID != childID { t.Fatal("assignment not persisted") } - _, nonempty = call("DELETE", fmt.Sprintf("/api/department/%d/delete", childID), "admin", nil) + _, nonempty = call("DELETE", fmt.Sprintf("/api/department/%d/delete", childID), "super", nil) if nonempty["code"] == float64(0) { t.Fatal("deleted department with member") } - _, cleared := call("PUT", fmt.Sprintf("/api/department/user/%d", user.ID), "admin", map[string]int{"department_id": 0}) + _, cleared := call("PUT", fmt.Sprintf("/api/department/user/%d", user.ID), "super", map[string]int{"department_id": 0}) if cleared["code"] != float64(0) { t.Fatal(cleared) } for _, n := range []int{childID, id} { - _, deleted := call("DELETE", fmt.Sprintf("/api/department/%d/delete", n), "admin", nil) + _, deleted := call("DELETE", fmt.Sprintf("/api/department/%d/delete", n), "super", nil) if deleted["code"] != float64(0) { t.Fatal(deleted) } } for attempt := 0; attempt < 2; attempt++ { - _, again := call("POST", "/api/department/create", "admin", map[string]any{"name": "Reusable"}) + _, again := call("POST", "/api/department/create", "super", map[string]any{"name": "Reusable"}) if again["code"] != float64(0) { t.Fatal("deleted sibling name could not be reused") } n := int(again["data"].(map[string]any)["id"].(float64)) - _, removed := call("DELETE", fmt.Sprintf("/api/department/%d/delete", n), "admin", nil) + _, removed := call("DELETE", fmt.Sprintf("/api/department/%d/delete", n), "super", nil) if removed["code"] != float64(0) { t.Fatal("repeated soft deletion collided") } } - _, listed := call("GET", "/api/department/list", "admin", nil) + _, listed := call("GET", "/api/department/list", "super", nil) if listed["data"].(map[string]any)["total"] != float64(0) { t.Fatal("deleted departments remain listed") } diff --git a/internal/system/api/role_handler.go b/internal/system/api/role_handler.go index acf8de0c..3d8dad5f 100644 --- a/internal/system/api/role_handler.go +++ b/internal/system/api/role_handler.go @@ -26,11 +26,15 @@ package api import ( + "errors" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" "github.com/GoSimplicity/AI-CloudOps/internal/system/service" "github.com/GoSimplicity/AI-CloudOps/pkg/base" "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" "github.com/gin-gonic/gin" + "gorm.io/gorm" ) type RoleHandler struct { @@ -166,7 +170,15 @@ func (h *RoleHandler) AssignRolesToUser(ctx *gin.Context) { user := ctx.MustGet("user").(jwt.UserClaims) - base.HandleRequest(ctx, &req, func() (interface{}, error) { + if err := base.BindRequest(ctx, &req); err != nil { + base.BadRequestWithDetails(ctx, err.Error(), "绑定数据失败") + return + } + if !h.guardSuperAdminRoles(ctx, req.UserID, req.RoleIds) { + return + } + + base.HandleRequest(ctx, nil, func() (interface{}, error) { return nil, h.svc.AssignRolesToUser(ctx.Request.Context(), req.UserID, req.RoleIds, user.Uid) }) } @@ -175,11 +187,57 @@ func (h *RoleHandler) AssignRolesToUser(ctx *gin.Context) { func (h *RoleHandler) RevokeRolesFromUser(ctx *gin.Context) { var req model.RevokeRolesFromUserRequest - base.HandleRequest(ctx, &req, func() (interface{}, error) { + if err := base.BindRequest(ctx, &req); err != nil { + base.BadRequestWithDetails(ctx, err.Error(), "绑定数据失败") + return + } + if !h.guardSuperAdminRoles(ctx, req.UserID, req.RoleIds) { + return + } + + base.HandleRequest(ctx, nil, func() (interface{}, error) { return nil, h.svc.RevokeRolesFromUser(ctx.Request.Context(), req.UserID, req.RoleIds) }) } +// guardSuperAdminRoles 授予或撤销内置超管角色只能由超管操作，防止持有角色分配 +// 权限的普通用户把自己（或他人）提升为超管、或撤销超管。roleIds 为空时 DAO 会撤销 +// 目标用户的全部角色，因此按目标用户当前角色判断。返回 false 时已写出响应。 +func (h *RoleHandler) guardSuperAdminRoles(ctx *gin.Context, userID int, roleIds []int) bool { + if permission.IsSuperAdmin(ctx) { + return true + } + touches := false + if len(roleIds) == 0 { + roles, err := h.svc.GetUserRoles(ctx.Request.Context(), userID) + if err != nil { + base.ErrorWithMessage(ctx, "获取用户角色失败") + return false + } + if roles != nil { + for _, role := range roles.Items { + touches = touches || permission.IsSuperAdminIdentity(role) + } + } + } + for _, id := range roleIds { + role, err := h.svc.GetRoleByID(ctx.Request.Context(), id) + if errors.Is(err, gorm.ErrRecordNotFound) { + continue + } + if err != nil { + base.ErrorWithMessage(ctx, "获取角色失败") + return false + } + touches = touches || permission.IsSuperAdminIdentity(role) + } + if touches { + base.ForbiddenError(ctx, "仅超级管理员可授予或撤销超级管理员角色") + return false + } + return true +} + func (h *RoleHandler) GetRoleUsers(ctx *gin.Context) { var req model.GetRoleUsersRequest diff --git a/internal/system/api/superadmin_guard_test.go b/internal/system/api/superadmin_guard_test.go new file mode 100644 index 00000000..f44fb78f --- /dev/null +++ b/internal/system/api/superadmin_guard_test.go @@ -0,0 +1,152 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http/httptest" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" + "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" + "gorm.io/gorm" +) + +const ( + superRoleID = 1 + memberRoleID = 2 + fakeRoleID = 3 // 普通角色借用 super_admin 编码 +) + +type guardRoles struct { + service.RoleService + targetRoles []*model.Role + assigned *[]int + revoked *[]int +} + +func (guardRoles) GetRoleByID(_ context.Context, id int) (*model.Role, error) { + switch id { + case superRoleID: + return &model.Role{Model: model.Model{ID: id}, Code: permission.SuperAdminRoleCode, IsSystem: 1, Status: 0}, nil + case memberRoleID: + return &model.Role{Model: model.Model{ID: id}, Code: "member", Status: 1}, nil + case fakeRoleID: + return &model.Role{Model: model.Model{ID: id}, Code: permission.SuperAdminRoleCode, IsSystem: 0, Status: 1}, nil + } + return nil, gorm.ErrRecordNotFound +} +func (r guardRoles) GetUserRoles(context.Context, int) (*model.ListResp[*model.Role], error) { + return &model.ListResp[*model.Role]{Items: r.targetRoles}, nil +} +func (r guardRoles) AssignRolesToUser(_ context.Context, _ int, roleIds []int, _ int) error { + *r.assigned = append(*r.assigned, roleIds...) + return nil +} +func (r guardRoles) RevokeRolesFromUser(_ context.Context, _ int, roleIds []int) error { + *r.revoked = append(*r.revoked, -1) + return nil +} + +func TestOnlySuperAdminCanGrantOrRevokeSuperAdminRole(t *testing.T) { + superTarget := []*model.Role{{Code: permission.SuperAdminRoleCode, IsSystem: 1, Status: 1}} + for _, tt := range []struct { + name string + path string + roleIds []int + targetRoles []*model.Role + callerSuper bool + allowed bool + }{ + {"assign member role", "/api/role/assign_users", []int{memberRoleID}, nil, false, true}, + {"assign custom role reusing code", "/api/role/assign_users", []int{fakeRoleID}, nil, false, true}, + // 即使超管角色被禁用，授予它也必须由超管完成，否则被重新启用时会立刻提权 + {"assign super role by ordinary", "/api/role/assign_users", []int{memberRoleID, superRoleID}, nil, false, false}, + {"assign super role by super", "/api/role/assign_users", []int{superRoleID}, nil, true, true}, + {"assign unknown role", "/api/role/assign_users", []int{99}, nil, false, true}, + {"revoke super role by ordinary", "/api/role/revoke_users", []int{superRoleID}, nil, false, false}, + {"revoke all roles of super admin by ordinary", "/api/role/revoke_users", []int{}, superTarget, false, false}, + {"revoke all roles of member by ordinary", "/api/role/revoke_users", []int{}, nil, false, true}, + {"revoke super role by super", "/api/role/revoke_users", []int{superRoleID}, nil, true, true}, + } { + t.Run(tt.name, func(t *testing.T) { + var assigned, revoked []int + router := gin.New() + router.Use(func(c *gin.Context) { + c.Set("user", jwt.UserClaims{Uid: 7, Username: "operator"}) + permission.MarkSuperAdmin(c, tt.callerSuper) + c.Next() + }) + NewRoleHandler(guardRoles{targetRoles: tt.targetRoles, assigned: &assigned, revoked: &revoked}).RegisterRouters(router) + body, _ := json.Marshal(map[string]any{"user_id": 8, "role_ids": tt.roleIds}) + req := httptest.NewRequest("POST", tt.path, bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + called := len(assigned)+len(revoked) > 0 + if called != tt.allowed { + t.Fatalf("service called=%v want %v (status=%d body=%s)", called, tt.allowed, rec.Code, rec.Body.String()) + } + if !tt.allowed && rec.Code != 403 { + t.Fatalf("status=%d want 403", rec.Code) + } + }) + } +} + +type profileRecorder struct { + service.UserService + called bool + allowAccount bool +} + +func (p *profileRecorder) UpdateProfile(_ context.Context, _ *model.UpdateProfileReq, allow bool) error { + p.called, p.allowAccount = true, allow + return nil +} + +func TestUpdateProfilePrivilegeComesFromSuperAdminRole(t *testing.T) { + for _, tt := range []struct { + name string + username string + super bool + target int + allowed bool + allowAccount bool + }{ + {"self edit", "member", false, 7, true, false}, + {"other user by ordinary", "member", false, 8, false, false}, + {"other user by username admin without role", "admin", false, 8, false, false}, + {"self edit by username admin without role", "admin", false, 7, true, false}, + {"other user by super admin", "ops-lead", true, 8, true, true}, + } { + t.Run(tt.name, func(t *testing.T) { + svc := &profileRecorder{} + router := gin.New() + router.Use(func(c *gin.Context) { + c.Set("user", jwt.UserClaims{Uid: 7, Username: tt.username}) + permission.MarkSuperAdmin(c, tt.super) + c.Next() + }) + NewUserHandler(svc, nil).RegisterRoutes(router) + body, _ := json.Marshal(map[string]any{"real_name": "n", "mobile": "1", "account_type": 1, "home_path": "/", "enable": 2}) + req := httptest.NewRequest("PUT", fmt.Sprintf("/api/user/profile/update/%d", tt.target), bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if svc.called != tt.allowed { + t.Fatalf("called=%v want %v (status=%d)", svc.called, tt.allowed, rec.Code) + } + if !tt.allowed && rec.Code != 403 { + t.Fatalf("status=%d want 403", rec.Code) + } + if svc.allowAccount != tt.allowAccount { + t.Fatalf("allowAccountChanges=%v want %v", svc.allowAccount, tt.allowAccount) + } + }) + } +} diff --git a/internal/system/api/user_handler.go b/internal/system/api/user_handler.go index 7a11e8b7..f8506475 100644 --- a/internal/system/api/user_handler.go +++ b/internal/system/api/user_handler.go @@ -31,6 +31,7 @@ import ( "github.com/GoSimplicity/AI-CloudOps/internal/constants" "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" "github.com/GoSimplicity/AI-CloudOps/internal/system/service" userutils "github.com/GoSimplicity/AI-CloudOps/internal/system/utils" "github.com/GoSimplicity/AI-CloudOps/pkg/base" @@ -41,6 +42,7 @@ import ( type UserHandler struct { service service.UserService jwt jwt2.Handler + routes func() gin.RoutesInfo } func NewUserHandler(service service.UserService, jwt jwt2.Handler) *UserHandler { @@ -51,6 +53,8 @@ func NewUserHandler(service service.UserService, jwt jwt2.Handler) *UserHandler } func (h *UserHandler) RegisterRoutes(server *gin.Engine) { + // 权限目录在请求时按当前已注册路由构建：AIOps 代理等路由在本方法之后才注册 + h.routes = server.Routes userGroup := server.Group("/api/user") { userGroup.POST("/signup", h.SignUp) @@ -59,6 +63,7 @@ func (h *UserHandler) RegisterRoutes(server *gin.Engine) { userGroup.POST("/logout", h.Logout) userGroup.GET("/profile", h.Profile) userGroup.GET("/codes", h.GetPermCode) + userGroup.GET("/permissions", h.GetPermissions) userGroup.GET("/detail/:id", h.GetUserDetail) userGroup.GET("/list", h.GetUserList) userGroup.POST("/change_password", h.ChangePassword) @@ -168,6 +173,26 @@ func (h *UserHandler) GetPermCode(ctx *gin.Context) { }) } +// GetPermissions 返回调用者本人的 v2 权限码（"METHOD /模板"），契约见 docs/api/user-permissions-v2.md。 +func (h *UserHandler) GetPermissions(ctx *gin.Context) { + uc, err := userutils.ExtractClaims(ctx) + if err != nil { + base.ErrorWithMessage(ctx, err.Error()) + return + } + if uc.Uid <= 0 { + base.ErrorWithMessage(ctx, "用户信息缺失") + return + } + var routes gin.RoutesInfo + if h.routes != nil { + routes = h.routes() + } + base.HandleRequest(ctx, nil, func() (interface{}, error) { + return h.service.GetPermissions(ctx.Request.Context(), uc.Uid, permission.BuildCatalog(routes)) + }) +} + func (h *UserHandler) GetUserList(ctx *gin.Context) { var req model.GetUserListReq @@ -188,6 +213,9 @@ func (h *UserHandler) ChangePassword(ctx *gin.Context) { req.UserID = uc.Uid base.HandleRequest(ctx, &req, func() (interface{}, error) { + // SEC-28: 以 token 中的 uid 为准，绑定之后再强制覆盖一次， + // 不依赖 BindRequest 的通用受保护字段机制。 + req.UserID = uc.Uid return nil, h.service.ChangePassword(ctx.Request.Context(), &req) }) } @@ -223,13 +251,14 @@ func (h *UserHandler) UpdateProfile(ctx *gin.Context) { } req.ID = id - if uc.Username != "admin" && uc.Uid != req.ID { + super := permission.IsSuperAdmin(ctx) + if !super && uc.Uid != req.ID { base.ForbiddenError(ctx, "无权限修改该用户信息") return } base.HandleRequest(ctx, &req, func() (interface{}, error) { - return nil, h.service.UpdateProfile(ctx.Request.Context(), &req, uc.Username == "admin") + return nil, h.service.UpdateProfile(ctx.Request.Context(), &req, super) }) } diff --git a/internal/system/api/user_permissions_test.go b/internal/system/api/user_permissions_test.go new file mode 100644 index 00000000..f47830eb --- /dev/null +++ b/internal/system/api/user_permissions_test.go @@ -0,0 +1,213 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http/httptest" + "strings" + "testing" + + k8sapi "github.com/GoSimplicity/AI-CloudOps/internal/k8s/api" + "github.com/GoSimplicity/AI-CloudOps/internal/middleware" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" + "github.com/GoSimplicity/AI-CloudOps/internal/system/dao" + "github.com/GoSimplicity/AI-CloudOps/internal/system/service" + treeapi "github.com/GoSimplicity/AI-CloudOps/internal/tree/api" + "github.com/GoSimplicity/AI-CloudOps/pkg/jwt" + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +// userRolesDAO 模拟 roleDAO.GetRoles：与线上一致只返回启用中的角色。 +type userRolesDAO struct { + dao.RoleDAO + roles []*model.Role + err error +} + +func (d userRolesDAO) GetRoles(context.Context, int) ([]*model.Role, error) { + if d.err != nil { + return nil, d.err + } + var enabled []*model.Role + for _, role := range d.roles { + if role.Status == 1 { + enabled = append(enabled, role) + } + } + return enabled, nil +} + +// middlewareRoles 让鉴权中间件读取与权限接口相同的角色来源。 +type middlewareRoles struct { + service.RoleService + dao userRolesDAO +} + +func (r middlewareRoles) GetUserRoles(ctx context.Context, uid int) (*model.ListResp[*model.Role], error) { + roles, err := r.dao.GetRoles(ctx, uid) + if err != nil { + return nil, err + } + return &model.ListResp[*model.Role]{Items: roles}, nil +} + +var testAIOpsCatalog = []permission.Route{ + {Method: "GET", Path: "/api/v1/assistant/config"}, + {Method: "GET", Path: "/api/v1/assistant/session/:session_id"}, + {Method: "POST", Path: "/api/v1/assistant/query"}, + {Method: "POST", Path: "/api/v1/autofix"}, + {Method: "POST", Path: "/api/v1/autofix/"}, +} + +// permissionsRouter 按线上顺序装配：登录态 -> 鉴权 -> 真实 handler 路由（业务 handler 由终止中间件代替）。 +func permissionsRouter(t *testing.T, roles userRolesDAO, uid int) *gin.Engine { + t.Helper() + gin.SetMode(gin.TestMode) + previous := permission.ProxyCatalog(permission.AIOpsProxyTemplate) + permission.SetProxyCatalog(permission.AIOpsProxyTemplate, testAIOpsCatalog) + t.Cleanup(func() { permission.SetProxyCatalog(permission.AIOpsProxyTemplate, previous) }) + + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: uid, Username: "operator"}); c.Next() }) + engine.Use(middleware.NewAuthMiddleware(middlewareRoles{dao: roles}).CheckAuth()) + engine.Use(func(c *gin.Context) { + if c.FullPath() != "/api/user/permissions" { + c.Header("X-Matched", c.FullPath()) + c.AbortWithStatus(204) + } + }) + NewUserHandler(service.NewUserService(nil, roles, zap.NewNop()), nil).RegisterRoutes(engine) + k8sapi.NewK8sEventHandler(nil).RegisterRouters(engine) + treeapi.NewCloudAccountHandler(nil).RegisterRouters(engine) + treeapi.NewTreeCloudHandler(nil, nil, nil, nil).RegisterRouters(engine) + NewDepartmentHandler(nil).RegisterRouters(engine) + // 与 registerAIOps 一样在其他路由之后注册代理 + engine.Any(permission.AIOpsProxyTemplate, func(c *gin.Context) { c.Status(204) }) + return engine +} + +type permissionsBody struct { + Code int `json:"code"` + Data model.UserPermissionsResp `json:"data"` + Raw map[string]json.RawMessage +} + +func fetchPermissions(t *testing.T, engine *gin.Engine) (permissionsBody, string) { + t.Helper() + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", "/api/user/permissions", nil)) + if rec.Code != 200 { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } + var body permissionsBody + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + return body, rec.Body.String() +} + +func TestUserPermissionsExpandsGrantsToMethodTemplates(t *testing.T) { + roles := userRolesDAO{roles: []*model.Role{ + {Status: 1, Apis: []*model.Api{ + {Method: 1, Path: "/api/k8s/clusters/:cluster_id/events/:namespace/:name"}, + {Method: 3, Path: "/api/tree/cloud/account/:id/status"}, + {Method: 1, Path: "/api/v1/assistant/*"}, + {Method: 1, Path: "/api/tree/*/list"}, // 中间通配：不再支持 + {Method: 2, Path: "/api/department/*"}, // 超管专属：授予无效 + {Method: 2, Path: "/api/user/signup"}, // 超管专属：授予无效 + {Method: 1, Path: "/api/user/codes"}, // 不经授权的接口不列出 + {Method: 1, Path: "/api/tree/cloud/9/detail"}, // 真实路径不是模板 + }}, + {Status: 0, Apis: []*model.Api{{Method: 4, Path: "/*"}}}, // 禁用角色 + }} + body, raw := fetchPermissions(t, permissionsRouter(t, roles, 5)) + want := []string{ + "GET /api/k8s/clusters/:cluster_id/events/:namespace/:name", + "GET /api/v1/assistant/config", + "GET /api/v1/assistant/session/:session_id", + "PUT /api/tree/cloud/account/:id/status", + } + if body.Code != 0 || body.Data.Version != 2 || body.Data.SuperAdmin || strings.Join(body.Data.Permissions, "\n") != strings.Join(want, "\n") { + t.Fatalf("unexpected response: %s", raw) + } +} + +func TestUserPermissionsForSuperAdminAndEmptyUser(t *testing.T) { + super := userRolesDAO{roles: []*model.Role{{Code: permission.SuperAdminRoleCode, IsSystem: 1, Status: 1}}} + body, raw := fetchPermissions(t, permissionsRouter(t, super, 1)) + if !body.Data.SuperAdmin || len(body.Data.Permissions) != 1 || body.Data.Permissions[0] != permission.SuperCode { + t.Fatalf("super admin: %s", raw) + } + + // 借用 super_admin 编码的普通角色不是超管 + impostor := userRolesDAO{roles: []*model.Role{{Code: permission.SuperAdminRoleCode, IsSystem: 0, Status: 1}}} + body, raw = fetchPermissions(t, permissionsRouter(t, impostor, 3)) + if body.Data.SuperAdmin || body.Data.Permissions == nil || len(body.Data.Permissions) != 0 { + t.Fatalf("impostor: %s", raw) + } + if !strings.Contains(raw, `"permissions":[]`) { + t.Fatalf("permissions must serialise as [] not null: %s", raw) + } +} + +func TestUserPermissionsReportsRoleLookupFailure(t *testing.T) { + engine := gin.New() + engine.Use(func(c *gin.Context) { c.Set("user", jwt.UserClaims{Uid: 5}); c.Next() }) + NewUserHandler(service.NewUserService(nil, userRolesDAO{err: errors.New("db down")}, zap.NewNop()), nil).RegisterRoutes(engine) + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest("GET", "/api/user/permissions", nil)) + var body permissionsBody + _ = json.Unmarshal(rec.Body.Bytes(), &body) + if body.Code == 0 || len(body.Data.Permissions) != 0 { + t.Fatalf("lookup failure must not return permissions: %s", rec.Body.String()) + } +} + +// 权限接口展开的列表必须与鉴权中间件的实际判定逐条一致。 +func TestUserPermissionsAgreeWithAuthorizationMiddleware(t *testing.T) { + for name, apis := range map[string][]*model.Api{ + "exact": {{Method: 1, Path: "/api/k8s/clusters/:cluster_id/events/:namespace/:name"}, {Method: 3, Path: "/api/tree/cloud/:id/status"}}, + "module": {{Method: 3, Path: "/api/tree/*"}, {Method: 1, Path: "/api/k8s/clusters/:cluster_id/events/*"}}, + "proxy prefix": {{Method: 2, Path: "/api/v1/autofix*"}, {Method: 1, Path: "/api/v1/assistant/session/:session_id"}}, + "whole proxy": {{Method: 2, Path: permission.AIOpsProxyTemplate}}, + "global": {{Method: 1, Path: "/*"}, {Method: 4, Path: "/*"}}, + } { + t.Run(name, func(t *testing.T) { + roles := userRolesDAO{roles: []*model.Role{{Status: 1, Apis: apis}}} + engine := permissionsRouter(t, roles, 5) + body, _ := fetchPermissions(t, engine) + granted := map[string]bool{} + for _, code := range body.Data.Permissions { + granted[code] = true + } + checked := 0 + for _, entry := range permission.BuildCatalog(engine.Routes()) { + if strings.Contains(entry.Path, "*") { + continue // gin 通配模板本身没有唯一的具体请求路径 + } + concrete := entry.Path + for _, part := range strings.Split(entry.Path, "/") { + if strings.HasPrefix(part, ":") { + concrete = strings.Replace(concrete, part, "v"+part[1:], 1) + } + } + rec := httptest.NewRecorder() + engine.ServeHTTP(rec, httptest.NewRequest(permission.MethodName(entry.Method), concrete, nil)) + if matched := rec.Header().Get("X-Matched"); rec.Code == 204 && entry.Via == "" && matched != entry.Path { + t.Fatalf("%s routed to %s", concrete, matched) + } + allowed := rec.Code == 204 + if allowed != granted[entry.Code()] { + t.Errorf("%s: middleware allowed=%v, permissions listed=%v", entry.Code(), allowed, granted[entry.Code()]) + } + checked++ + } + if checked < 40 { + t.Fatalf("catalog too small: %d", checked) + } + }) + } +} diff --git a/internal/system/service/audit_service.go b/internal/system/service/audit_service.go index 319a7bc3..d923f091 100644 --- a/internal/system/service/audit_service.go +++ b/internal/system/service/audit_service.go @@ -207,6 +207,7 @@ func (s *auditService) GetAuditTypes(ctx context.Context) ([]model.AuditTypeInfo {Type: "UPDATE", Description: "更新操作", Category: "数据操作"}, {Type: "DELETE", Description: "删除操作", Category: "数据操作"}, {Type: "VIEW", Description: "查看操作", Category: "数据访问"}, + {Type: "REVEAL", Description: "查看敏感明文（如 Secret reveal）", Category: "数据访问"}, {Type: "LOGIN", Description: "登录操作", Category: "身份认证"}, {Type: "LOGOUT", Description: "登出操作", Category: "身份认证"}, {Type: "EXPORT", Description: "导出操作", Category: "数据导出"}, diff --git a/internal/system/service/change_password_regression_test.go b/internal/system/service/change_password_regression_test.go new file mode 100644 index 00000000..2cfda05c --- /dev/null +++ b/internal/system/service/change_password_regression_test.go @@ -0,0 +1,61 @@ +package service + +import ( + "context" + "errors" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/constants" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/system/dao" + userutils "github.com/GoSimplicity/AI-CloudOps/internal/system/utils" + "go.uber.org/zap" +) + +type changePasswordDAO struct { + dao.UserDAO + users map[int]*model.User + changedID int + changed string +} + +func (d *changePasswordDAO) GetByID(_ context.Context, id int) (*model.User, error) { + if u, ok := d.users[id]; ok { + return u, nil + } + return nil, errors.New("not found") +} +func (d *changePasswordDAO) ChangePassword(_ context.Context, uid int, h string) error { + d.changedID, d.changed = uid, h + return nil +} + +func TestChangePasswordVerifiesOldPasswordOfTokenUser(t *testing.T) { + selfOld, _ := userutils.HashPassword("self-old-x") + otherOld, _ := userutils.HashPassword("other-old-x") + d := &changePasswordDAO{users: map[int]*model.User{ + 7: {Model: model.Model{ID: 7}, Password: selfOld}, + 99: {Model: model.Model{ID: 99}, Password: otherOld}, + }} + s := NewUserService(d, nil, zap.NewNop()) + err := s.ChangePassword(context.Background(), &model.ChangePasswordReq{UserID: 7, Password: "wrong-x", NewPassword: "new-pass-x", ConfirmPassword: "new-pass-x"}) + if !errors.Is(err, constants.ErrorPasswordIncorrect) || d.changedID != 0 { + t.Fatalf("wrong old password accepted: %v", err) + } + err = s.ChangePassword(context.Background(), &model.ChangePasswordReq{UserID: 7, Password: "self-old-x", NewPassword: "new-pass-x", ConfirmPassword: "new-pass-x"}) + if err != nil || d.changedID != 7 || userutils.ComparePassword(d.changed, "new-pass-x") != nil { + t.Fatalf("correct old password failed: %v", err) + } + t.Logf("wrong old pwd rejected; correct old pwd -> dao.ChangePassword(uid=%d, bcrypt hash)", d.changedID) +} + +func TestChangePasswordRejectsNonPositiveUserID(t *testing.T) { + d := &changePasswordDAO{users: map[int]*model.User{}} + s := NewUserService(d, nil, zap.NewNop()) + for _, uid := range []int{0, -1} { + err := s.ChangePassword(context.Background(), &model.ChangePasswordReq{UserID: uid, Password: "old-x", NewPassword: "new-pass-x", ConfirmPassword: "new-pass-x"}) + if err == nil || d.changedID != 0 { + t.Fatalf("uid=%d should be rejected, err=%v", uid, err) + } + } +} diff --git a/internal/system/service/user_service.go b/internal/system/service/user_service.go index f968cc88..ad124b18 100644 --- a/internal/system/service/user_service.go +++ b/internal/system/service/user_service.go @@ -33,6 +33,7 @@ import ( "github.com/GoSimplicity/AI-CloudOps/internal/constants" "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/internal/permission" "github.com/GoSimplicity/AI-CloudOps/internal/system/dao" userutils "github.com/GoSimplicity/AI-CloudOps/internal/system/utils" "go.uber.org/zap" @@ -44,6 +45,7 @@ type UserService interface { Login(ctx context.Context, user *model.UserLoginReq) (*model.User, error) GetProfile(ctx context.Context, uid int) (*model.User, error) GetPermCode(ctx context.Context, uid int) ([]string, error) + GetPermissions(ctx context.Context, uid int, catalog []permission.CatalogEntry) (*model.UserPermissionsResp, error) GetUserDetail(ctx context.Context, uid int) (*model.User, error) GetUserList(ctx context.Context, req *model.GetUserListReq) (model.ListResp[*model.User], error) ChangePassword(ctx context.Context, req *model.ChangePasswordReq) error @@ -125,6 +127,23 @@ func (us *userService) GetPermCode(ctx context.Context, uid int) ([]string, erro return codes, nil } +// GetPermissions 把用户角色授予按与鉴权中间件相同的规则展开为 "METHOD /模板" 权限码。 +// 超管只返回超管码 "*"。 +func (us *userService) GetPermissions(ctx context.Context, uid int, catalog []permission.CatalogEntry) (*model.UserPermissionsResp, error) { + roles, err := us.roleDao.GetRoles(ctx, uid) + if err != nil { + us.logger.Error("获取用户角色失败", zap.Int("uid", uid), zap.Error(err)) + return nil, err + } + if permission.HasSuperAdminRole(roles) { + return &model.UserPermissionsResp{Version: model.UserPermissionsVersion, SuperAdmin: true, Permissions: []string{permission.SuperCode}}, nil + } + return &model.UserPermissionsResp{ + Version: model.UserPermissionsVersion, + Permissions: permission.Expand(permission.GrantsFromRoles(roles), catalog), + }, nil +} + func (us *userService) GetUserList(ctx context.Context, req *model.GetUserListReq) (model.ListResp[*model.User], error) { page := req.Page if page <= 0 { @@ -165,6 +184,10 @@ func (us *userService) GetUserList(ctx context.Context, req *model.GetUserListRe // ChangePassword 修改密码 func (us *userService) ChangePassword(ctx context.Context, req *model.ChangePasswordReq) error { + if req == nil || req.UserID <= 0 { + us.logger.Error("修改密码缺少有效的用户ID") + return errors.New("无效的用户ID") + } if req.Password == req.NewPassword { us.logger.Error("新密码不能与旧密码相同") return errors.New("新密码不能与旧密码相同") diff --git a/mock/user_mock.go b/mock/user_mock.go index b463fb70..2d4205b5 100644 --- a/mock/user_mock.go +++ b/mock/user_mock.go @@ -26,42 +26,87 @@ package mock import ( + "errors" + "fmt" "log" + "os" + "github.com/GoSimplicity/AI-CloudOps/internal/constants" "github.com/GoSimplicity/AI-CloudOps/internal/model" + "github.com/GoSimplicity/AI-CloudOps/pkg/envmode" "golang.org/x/crypto/bcrypt" "gorm.io/gorm" ) const ( - AdminUsername = "admin" - AdminPassword = "admin" - AdminAccountType = 2 + AdminUsername = "admin" + // AdminAccountType 与 bootstrapAdmin 创建的管理员保持一致：管理员是人工登录账号，不是服务账号。 + AdminAccountType = constants.AccountTypeUser + + // MockAdminPasswordEnv 是 mock 管理员口令的环境变量名。未设置时不创建 mock 管理员。 + MockAdminPasswordEnv = "MOCK_ADMIN_PASSWORD" + // MinMockAdminPasswordLen 与 BOOTSTRAP_ADMIN_PASSWORD 的最小长度保持一致。 + MinMockAdminPasswordLen = 12 ) +// ErrMockAdminPasswordTooShort 表示 MOCK_ADMIN_PASSWORD 长度不足。 +var ErrMockAdminPasswordTooShort = errors.New(MockAdminPasswordEnv + " 至少12字符") + type UserMock struct { - db *gorm.DB + db *gorm.DB + getenv func(string) string } func NewUserMock(db *gorm.DB) *UserMock { return &UserMock{ - db: db, + db: db, + getenv: takeSecretEnv, + } +} + +// takeSecretEnv 读取口令类环境变量后立即从进程环境中删除，避免被 cron 等子进程继承。 +func takeSecretEnv(key string) string { + value := os.Getenv(key) + if err := os.Unsetenv(key); err != nil { + log.Printf("[SECURITY] 清除环境变量 %s 失败，子进程可能继承该变量: %v\n", key, err) } + return value } +// CreateUserAdmin 在空用户表上创建 mock 管理员。 +// 口令只从环境变量 MOCK_ADMIN_PASSWORD 读取，代码中不再内置任何口令； +// 口令读取后立即从进程环境中删除；未设置时跳过创建（不生成、不落盘、不打印口令）。 +// 设置但不足 12 字符时：生产环境返回错误；非生产环境打印 [SECURITY] 告警并跳过创建。 func (u *UserMock) CreateUserAdmin() error { + // 先读取（并清除）口令，保证即使用户表非空也不会把口令留在进程环境中。 + password := u.getenv(MockAdminPasswordEnv) + // 检查是否已经初始化过用户 var count int64 - u.db.Model(&model.User{}).Count(&count) + if err := u.db.Model(&model.User{}).Count(&count).Error; err != nil { + return fmt.Errorf("查询用户数量失败: %w", err) + } if count > 0 { log.Println("[用户已经初始化过,跳过Mock]") return nil } + if password == "" { + log.Printf("[用户模块Mock] 未设置 %s，跳过创建 mock 管理员；如需本地管理员，请设置该变量（至少%d字符）或使用 BOOTSTRAP_ADMIN_PASSWORD\n", MockAdminPasswordEnv, MinMockAdminPasswordLen) + return nil + } + if len(password) < MinMockAdminPasswordLen { + if envmode.IsProduction() { + return ErrMockAdminPasswordTooShort + } + log.Printf("[SECURITY] %s 少于%d字符，非生产环境跳过创建 mock 管理员；请改为至少%d字符的强口令后重启\n", MockAdminPasswordEnv, MinMockAdminPasswordLen, MinMockAdminPasswordLen) + return nil + } + log.Println("[用户模块Mock开始]") // 生成加密密码 - hashedPassword, err := bcrypt.GenerateFromPassword([]byte(AdminPassword), bcrypt.DefaultCost) + hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { log.Printf("生成密码失败: %v\n", err) log.Println("[用户模块Mock结束]") diff --git a/mock/user_mock_test.go b/mock/user_mock_test.go new file mode 100644 index 00000000..cb094c81 --- /dev/null +++ b/mock/user_mock_test.go @@ -0,0 +1,211 @@ +package mock + +import ( + "bytes" + "errors" + "log" + "os" + "strings" + "testing" + + "github.com/GoSimplicity/AI-CloudOps/internal/constants" + "github.com/GoSimplicity/AI-CloudOps/internal/model" + "golang.org/x/crypto/bcrypt" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// 测试用口令，运行时拼接，不是任何环境的真实口令 +var fixturePassword = "mock-fixture-" + strings.Repeat("x", 6) + "-0001" + +func newUserMockDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + pool.SetMaxOpenConns(1) + t.Cleanup(func() { _ = pool.Close() }) + if err := db.AutoMigrate(&model.User{}); err != nil { + t.Fatal(err) + } + return db +} + +func captureLog(t *testing.T) *bytes.Buffer { + t.Helper() + buf := &bytes.Buffer{} + log.SetOutput(buf) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + return buf +} + +func newMockWithEnv(db *gorm.DB, value string) *UserMock { + m := NewUserMock(db) + m.getenv = func(key string) string { + if key == MockAdminPasswordEnv { + return value + } + return "" + } + return m +} + +func countUsers(t *testing.T, db *gorm.DB) int64 { + t.Helper() + var n int64 + if err := db.Model(&model.User{}).Count(&n).Error; err != nil { + t.Fatal(err) + } + return n +} + +func TestCreateUserAdminSkipsWithoutEnvPassword(t *testing.T) { + db := newUserMockDB(t) + logs := captureLog(t) + if err := newMockWithEnv(db, "").CreateUserAdmin(); err != nil { + t.Fatal(err) + } + if n := countUsers(t, db); n != 0 { + t.Fatalf("未设置 %s 时不应创建管理员, users=%d", MockAdminPasswordEnv, n) + } + if !strings.Contains(logs.String(), MockAdminPasswordEnv) { + t.Fatal("跳过时应提示如何设置口令") + } +} + +func TestCreateUserAdminRejectsShortPassword(t *testing.T) { + t.Setenv("ENV", "production") + db := newUserMockDB(t) + short := strings.Repeat("s", MinMockAdminPasswordLen-1) + logs := captureLog(t) + if err := newMockWithEnv(db, short).CreateUserAdmin(); !errors.Is(err, ErrMockAdminPasswordTooShort) { + t.Fatalf("err=%v", err) + } + if n := countUsers(t, db); n != 0 { + t.Fatalf("users=%d", n) + } + if strings.Contains(logs.String(), short) { + t.Fatal("日志不得包含口令") + } +} + +func TestCreateUserAdminUsesEnvPasswordAndNeverLogsIt(t *testing.T) { + db := newUserMockDB(t) + logs := captureLog(t) + if err := newMockWithEnv(db, fixturePassword).CreateUserAdmin(); err != nil { + t.Fatal(err) + } + var admin model.User + if err := db.Where("username = ?", AdminUsername).First(&admin).Error; err != nil { + t.Fatal(err) + } + if admin.Password == fixturePassword { + t.Fatal("口令不得明文入库") + } + if bcrypt.CompareHashAndPassword([]byte(admin.Password), []byte(fixturePassword)) != nil { + t.Fatal("入库哈希应对应环境变量中的口令") + } + if strings.Contains(logs.String(), fixturePassword) { + t.Fatal("日志不得包含口令") + } +} + +func TestCreateUserAdminDoesNotTouchExistingUsers(t *testing.T) { + db := newUserMockDB(t) + if err := db.Create(&model.User{Username: "someone", Password: "hash", RealName: "x", Enable: 1}).Error; err != nil { + t.Fatal(err) + } + if err := newMockWithEnv(db, fixturePassword).CreateUserAdmin(); err != nil { + t.Fatal(err) + } + if n := countUsers(t, db); n != 1 { + t.Fatalf("已有用户时不应再创建管理员, users=%d", n) + } +} + +func TestNewUserMockReadsProcessEnv(t *testing.T) { + t.Setenv(MockAdminPasswordEnv, fixturePassword) + if got := NewUserMock(nil).getenv(MockAdminPasswordEnv); got != fixturePassword { + t.Fatal("默认应从进程环境变量读取口令") + } + if os.Getenv(MockAdminPasswordEnv) != "" { + t.Fatal("读取后应立即清除环境变量") + } + if _, ok := os.LookupEnv(MockAdminPasswordEnv); ok { + t.Fatal("读取后应从进程环境中删除") + } +} + +func TestCreateUserAdminWarnsOnShortPasswordOutsideProduction(t *testing.T) { + for _, env := range []string{"development", "Dev", " dev "} { + t.Run(env, func(t *testing.T) { + t.Setenv("ENV", env) + db := newUserMockDB(t) + short := strings.Repeat("s", MinMockAdminPasswordLen-1) + logs := captureLog(t) + if err := newMockWithEnv(db, short).CreateUserAdmin(); err != nil { + t.Fatalf("非生产环境短口令只告警: %v", err) + } + if n := countUsers(t, db); n != 0 { + t.Fatalf("短口令不应创建管理员, users=%d", n) + } + out := logs.String() + if !strings.Contains(out, "[SECURITY]") || !strings.Contains(out, MockAdminPasswordEnv) { + t.Fatal("应打印显眼的 [SECURITY] 告警并指明变量") + } + if strings.Contains(out, short) { + t.Fatal("日志不得包含口令") + } + }) + } +} + +func TestCreateUserAdminCountErrorPropagates(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + t.Cleanup(func() { _ = pool.Close() }) + logs := captureLog(t) + // 未建表：用户计数查询必然失败，错误必须透传而不是被当作空表继续创建 + err = newMockWithEnv(db, fixturePassword).CreateUserAdmin() + if err == nil || !strings.Contains(err.Error(), "查询用户数量失败") { + t.Fatalf("err=%v", err) + } + if strings.Contains(err.Error(), fixturePassword) || strings.Contains(logs.String(), fixturePassword) { + t.Fatal("错误与日志不得包含口令") + } +} + +func TestCreateUserAdminClearsProcessEnvEvenWhenUsersExist(t *testing.T) { + t.Setenv("ENV", "development") + t.Setenv(MockAdminPasswordEnv, fixturePassword) + db := newUserMockDB(t) + if err := db.Create(&model.User{Username: "someone", Password: "hash", RealName: "x", Enable: 1}).Error; err != nil { + t.Fatal(err) + } + if err := NewUserMock(db).CreateUserAdmin(); err != nil { + t.Fatal(err) + } + if _, ok := os.LookupEnv(MockAdminPasswordEnv); ok { + t.Fatal("即使跳过创建也应清除口令环境变量") + } +} + +func TestCreateUserAdminUsesUserAccountType(t *testing.T) { + t.Setenv("ENV", "development") + db := newUserMockDB(t) + if err := newMockWithEnv(db, fixturePassword).CreateUserAdmin(); err != nil { + t.Fatal(err) + } + var admin model.User + if err := db.Where("username = ?", AdminUsername).First(&admin).Error; err != nil { + t.Fatal(err) + } + if admin.AccountType != constants.AccountTypeUser || AdminAccountType != constants.AccountTypeUser { + t.Fatalf("mock 管理员应与 bootstrap 一致使用 AccountTypeUser, got=%d", admin.AccountType) + } +} diff --git a/pkg/base/apiresponse.go b/pkg/base/apiresponse.go index f74586f8..25833e42 100644 --- a/pkg/base/apiresponse.go +++ b/pkg/base/apiresponse.go @@ -98,6 +98,20 @@ func BadRequest(c *gin.Context, code int, data interface{}, message string) { }) } +// Conflict 资源版本冲突的返回，使用HTTP 409状态码 +func Conflict(c *gin.Context, code int, data interface{}, message string) { + c.JSON(http.StatusConflict, ApiResponse{ + Code: code, + Data: data, + Message: message, + }) +} + +// ConflictError 资源版本冲突（乐观锁失败）的返回：HTTP 409 + code=1 +func ConflictError(c *gin.Context, message string) { + Conflict(c, StatusError, map[string]interface{}{}, message) +} + // Forbidden 无权限的返回，使用HTTP 403状态码 func Forbidden(c *gin.Context, data interface{}, message string) { c.JSON(http.StatusForbidden, ApiResponse{ diff --git a/pkg/di/wire_gen.go b/pkg/di/wire_gen.go index bf1584d0..dc365769 100644 --- a/pkg/di/wire_gen.go +++ b/pkg/di/wire_gen.go @@ -128,7 +128,7 @@ func ProvideApp(cfg *config.Config) (*app.App, error) { statefulSetService := service3.NewStatefulSetService(statefulSetManager, logger) k8sStatefulSetHandler := api3.NewK8sStatefulSetHandler(statefulSetService) serviceAccountManager := manager.NewServiceAccountManager(logger, k8sClient) - serviceAccountService := service3.NewServiceAccountService(serviceAccountManager, logger) + serviceAccountService := service3.NewServiceAccountService(serviceAccountManager, auditService, logger) k8sServiceAccountHandler := api3.NewK8sServiceAccountHandler(serviceAccountService) roleManager := manager.NewRoleManager(k8sClient, logger) serviceRoleService := service3.NewRoleService(roleManager, logger) @@ -146,7 +146,7 @@ func ProvideApp(cfg *config.Config) (*app.App, error) { configMapService := service3.NewConfigMapService(k8sClient, configMapManager, logger) k8sConfigMapHandler := api3.NewK8sConfigMapHandler(configMapService) secretManager := manager.NewSecretManager(k8sClient, logger) - secretService := service3.NewSecretService(secretManager, logger) + secretService := service3.NewSecretService(secretManager, auditService, logger) k8sSecretHandler := api3.NewK8sSecretHandler(secretService) alertManagerEventDAO := alert.NewAlertManagerEventDAO(db, logger, userDAO) scrapePoolDAO := scrape.NewScrapePoolDAO(db, logger, userDAO) diff --git a/pkg/envmode/envmode.go b/pkg/envmode/envmode.go new file mode 100644 index 00000000..48814190 --- /dev/null +++ b/pkg/envmode/envmode.go @@ -0,0 +1,83 @@ +// Package envmode 统一解析后端进程的运行环境（环境变量 ENV）。 +// +// 只依赖标准库，供 internal/config、pkg/ssh 等包复用同一套判定， +// 避免各处直接比较字符串而因大小写、空白或别名把生产环境误判为非生产环境。 +// +// 规则（窝窝头 2026-10-24 授权口径）：先 TrimSpace 再转小写； +// production、prod、prd 归为 production；development、dev 归为 development； +// 未设置、空串（含只有空白）和其它任何取值（包括 test、testing）一律拒绝。 +package envmode + +import ( + "errors" + "fmt" + "os" + "strings" + "unicode/utf8" +) + +// EnvKey 是承载运行环境的环境变量名。 +const EnvKey = "ENV" + +// Mode 表示规范化后的运行环境。 +type Mode string + +const ( + // Development 开发环境，对应 config/config.development.yaml。 + Development Mode = "development" + // Production 生产环境，对应 config/config.production.yaml，并启用全部生产校验。 + Production Mode = "production" +) + +// Allowed 是错误信息里列出的允许取值（大小写不敏感，首尾空白忽略）。 +const Allowed = "production/prod/prd（生产）、development/dev（开发）" + +var ( + // ErrUnset 表示 ENV 未设置、为空串或只有空白。 + ErrUnset = errors.New("环境变量 ENV 未设置或为空，允许的取值：" + Allowed) + // ErrUnknown 表示 ENV 取值不在允许列表里。 + ErrUnknown = errors.New("环境变量 ENV 取值无法识别") +) + +// maxShownLen 是错误信息里回显 ENV 原值的最大字符数；更长的值只报长度， +// 避免把误填进 ENV 的长串（例如口令）原样打进日志。 +const maxShownLen = 16 + +var aliases = map[string]Mode{ + "production": Production, + "prod": Production, + "prd": Production, + "development": Development, + "dev": Development, +} + +// Parse 按上面的规则解析 raw。失败时返回 ErrUnset 或包装了 ErrUnknown 的错误。 +func Parse(raw string) (Mode, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "", ErrUnset + } + if m, ok := aliases[strings.ToLower(trimmed)]; ok { + return m, nil + } + shown := fmt.Sprintf("%q", trimmed) + if n := utf8.RuneCountInString(trimmed); n > maxShownLen { + shown = fmt.Sprintf("（长度 %d，不回显）", n) + } + return "", fmt.Errorf("%w: %s，允许的取值：%s", ErrUnknown, shown, Allowed) +} + +// Current 解析当前进程的 ENV。 +func Current() (Mode, error) { + return Parse(os.Getenv(EnvKey)) +} + +// IsProduction 判断当前进程是否按生产环境处理。 +// +// 失败即关闭：只有 ENV 明确解析为 development 时返回 false； +// 生产别名、未设置、空串和无法识别的取值都返回 true。 +// 启动入口 config.Load 已对后三种直接拒绝启动，这里兜住绕过 Load 的调用方。 +func IsProduction() bool { + m, err := Current() + return err != nil || m == Production +} diff --git a/pkg/envmode/envmode_test.go b/pkg/envmode/envmode_test.go new file mode 100644 index 00000000..79911a09 --- /dev/null +++ b/pkg/envmode/envmode_test.go @@ -0,0 +1,158 @@ +package envmode + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" + "testing" + "unicode/utf8" +) + +// unsetForTest 在 t.Setenv 登记恢复之后真正删掉 ENV，覆盖“未设置”而不只是“空串”。 +func unsetForTest(t *testing.T) { + t.Helper() + if err := os.Unsetenv(EnvKey); err != nil { + t.Fatal(err) + } +} + +func TestParseAccepts(t *testing.T) { + for raw, want := range map[string]Mode{ + "production": Production, "prod": Production, "prd": Production, + "PRODUCTION": Production, "Prod": Production, "pRd": Production, + " production": Production, "prod \t": Production, "\n PRD \r\n": Production, + "development": Development, "dev": Development, + "DEVELOPMENT": Development, "Dev": Development, "\tdev ": Development, + } { + got, err := Parse(raw) + if err != nil || got != want { + t.Errorf("Parse(%q) = %q, %v；want %q, nil", raw, got, err, want) + } + } +} + +func TestParseRejectsUnsetAndBlank(t *testing.T) { + for _, raw := range []string{"", " ", "\t", "\r\n", " \t \n "} { + got, err := Parse(raw) + if !errors.Is(err, ErrUnset) || got != "" { + t.Errorf("Parse(%q) = %q, %v；want ErrUnset", raw, got, err) + } + } +} + +func TestParseRejectsOthers(t *testing.T) { + for _, raw := range []string{ + "test", "testing", "TEST", "staging", "local", "pro", "produc", "production1", + "prod-cn", "dev2", "devel", "develop", "p rod", "prod\x00", "ｐｒｏｄ", "\"prod\"", "'dev'", + } { + got, err := Parse(raw) + if !errors.Is(err, ErrUnknown) || got != "" { + t.Errorf("Parse(%q) = %q, %v；want ErrUnknown", raw, got, err) + continue + } + if !strings.Contains(err.Error(), Allowed) { + t.Errorf("Parse(%q) 的错误没有列出允许取值：%v", raw, err) + } + } + if !strings.Contains(ErrUnset.Error(), Allowed) { + t.Errorf("ErrUnset 没有列出允许取值：%v", ErrUnset) + } +} + +func TestParseErrorDoesNotEchoLongValues(t *testing.T) { + secret := "sk-" + strings.Repeat("Z9", 20) + _, err := Parse(secret) + if err == nil || strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), "Z9Z9") { + t.Fatalf("长值被回显：%v", err) + } + if !strings.Contains(err.Error(), "长度 43") { + t.Fatalf("长值应只报长度：%v", err) + } + // 短值用 %q 回显，控制字符被转义而不是原样写进日志 + _, err = Parse("ab\x1b[0m") + if err == nil || strings.Contains(err.Error(), "\x1b") || !strings.Contains(err.Error(), `"ab\x1b[0m"`) { + t.Fatalf("短值应经 %%q 转义回显：%v", err) + } +} + +func TestCurrentAndIsProduction(t *testing.T) { + for _, tt := range []struct { + raw string + set bool + mode Mode + wantErr bool + prod bool + }{ + {"", false, "", true, true}, + {"", true, "", true, true}, + {" ", true, "", true, true}, + {"Prod", true, Production, false, true}, + {" prd ", true, Production, false, true}, + {"DEV", true, Development, false, false}, + {"development", true, Development, false, false}, + {"testing", true, "", true, true}, + {"staging", true, "", true, true}, + } { + if tt.set { + t.Setenv(EnvKey, tt.raw) + } else { + t.Setenv(EnvKey, "") + unsetForTest(t) + } + m, err := Current() + if m != tt.mode || (err != nil) != tt.wantErr { + t.Errorf("ENV=%q(set=%v): Current() = %q, %v", tt.raw, tt.set, m, err) + } + if got := IsProduction(); got != tt.prod { + t.Errorf("ENV=%q(set=%v): IsProduction() = %v, want %v", tt.raw, tt.set, got, tt.prod) + } + } +} + +// P2-2：回显上限按字符（rune）计数，16 个字符以内经 %q 回显，17 个起只报长度。 +// 多字节用例防止按字节计数（M10），16/17 两侧用例防止边界错位（M12）。 +func TestParseEchoBoundaryCountsRunes(t *testing.T) { + for _, tt := range []struct { + name string + raw string + runes int + shown bool + }{ + {"ascii-16", strings.Repeat("q", 16), 16, true}, + {"ascii-17", strings.Repeat("q", 17), 17, false}, + {"cjk-16", strings.Repeat("环", 16), 16, true}, // 48 字节 + {"cjk-17", strings.Repeat("环", 17), 17, false}, + {"latin1-16", strings.Repeat("é", 16), 16, true}, // 32 字节 + {"emoji-5", strings.Repeat("😀", 5), 5, true}, // 20 字节 + {"mixed-16", "dev环境" + strings.Repeat("x", 11), 16, true}, + {"mixed-17", "dev环境" + strings.Repeat("x", 12), 17, false}, + {"padded-cjk-16", " " + strings.Repeat("产", 16) + "\t", 16, true}, // 先去首尾空白再计数 + } { + t.Run(tt.name, func(t *testing.T) { + _, err := Parse(tt.raw) + if !errors.Is(err, ErrUnknown) { + t.Fatalf("Parse(%q) 应返回 ErrUnknown，got %v", tt.raw, err) + } + trimmed := strings.TrimSpace(tt.raw) + if n := utf8.RuneCountInString(trimmed); n != tt.runes { + t.Fatalf("用例自检：字符数 %d，期望 %d", n, tt.runes) + } + msg := err.Error() + quoted := strconv.Quote(trimmed) + if tt.shown { + if !strings.Contains(msg, quoted) || strings.Contains(msg, "不回显") { + t.Fatalf("%d 个字符应原样回显，got %q", tt.runes, msg) + } + return + } + if strings.Contains(msg, trimmed) || strings.Contains(msg, quoted) { + t.Fatalf("%d 个字符不应回显原值，got %q", tt.runes, msg) + } + if want := fmt.Sprintf("（长度 %d，不回显）", tt.runes); !strings.Contains(msg, want) { + t.Fatalf("应按字符数报长度 %q，got %q", want, msg) + } + }) + } +} diff --git a/pkg/ssh/hostkey.go b/pkg/ssh/hostkey.go index f308dd08..6d15b459 100644 --- a/pkg/ssh/hostkey.go +++ b/pkg/ssh/hostkey.go @@ -4,18 +4,21 @@ import ( "fmt" "os" - "github.com/GoSimplicity/AI-CloudOps/pkg/runenv" + "github.com/GoSimplicity/AI-CloudOps/pkg/envmode" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" ) // HostKeyCallback uses a provisioned OpenSSH trust store, never trust-on-first-use. +// Without SSH_KNOWN_HOSTS, host key checking is skipped only when ENV explicitly +// resolves to development (see pkg/envmode). func HostKeyCallback() (ssh.HostKeyCallback, error) { if file := os.Getenv("SSH_KNOWN_HOSTS"); file != "" { return knownhosts.New(file) } - if runenv.IsProduction() { - return nil, fmt.Errorf("生产 SSH 连接必须配置 SSH_KNOWN_HOSTS") + // 失败即关闭：只有 ENV 明确为开发环境时才允许跳过主机密钥校验 + if envmode.IsProduction() { + return nil, fmt.Errorf("生产（或 ENV 未设置/无法识别）时 SSH 连接必须配置 SSH_KNOWN_HOSTS") } return ssh.InsecureIgnoreHostKey(), nil } diff --git a/pkg/ssh/hostkey_test.go b/pkg/ssh/hostkey_test.go index c2b7caad..211dbfec 100644 --- a/pkg/ssh/hostkey_test.go +++ b/pkg/ssh/hostkey_test.go @@ -1,8 +1,97 @@ package ssh -import "testing" +import ( + "crypto/ed25519" + "crypto/rand" + "os" + "path/filepath" + "testing" -// 生产判断与 runenv 一致：去首尾空白、忽略大小写 + cryptossh "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +func setEnvForHostKey(t *testing.T, raw string, set bool) { + t.Helper() + t.Setenv("ENV", raw) + if !set { + if err := os.Unsetenv("ENV"); err != nil { + t.Fatal(err) + } + } +} + +// 没有 SSH_KNOWN_HOSTS 时：只有明确的开发环境允许跳过主机密钥校验； +// 生产别名、未设置、空白、未知取值都必须报错（强制主机密钥校验）。 +func TestHostKeyCallbackWithoutKnownHosts(t *testing.T) { + t.Setenv("SSH_KNOWN_HOSTS", "") + for _, tt := range []struct { + raw string + set bool + wantErr bool + }{ + {"production", true, true}, {"prod", true, true}, {"PRD", true, true}, {" Prod\t", true, true}, + {"", false, true}, {"", true, true}, {" ", true, true}, + {"test", true, true}, {"testing", true, true}, {"staging", true, true}, + {"development", true, false}, {"dev", true, false}, {" DEV ", true, false}, + } { + setEnvForHostKey(t, tt.raw, tt.set) + cb, err := HostKeyCallback() + if (err != nil) != tt.wantErr || (cb == nil) != tt.wantErr { + t.Errorf("ENV=%q(set=%v): HostKeyCallback() err=%v cb!=nil=%v，期望报错=%v", tt.raw, tt.set, err, cb != nil, tt.wantErr) + } + } +} + +// 配了 SSH_KNOWN_HOSTS 时任何环境都用它校验：已知主机通过、未知主机拒绝。 +func TestHostKeyCallbackUsesKnownHosts(t *testing.T) { + pub, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + key, err := cryptossh.NewPublicKey(pub) + if err != nil { + t.Fatal(err) + } + otherPub, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + other, err := cryptossh.NewPublicKey(otherPub) + if err != nil { + t.Fatal(err) + } + file := filepath.Join(t.TempDir(), "known_hosts") + if err := os.WriteFile(file, []byte(knownhosts.Line([]string{"host.example.test:22"}, key)+"\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("SSH_KNOWN_HOSTS", file) + for _, raw := range []string{" PROD ", "dev"} { + setEnvForHostKey(t, raw, true) + cb, err := HostKeyCallback() + if err != nil || cb == nil { + t.Fatalf("ENV=%q: HostKeyCallback() = %v", raw, err) + } + addr := &fakeAddr{"203.0.113.7:22"} + if err := cb("host.example.test:22", addr, key); err != nil { + t.Errorf("ENV=%q: 已知主机被拒：%v", raw, err) + } + if err := cb("host.example.test:22", addr, other); err == nil { + t.Errorf("ENV=%q: 主机密钥不符却通过了", raw) + } + } + t.Setenv("SSH_KNOWN_HOSTS", filepath.Join(t.TempDir(), "missing")) + if _, err := HostKeyCallback(); err == nil { + t.Error("SSH_KNOWN_HOSTS 指向不存在的文件时应报错") + } +} + +type fakeAddr struct{ s string } + +func (a *fakeAddr) Network() string { return "tcp" } +func (a *fakeAddr) String() string { return a.s } + +// （来自 w1）生产别名的大小写/首尾空白变体都按生产处理（判定以 pkg/envmode 为准） func TestHostKeyCallbackRequiresKnownHostsInProductionVariants(t *testing.T) { t.Setenv("SSH_KNOWN_HOSTS", "") for _, env := range []string{"production", " Production ", "PRODUCTION"} { diff --git a/pkg/terminal/exec_close_test.go b/pkg/terminal/exec_close_test.go new file mode 100644 index 00000000..55b8be48 --- /dev/null +++ b/pkg/terminal/exec_close_test.go @@ -0,0 +1,392 @@ +package terminal + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/gorilla/websocket" + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/httpstream" + "k8s.io/apimachinery/pkg/util/httpstream/spdy" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// internalExecDetail 模拟 exec 失败时 API Server 返回的内部错误细节，不得出现在关闭帧 reason 中 +const internalExecDetail = "INTERNAL-DETAIL runtime exec failed: oci runtime error at /run/containerd/secret-path" + +// fakeKubeletExec 模拟 API Server 的 pods/exec SPDY 子协议（v4.channel.k8s.io）： +// 非 TTY 的探测命令直接成功退出；TTY 交互 shell 回显输入，收到 exit 后正常结束。 +// failTTY 为 true 时 TTY 交互 shell 立即以内部错误失败 +func fakeKubeletExec(t *testing.T, failTTY bool) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/exec") { + http.NotFound(w, r) + return + } + q := r.URL.Query() + tty, stdin, stdout, stderr := q.Get("tty") == "true", q.Get("stdin") == "true", q.Get("stdout") == "true", q.Get("stderr") == "true" + if _, err := httpstream.Handshake(r, w, []string{"v4.channel.k8s.io"}); err != nil { + return + } + streamCh := make(chan httpstream.Stream, 8) + conn := spdy.NewResponseUpgrader().UpgradeResponse(w, r, func(s httpstream.Stream, _ <-chan struct{}) error { + streamCh <- s + return nil + }) + if conn == nil { + return + } + defer conn.Close() + + expected := 1 // error + for _, on := range []bool{stdin, stdout, stderr && !tty, tty} { + if on { + expected++ + } + } + streams := map[string]httpstream.Stream{} + for len(streams) < expected { + select { + case s := <-streamCh: + streams[s.Headers().Get(v1.StreamType)] = s + case <-time.After(5 * time.Second): + return + } + } + if resize := streams[v1.StreamTypeResize]; resize != nil { + go func() { _, _ = io.Copy(io.Discard, resize) }() + } + if tty && failTTY { + _ = streams[v1.StreamTypeStdout].Close() + status := `{"metadata":{},"status":"Failure","message":"` + internalExecDetail + `","reason":"InternalError","code":500}` + _, _ = streams[v1.StreamTypeError].Write([]byte(status)) + _ = streams[v1.StreamTypeError].Close() + select { + case <-conn.CloseChan(): + case <-time.After(5 * time.Second): + } + return + } + if tty { + in, out := streams[v1.StreamTypeStdin], streams[v1.StreamTypeStdout] + var typed strings.Builder + buf := make([]byte, 256) + for !strings.Contains(typed.String(), "exit") { + n, err := in.Read(buf) + if n > 0 { + typed.Write(buf[:n]) + _, _ = out.Write(buf[:n]) + } + if err != nil { + return + } + } + _ = out.Close() + } else if out := streams[v1.StreamTypeStdout]; out != nil { + _ = out.Close() + } + errStream := streams[v1.StreamTypeError] + _, _ = errStream.Write([]byte(`{"metadata":{},"status":"Success"}`)) + _ = errStream.Close() + select { + case <-conn.CloseChan(): + case <-time.After(5 * time.Second): + } + })) +} + +func leakedTerminalGoroutines() []string { + buf := make([]byte, 1<<20) + buf = buf[:runtime.Stack(buf, true)] + var leaked []string + for _, g := range strings.Split(string(buf), "\n\n") { + if strings.Contains(g, "leakedTerminalGoroutines") { + continue // 正在执行检查的测试goroutine自身 + } + for _, marker := range []string{"AI-CloudOps/pkg/terminal.", "client-go/tools/remotecommand", "gorilla/websocket", "httpstream/spdy"} { + if strings.Contains(g, marker) { + leaked = append(leaked, g) + break + } + } + } + return leaked +} + +// runTerminalSession 通过真实的 HandleSession 跑一次终端会话：发送 resize 与 exit 输入， +// 读到关闭为止，并在全部服务关闭后检查 goroutine 回到基线 +func runTerminalSession(t *testing.T, failTTY bool) (string, *websocket.CloseError) { + t.Helper() + baseline := runtime.NumGoroutine() + + kubelet := fakeKubeletExec(t, failTTY) + cfg := &rest.Config{Host: kubelet.URL} + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + t.Fatal(err) + } + + handlerDone := make(chan struct{}) + wsServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer close(handlerDone) + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + return + } + defer conn.Close() // 与 internal/k8s/api/pod.go 的 PodExec 处理一致 + NewTerminalHandler(client, cfg, zap.NewNop()).HandleSession(r.Context(), "sh", "default", "pod", "app", conn) + })) + + peer, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(wsServer.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + if err := peer.WriteJSON(Message{Op: "resize", RowSize: 40, ColSize: 120}); err != nil { + t.Fatal(err) + } + if err := peer.WriteJSON(Message{Op: "stdin", Data: "exit\r"}); err != nil { + t.Fatal(err) + } + + _ = peer.SetReadDeadline(time.Now().Add(30 * time.Second)) + var output strings.Builder + var closeErr error + for { + var msg Message + if err := peer.ReadJSON(&msg); err != nil { + closeErr = err + break + } + output.WriteString(msg.Data) + } + + select { + case <-handlerDone: + case <-time.After(10 * time.Second): + t.Fatal("服务端终端处理未退出") + } + _ = peer.Close() + wsServer.Close() + kubelet.Close() + + deadline := time.Now().Add(5 * time.Second) + for { + leaked := leakedTerminalGoroutines() + if len(leaked) == 0 && runtime.NumGoroutine() <= baseline+2 { + break + } + if time.Now().After(deadline) { + t.Fatalf("存在 goroutine 泄漏 (baseline=%d now=%d):\n%s", baseline, runtime.NumGoroutine(), strings.Join(leaked, "\n\n")) + } + time.Sleep(50 * time.Millisecond) + } + + var ce *websocket.CloseError + if !asCloseError(closeErr, &ce) { + t.Fatalf("应收到 WebSocket 关闭帧, got %v", closeErr) + } + return output.String(), ce +} + +func TestExitSendsNormalClosureWithoutGoroutineLeak(t *testing.T) { + output, ce := runTerminalSession(t, false) + if !strings.Contains(output, "exit") { + t.Fatalf("未收到 shell 回显, output=%q", output) + } + if ce.Code != websocket.CloseNormalClosure { + t.Fatalf("exit 后应收到关闭码 1000, got %v", ce) + } +} + +func TestExecErrorSendsInternalErrorClosureWithoutGoroutineLeak(t *testing.T) { + _, ce := runTerminalSession(t, true) + if ce.Code != websocket.CloseInternalServerErr { + t.Fatalf("exec 出错后应收到关闭码 1011, got %v", ce) + } + if len(ce.Text) == 0 || len(ce.Text) > 123 { + t.Fatalf("关闭帧 reason 长度应在 1..123 字节, got %d (%q)", len(ce.Text), ce.Text) + } + if ce.Text != terminalErrorCloseReason { + t.Fatalf("关闭帧 reason 应为固定的用户可读说明, got %q", ce.Text) + } + for _, detail := range []string{"INTERNAL-DETAIL", "oci runtime", "/run/containerd", "所有Shell", "最后一个错误"} { + if strings.Contains(ce.Text, detail) { + t.Fatalf("关闭帧 reason 不应包含内部错误细节 %q: %q", detail, ce.Text) + } + } +} + +func TestTruncateCloseReasonRespectsByteLimitAndUTF8(t *testing.T) { + long := strings.Repeat("终", 60) // 180 字节 + got := truncateCloseReason(long) + if len(got) > maxCloseReasonBytes || !utf8.ValidString(got) { + t.Fatalf("截断结果非法: len=%d valid=%v", len(got), utf8.ValidString(got)) + } + if truncateCloseReason(terminalErrorCloseReason) != terminalErrorCloseReason { + t.Fatal("短 reason 不应被截断") + } + if len(websocket.FormatCloseMessage(websocket.CloseInternalServerErr, got)) > 125 { + t.Fatal("关闭帧载荷超过控制帧上限") + } +} + +func asCloseError(err error, target **websocket.CloseError) bool { + ce, ok := err.(*websocket.CloseError) + if ok { + *target = ce + } + return ok +} + +// CloseNormally/CloseWithError 在没有其他读者时自行等待对端回应，且有超时上限 +func TestCloseIsBoundedWhenPeerNeverReplies(t *testing.T) { + cases := []struct { + name string + code int + reason string + close func(*Session) error + }{ + {"normal", websocket.CloseNormalClosure, "", (*Session).CloseNormally}, + {"error", websocket.CloseInternalServerErr, terminalErrorCloseReason, (*Session).CloseWithError}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ready := make(chan *websocket.Conn, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err == nil { + ready <- conn + } + })) + defer srv.Close() + peer, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + defer peer.Close() + conn := <-ready + _, cancel := context.WithCancel(context.Background()) + s := &Session{conn: conn, logger: zap.NewNop(), cancel: cancel} + start := time.Now() + if err := tc.close(s); err != nil { + t.Fatal(err) + } + if elapsed := time.Since(start); elapsed > closeGracePeriod+500*time.Millisecond { + t.Fatalf("关闭等待过久: %v", elapsed) + } + // 对端不读也不回应，但关闭帧已经写出，可以被读到 + _ = peer.SetReadDeadline(time.Now().Add(time.Second)) + _, _, err = peer.ReadMessage() + var ce *websocket.CloseError + if !asCloseError(err, &ce) || ce.Code != tc.code || ce.Text != tc.reason { + t.Fatalf("应读到关闭码 %d reason %q, got %v", tc.code, tc.reason, err) + } + }) + } +} + +// 对端持续发送 Pong（并夹杂输入），但从不回应关闭帧：Pong 不得把读期限延长回 pongWait， +// 无论是关闭方自己读取，还是 stdin 复制 goroutine 正在读取，关闭都必须在上限内结束 +func TestCloseIsBoundedWhilePeerFloodsPongs(t *testing.T) { + // readerMode: 0=无并发reader（关闭方自己读）；1=stdin reader 持续读； + // 2=stdin reader 在关闭开始时停止读取（exec 结束后 stdin 复制随之停止的情形） + for readerMode, name := range []string{"close-reads-itself", "stdin-reader-active", "stdin-reader-stops-on-close"} { + readerMode := readerMode + t.Run(name, func(t *testing.T) { + ready := make(chan *websocket.Conn, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err == nil { + ready <- conn + } + })) + defer srv.Close() + peer, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(srv.URL, "http"), nil) + if err != nil { + t.Fatal(err) + } + defer peer.Close() + conn := <-ready + _, cancel := context.WithCancel(context.Background()) + s := &Session{conn: conn, logger: zap.NewNop(), cancel: cancel} + (&terminaler{logger: zap.NewNop()}).setupPongHandler(s) + + // 对端：只写不读（因此永远不会回应关闭帧），不停发送 Pong/Ping，并周期性发送输入让 reader 释放读锁 + stopPeer := make(chan struct{}) + peerDone := make(chan struct{}) + go func() { + defer close(peerDone) + stdin, _ := json.Marshal(Message{Op: "stdin", Data: "x"}) + for i := 0; ; i++ { + select { + case <-stopPeer: + return + default: + } + if err := peer.WriteControl(websocket.PongMessage, nil, time.Now().Add(time.Second)); err != nil { + return + } + if err := peer.WriteControl(websocket.PingMessage, nil, time.Now().Add(time.Second)); err != nil { + return + } + if i%16 == 0 { + if err := peer.WriteMessage(websocket.TextMessage, stdin); err != nil { + return + } + } + time.Sleep(100 * time.Microsecond) + } + }() + defer func() { close(stopPeer); <-peerDone }() + + readerDone := make(chan struct{}) + stopReading := make(chan struct{}) + if readerMode > 0 { + go func() { + defer close(readerDone) + buf := make([]byte, 64) + for { + if _, err := s.Read(buf); err != nil { + return + } + if readerMode == 2 { + select { + case <-stopReading: + return + default: + } + } + } + }() + } else { + close(readerDone) + } + + time.Sleep(200 * time.Millisecond) // 让 Pong 洪流先跑起来 + close(stopReading) + start := time.Now() + if err := s.CloseNormally(); err != nil { + t.Fatal(err) + } + if elapsed := time.Since(start); elapsed > closeGracePeriod+500*time.Millisecond { + t.Fatalf("对端持续发送 Pong 时关闭等待过久: %v", elapsed) + } + select { + case <-readerDone: + case <-time.After(2 * time.Second): + t.Fatal("stdin reader 未在关闭后退出") + } + }) + } +} diff --git a/pkg/terminal/terminal.go b/pkg/terminal/terminal.go index 1d99b06c..eecd1583 100644 --- a/pkg/terminal/terminal.go +++ b/pkg/terminal/terminal.go @@ -36,6 +36,7 @@ import ( "sync" "sync/atomic" "time" + "unicode/utf8" "github.com/gorilla/websocket" "go.uber.org/zap" @@ -52,6 +53,11 @@ const ( endOfTransmission = "\u0004" // 传输结束标志 pongWait = 30 * time.Second // Pong消息等待时间 pingPeriod = (pongWait * 9) / 10 // Ping发送间隔（必须小于pongWait） + closeGracePeriod = time.Second // 发送关闭帧后等待对端回应关闭帧的最长时间 + // maxCloseReasonBytes 关闭帧reason的字节上限（RFC 6455控制帧载荷125字节减去2字节关闭码） + maxCloseReasonBytes = 123 + // terminalErrorCloseReason exec异常结束时关闭帧携带的用户可读说明，不包含内部错误细节 + terminalErrorCloseReason = "终端会话异常结束" defaultTerminalRows = 25 // 默认终端行数 defaultTerminalCols = 80 // 默认终端列数 @@ -82,6 +88,12 @@ type Session struct { sizeMu sync.Mutex // resize发送与关闭 pending []byte // 尚未交付给调用者的stdin字节 cancel context.CancelFunc // 断开输入时取消所属exec上下文 + + closing int32 // 已发送关闭帧，正在等待对端回应 + deadlineMu sync.Mutex // 串行化closing标记与读期限写入，避免Pong在关闭后把读期限延长回pongWait + peerInit sync.Once // 惰性创建peerClosed + peerOnce sync.Once // 只关闭一次peerClosed + peerClosed chan struct{} // 输入读取结束（对端关闭或连接断开） } /* @@ -151,6 +163,98 @@ func (t *Session) Write(p []byte) (int, error) { return len(p), nil } +func (t *Session) peerDone() chan struct{} { + t.peerInit.Do(func() { t.peerClosed = make(chan struct{}) }) + return t.peerClosed +} + +func (t *Session) markPeerDone() { + done := t.peerDone() + t.peerOnce.Do(func() { close(done) }) +} + +// CloseNormally 在exec正常结束时使用：先发送1000正常关闭帧， +// 在closeGracePeriod内等待对端回应关闭帧后再断开连接，避免客户端只看到1006 +func (t *Session) CloseNormally() error { + return t.closeWithCode(websocket.CloseNormalClosure, "") +} + +// CloseWithError 在exec异常结束时使用：先发送1011关闭帧（reason为固定的用户可读说明，不含内部错误细节）， +// 与CloseNormally相同地有限等待对端回应后再断开连接 +func (t *Session) CloseWithError() error { + return t.closeWithCode(websocket.CloseInternalServerErr, terminalErrorCloseReason) +} + +// closeWithCode 发送指定关闭码的关闭帧，写超时与等待均以closeGracePeriod为上限，最后断开连接 +func (t *Session) closeWithCode(code int, reason string) error { + if atomic.LoadInt32(&t.closed) == 1 { + return nil + } + if t.conn != nil { + deadline := time.Now().Add(closeGracePeriod) + t.beginClosing(deadline) + // Gorilla的WriteControl可与其他写并发调用；不持writeMu，避免被卡住的数据帧写阻塞关闭 + message := websocket.FormatCloseMessage(code, truncateCloseReason(reason)) + if err := t.conn.WriteControl(websocket.CloseMessage, message, deadline); err == nil { + t.awaitPeerClose(deadline) + } else { + t.logger.Debug("发送WebSocket关闭帧失败", zap.Error(err)) + } + } + return t.Close() +} + +// truncateCloseReason 保证关闭帧reason不超过协议上限（125字节控制帧载荷减去2字节关闭码），且不截断UTF-8字符 +func truncateCloseReason(reason string) string { + if len(reason) <= maxCloseReasonBytes { + return reason + } + cut := maxCloseReasonBytes + for cut > 0 && !utf8.RuneStart(reason[cut]) { + cut-- + } + return reason[:cut] +} + +// beginClosing 在同一把锁下标记closing并收紧读期限，此后Pong不能再延长读期限 +func (t *Session) beginClosing(deadline time.Time) { + t.deadlineMu.Lock() + defer t.deadlineMu.Unlock() + atomic.StoreInt32(&t.closing, 1) + _ = t.conn.SetReadDeadline(deadline) +} + +// extendReadDeadlineOnPong 收到Pong时延长读期限；已开始关闭时不延长。检查与写入在同一把锁下完成 +func (t *Session) extendReadDeadlineOnPong() { + t.deadlineMu.Lock() + defer t.deadlineMu.Unlock() + if atomic.LoadInt32(&t.closed) == 1 || atomic.LoadInt32(&t.closing) == 1 { + return + } + _ = t.conn.SetReadDeadline(time.Now().Add(pongWait)) +} + +// awaitPeerClose 等待对端回应关闭帧。若无其他goroutine在读，则自行读取直至收到关闭帧或超时； +// 若stdin复制goroutine仍阻塞在Read中，由它读到关闭帧后通知，超时则直接断开 +func (t *Session) awaitPeerClose(deadline time.Time) { + if t.readMu.TryLock() { + defer t.readMu.Unlock() + // 拿到读锁后再收紧一次读期限：之前持锁的reader可能在关闭开始前刚处理过Pong + _ = t.conn.SetReadDeadline(deadline) + for { + if _, _, err := t.conn.NextReader(); err != nil { + return + } + } + } + timer := time.NewTimer(time.Until(deadline)) + defer timer.Stop() + select { + case <-t.peerDone(): + case <-timer.C: + } +} + func (t *Session) Close() error { if !atomic.CompareAndSwapInt32(&t.closed, 0, 1) { return nil @@ -223,6 +327,7 @@ func (t *Session) Read(p []byte) (int, error) { return t.rejectInput() } if err != nil { + t.markPeerDone() if t.cancel != nil { t.cancel() } @@ -382,11 +487,8 @@ func (t *terminaler) setupPongHandler(session *Session) { } t.logger.Debug("接收到Pong消息") - if atomic.LoadInt32(&session.closed) == 1 { - return nil - } - - session.conn.SetReadDeadline(time.Now().Add(pongWait)) // nolint + // 已发送关闭帧时不再延长读期限，保证关闭等待有上限 + session.extendReadDeadlineOnPong() return nil }) } @@ -425,7 +527,12 @@ func (t *terminaler) handleTerminalSession(ctx context.Context, shell, namespace // 尝试执行终端命令，使用fallback机制 err := t.executeTerminalCommandWithFallback(ctx, namespace, podName, containerName, fallbackShells, session) - if err != nil && !errors.Is(err, context.Canceled) { + if err == nil { + // exec正常结束（如用户输入exit）：先发1000正常关闭帧再断开，客户端不再收到1006 + if closeErr := session.CloseNormally(); closeErr != nil { + t.logger.Error("关闭终端会话失败", zap.Error(closeErr)) + } + } else if !errors.Is(err, context.Canceled) { // 检查连接状态，避免向已关闭连接发送消息 if atomic.LoadInt32(&session.closed) == 0 { // 格式化用户友好的错误消息 @@ -435,6 +542,10 @@ func (t *terminaler) handleTerminalSession(ctx context.Context, shell, namespace if writeErr := t.writeErrorMessage(session, errorMsg); writeErr != nil { t.logger.Error("发送错误消息失败", zap.Error(writeErr)) } + // 再发1011关闭帧后断开，客户端可区分异常结束与正常exit（1000），而不是只看到1006 + if closeErr := session.CloseWithError(); closeErr != nil { + t.logger.Error("关闭终端会话失败", zap.Error(closeErr)) + } } else { t.logger.Debug("终端会话执行失败，但连接已关闭，跳过错误消息发送", zap.Error(err)) } diff --git a/scripts/check-webhook-env.sh b/scripts/check-webhook-env.sh new file mode 100755 index 00000000..38f46fd9 --- /dev/null +++ b/scripts/check-webhook-env.sh @@ -0,0 +1,227 @@ +#!/bin/sh +# 只读检查：webhook（cmd/webhook）升级到 sched/w1-mantou（0146077 起）前，配置文件 / 环境变量里的 webhook 字段是否合规。 +# 只输出字段名和“合规 / 不合规 / 告警”及原因类别，从不输出任何取值；密钥字段（mysql.addr、WEBHOOK_MYSQL_ADDR、 +# app_id / app_secret 等）的值在 awk 里直接丢弃、不进入 shell 变量。 +# +# 用法: scripts/check-webhook-env.sh [--config ] [--env-file <.env>] +# --config 要检查的 webhook 配置文件（与 cmd/webhook --config 相同）；不给则只检查环境变量 + 内置默认值。 +# --env-file 从 env 文件读取 WEBHOOK_* 的 5 个非密钥变量（不 source、不执行）；不给则读当前进程环境变量。 +# 取值优先级与 loader 相同：环境变量（空串视为未设置）> 配置文件 webhook.* > 内置默认值。 +# 检查项（仅检查下列字段与块状 YAML 写法；不能替代 Go loader 的完整校验）： +# 根节点 根节点下出现 webhook 字段（平铺，含大写 PORT）→ 不合规（loader 拒绝启动）； +# 没有 webhook 根节点或其值为空 → 告警（loader 只告警、webhook.* 全取环境变量/默认值） +# webhook.port 不带符号、不带前导零的十进制整数，1-65535（loader 会把 0080 当 80，这里算不合规） +# fixed_workers / default_upgrade_minutes / common_map_renew_interval_seconds 十进制整数：fixed_workers 1-1024、default_upgrade_minutes 1-10080、common_map_renew_interval_seconds 1-86400 +# im_feishu.request_timeout_seconds 十进制整数 1-600（0 和负数均拒绝） +# 同一层重复的键（port 和 PORT 同写，或任意仅大小写不同 / 完全相同的同名键，如 front_domain 与 FRONT_DOMAIN、 +# im_feishu 下的 app_id 与 APP_ID；loader 键名不区分大小写，重复键拒绝启动）→ 不合规（review3g） +# 写法（review3h） 内联表/数组、锚点/别名/<<、转义键、? 复杂键、制表符缩进 → 不合规（请人工检查）；多文档 → 不合规（loader 拒绝启动） +# 退出码: 0 = 无不合规（可能有告警），1 = 有不合规，2 = 用法错误或文件不可读。 +# 兼容 POSIX sh / bash --posix / macOS（bash 3.2、BSD awk）。 +set -u +CFG=""; ENVF="" +while [ $# -gt 0 ]; do + case "$1" in + --config) [ $# -ge 2 ] || { echo "--config 需要参数" >&2; exit 2; }; CFG="$2"; shift 2 ;; + --env-file) [ $# -ge 2 ] || { echo "--env-file 需要参数" >&2; exit 2; }; ENVF="$2"; shift 2 ;; + -h|--help) sed -n '2,20p' "$0"; exit 0 ;; + *) echo "未知参数: $1" >&2; exit 2 ;; + esac +done +[ -z "$CFG" ] || [ -r "$CFG" ] || { echo "配置文件不可读: --config 指定的文件" >&2; exit 2; } +[ -z "$ENVF" ] || [ -r "$ENVF" ] || { echo "env 文件不可读: --env-file 指定的文件" >&2; exit 2; } + +NG=0; WARN=0 +say() { printf '%s: %s\n' "$1" "$2"; } + +# ---------- 环境变量来源 ---------- +# envval <变量名>：只允许下面 5 个非密钥变量 +envval() { + case "$1" in + WEBHOOK_PORT|WEBHOOK_FIXED_WORKERS|WEBHOOK_DEFAULT_UPGRADE_MINUTES|WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS|WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS) ;; + *) return 1 ;; + esac + if [ -n "$ENVF" ]; then + # 只取 KEY=... 精确匹配的最后一行；去掉 CR、前缀 export、成对的外层引号 + K="$1" awk '{ sub(/\r$/, ""); sub(/^[ \t]*export[ \t]+/, "") } + index($0, ENVIRON["K"] "=") == 1 { v = substr($0, length(ENVIRON["K"]) + 2); got = 1 } + END { if (got) { if (v ~ /^".*"$/ || v ~ /^\047.*\047$/) v = substr(v, 2, length(v) - 2); printf "%s", v } }' "$ENVF" + else + eval "printf '%s' \"\${$1:-}\"" + fi +} + +# ---------- 配置文件解析（只认本仓库 webhook.yaml 的块状写法）---------- +# 输出（制表符分隔）：TOP <小写键> | ROOT | WH <小写键> [<值>] | IM <小写键> [<值>] | DUP <键路径> +# | UNSUP <类别> <行号> | MULTI <行号> +# 只有 5 个检查字段输出值，其余键（含 mysql.addr、app_secret）只输出键名；UNSUP / MULTI 只输出类别和行号。 +# review3h P3-d： +# - 先去注释再分类：引号外、行首或前面是空格/制表符的 # 起到行尾都是注释（webhook:# 注释 不再被当成单行值）； +# - 多文档：第一个文档之后的 ---（或 ...）起不再解析，输出 MULTI 由调用方判不合规； +# - 块标量（| / > 及其修饰符）的内容行不当键解析（里面写 port: / webhook: 不误报）； +# - 本脚本不支持、可能让结论失真的写法输出 UNSUP，调用方判“不合规/请人工检查”：内联表/数组（{...} / [...]，空 {} [] 除外）、 +# 锚点/别名（&x / *x）、合并键 <<、带转义的引号键（"po\x72t"）、复杂键 ?、制表符缩进。 +parse_cfg() { + awk ' + BEGIN { li = -1; sd = 0; bs = -1; started = 0 } + function trim(s) { sub(/^[ \t]+/, "", s); sub(/[ \t]+$/, "", s); return s } + # 去注释：引号只在标量开头（行首、或前一个非空白字符是 : - ? , [ {）才算开始；双引号内反斜杠转义下一个字符，单引号内两个连续单引号表示一个单引号 + function stripc(s, i, n, ch, q, p) { + q = ""; p = ""; n = length(s) + for (i = 1; i <= n; i++) { + ch = substr(s, i, 1) + if (q != "") { + if (q == "\"" && ch == "\\") { i++; continue } + if (ch == q) { + if (q == "\047" && substr(s, i + 1, 1) == "\047") { i++; continue } + q = ""; p = ch + } + continue + } + if ((ch == "\"" || ch == "\047") && (p == "" || p == ":" || p == "-" || p == "?" || p == "," || p == "[" || p == "{")) { q = ch; continue } + if (ch == "#" && (i == 1 || substr(s, i - 1, 1) == " " || substr(s, i - 1, 1) == "\t")) return substr(s, 1, i - 1) + if (ch != " " && ch != "\t") p = ch + } + return s + } + function val(s, q, i) { # 去外层引号；带引号时保留引号内原样（含空格） + s = trim(s) + q = substr(s, 1, 1) + if (q == "\"" || q == "\047") { i = index(substr(s, 2), q); if (i > 0) return substr(s, 2, i - 1); return substr(s, 2) } + sub(/[ \t]+#.*$/, "", s); return trim(s) + } + function want(k) { return k == "port" || k == "fixed_workers" || k == "default_upgrade_minutes" || k == "common_map_renew_interval_seconds" || k == "request_timeout_seconds" } + function unsup(c) { print "UNSUP\t" c "\t" NR } + { + sub(/\r$/, "") + if ($0 ~ /^(---|\.\.\.)([ \t]|$)/) { if (started) { print "MULTI\t" NR; exit } ; next } + if (bs >= 0) { # 块标量内容：空行或缩进大于引出它的那一行 + if ($0 ~ /^[ \t]*$/) next + match($0, /^ */); if (RLENGTH > bs) next + bs = -1 + } + $0 = stripc($0) + if ($0 ~ /^[ \t]*$/) next + started = 1 + if ($0 ~ /^ *\t/) { unsup("制表符缩进"); next } + match($0, /^ */); ind = RLENGTH + line = substr($0, ind + 1) + if (line ~ /^\?([ \t]|$)/ || line ~ /^-[ \t]+\?([ \t]|$)/) { unsup("复杂键 ?"); next } + if (line ~ /^-[ \t]+[{[]/) unsup("内联表/数组") + c = index(line, ":"); if (c == 0) next + rk = trim(substr(line, 1, c - 1)); sub(/^-[ \t]+/, "", rk) + if (rk ~ /^["\047]/ && index(rk, "\\") > 0) unsup("转义键") + k = tolower(trim(substr(line, 1, c - 1))); gsub(/^["\047]|["\047]$/, "", k) + rest = substr(line, c + 1) + r = trim(rest) + if (rk == "<<" || rk ~ /^-[ \t]+<<$/) unsup("合并键 <<") + if (r ~ /^[&*]/) unsup("锚点/别名") + if (r ~ /^[{[]/ && r != "{}" && r != "[]" && !(ind == 0 && k == "webhook")) unsup("内联表/数组") + if (r ~ /^[|>][-+0-9]*$/) bs = ind + # 重复键（review3g）：按缩进维护键路径，同一父节点下小写后相同的键算重复；列表（- 开头）里的键不参与 + if (li >= 0 && ind <= li && line !~ /^-/) li = -1 + if (line ~ /^-/) { if (li < 0 || ind < li) li = ind } + else if (li < 0) { + while (sd > 0 && sind[sd] >= ind) sd-- + full = (sd > 0 ? skey[sd] "." k : k) + if (++seen[full] == 2) print "DUP\t" full + sd++; sind[sd] = ind; skey[sd] = full + } + if (ind == 0) { + print "TOP\t" k + inwh = 0; inim = 0 + if (k == "webhook") { + v = val(rest) + if (v == "" || v == "~" || v == "null" || v == "{}") { inwh = 1; wi = -1; print "ROOT\tempty" } + else print "ROOT\tinline" + } + next + } + if (!inwh) next + if (wi < 0) { wi = ind; print "ROOT\tblock" } + if (ind == wi) { + inim = (k == "im_feishu"); ii = -1 + if (want(k) && k != "request_timeout_seconds") print "WH\t" k "\t" val(rest); else print "WH\t" k + next + } + if (ind > wi && inim) { + if (ii < 0) ii = ind + if (ind != ii) next + if (k == "request_timeout_seconds") print "IM\t" k "\t" val(rest); else print "IM\t" k + } + }' "$1" +} + +TAB="$(printf '\t')" +# 临时文件（review3h P3-e）：用 mktemp 创建（不用可预测的 $$ 文件名），创建失败或不是普通空文件就退出 2，不继续出结论 +P="" +trap '[ -z "$P" ] || rm -f "$P"' EXIT +trap '[ -z "$P" ] || rm -f "$P"; exit 2' HUP INT TERM +P="$(mktemp "${TMPDIR:-/tmp}/check-webhook-env.XXXXXX" 2>/dev/null)" || P="" +if [ -z "$P" ] || [ ! -f "$P" ] || [ -L "$P" ] || [ -s "$P" ]; then + echo "无法创建临时文件（检查 TMPDIR 是否存在且可写）" >&2; P=""; exit 2 +fi +if [ -n "$CFG" ]; then parse_cfg "$CFG" >"$P" || { echo "配置文件解析失败" >&2; exit 2; }; fi + +# ---------- 根节点 ---------- +if [ -n "$CFG" ]; then + flat="" + for k in port fixed_workers front_domain backend_domain default_upgrade_minutes alert_manager_api common_map_renew_interval_seconds im_feishu; do + if grep -q "^TOP${TAB}${k}\$" "$P"; then flat="${flat}${flat:+, }${k}"; fi + done + nwh="$(grep -c "^TOP${TAB}webhook\$" "$P")" + if [ -n "$flat" ]; then say "配置文件根节点" "不合规（根节点下有 webhook 字段 ${flat}，loader 拒绝启动；移到 webhook: 下）"; NG=$((NG+1)) + elif [ "$nwh" -gt 1 ]; then say "配置文件根节点" "不合规（webhook 根节点重复）"; NG=$((NG+1)) + elif grep -q "^ROOT${TAB}inline\$" "$P"; then say "配置文件根节点" "不合规（webhook: 写成了单行值，本脚本只支持块状写法，请人工检查）"; NG=$((NG+1)) + elif grep -q "^ROOT${TAB}block\$" "$P"; then say "配置文件根节点" "合规" + else say "配置文件根节点" "告警（没有 webhook 根节点或其值为空：loader 只告警，webhook.* 全部取环境变量/默认值）"; WARN=$((WARN+1)); fi +else + say "配置文件根节点" "未检查（没有 --config）" +fi +if [ -n "$CFG" ]; then + # 不支持的写法（review3h P3-d）：只报类别和行号，不报取值 + uns="$(grep "^UNSUP${TAB}" "$P" | awk -F "$TAB" '{ printf "%s%s 第 %s 行", (NR > 1 ? ", " : ""), $2, $3 }')" + if [ -n "$uns" ]; then say "配置文件写法" "不合规（含本脚本不支持的 YAML 写法：${uns}；请人工检查）"; NG=$((NG+1)) + elif grep -q "^MULTI${TAB}" "$P"; then + say "配置文件写法" "不合规（多文档：第 $(grep "^MULTI${TAB}" "$P" | cut -f2) 行起有后续文档，loader 拒绝启动）"; NG=$((NG+1)) + else say "配置文件写法" "合规"; fi + dups="$(grep "^DUP${TAB}" "$P" | cut -f2 | tr '\n' ' ' | sed 's/ $//; s/ /, /g')" + if [ -n "$dups" ]; then say "配置文件重复键" "不合规（同一层有仅大小写不同或完全相同的键：${dups}；loader 键名不区分大小写，重复键拒绝启动）"; NG=$((NG+1)) + else say "配置文件重复键" "合规"; fi +fi + +# ---------- 字段 ---------- +# check <显示名> <环境变量名> <解析行前缀 WH|IM> <下限> <上限|-> <默认值> +check() { + name="$1"; ev="$2"; kind="$3"; yk="$4"; lo="$5"; hi="$6"; def="$7" + n="$(grep -E -c "^${kind}${TAB}${yk}(${TAB}|\$)" "$P")" + if [ "$n" -gt 1 ]; then say "$name" "不合规（配置文件同一层重复写了 ${yk}，键名不区分大小写，重复键拒绝启动）"; NG=$((NG+1)); return; fi + v="$(envval "$ev")"; src="环境变量 ${ev}" + if [ -z "$v" ]; then + if [ "$n" -eq 1 ]; then + line="$(grep -E "^${kind}${TAB}${yk}(${TAB}|\$)" "$P")" + v="${line#"${kind}${TAB}${yk}"}"; v="${v#"${TAB}"}"; src="配置文件" + if [ -z "$v" ]; then say "$name" "不合规（配置文件里是空值，删掉该键或写整数）"; NG=$((NG+1)); return; fi + else + v="$def"; src="内置默认值" + fi + fi + case "$v" in + 0) ;; + *[!0-9]*) say "$name" "不合规（${src}：不是十进制整数，含符号、小数点、空格、字母或进制前缀）"; NG=$((NG+1)); return ;; + 0*) say "$name" "不合规（${src}：有前导零）"; NG=$((NG+1)); return ;; + esac + if [ "${#v}" -gt 9 ]; then say "$name" "不合规（${src}：数值过大）"; NG=$((NG+1)); return; fi + if [ "$v" -lt "$lo" ]; then say "$name" "不合规（${src}：小于 ${lo}）"; NG=$((NG+1)); return; fi + if [ "$hi" != "-" ] && [ "$v" -gt "$hi" ]; then say "$name" "不合规（${src}：大于 ${hi}）"; NG=$((NG+1)); return; fi + say "$name" "合规（${src}）" +} +check webhook.port WEBHOOK_PORT WH port 1 65535 8889 +check webhook.fixed_workers WEBHOOK_FIXED_WORKERS WH fixed_workers 1 1024 10 +check webhook.default_upgrade_minutes WEBHOOK_DEFAULT_UPGRADE_MINUTES WH default_upgrade_minutes 1 10080 60 +check webhook.common_map_renew_interval_seconds WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS WH common_map_renew_interval_seconds 1 86400 300 +check webhook.im_feishu.request_timeout_seconds WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS IM request_timeout_seconds 1 600 10 + +if [ "$NG" -gt 0 ]; then echo "结果: 不合规 ${NG} 项，告警 ${WARN} 项"; exit 1; fi +echo "结果: 合规（告警 ${WARN} 项）" +exit 0 diff --git a/scripts/release-env.py b/scripts/release-env.py index 64efef0d..d54b106a 100644 --- a/scripts/release-env.py +++ b/scripts/release-env.py @@ -1,13 +1,91 @@ #!/usr/bin/env python3 """Create local release secrets without printing them or overwriting an existing file.""" +import argparse import os from pathlib import Path +import re import secrets +import sys + +# 与 deploy/release/compose.yaml 中 mysql 服务的 MYSQL_USER / MYSQL_DATABASE 及服务名保持一致 +MYSQL_USER = "cloudops" +MYSQL_DATABASE = "cloudops" +DEFAULT_MYSQL_HOST = "mysql:3306" +# webhook（cmd/webhook）经 gorm.io/driver/mysql 解析 go-sql-driver DSN，参数与 backend 的 MYSQL_ADDR 相同 +DSN_PARAMS = "charset=utf8mb4&parseTime=True&loc=Local" +# 口令直接拼进 DSN 且 .env 不加引号：只接受不含 DSN 分隔符和 Compose 插值字符的值 +SAFE_VALUE = re.compile(r"^[A-Za-z0-9._~-]+$") +SAFE_HOST = re.compile(r"^[A-Za-z0-9.-]+:[0-9]{1,5}$") + + +def webhook_dsn(password, host): + """Build WEBHOOK_MYSQL_ADDR; never print or log the result.""" + if not SAFE_VALUE.match(password): + raise ValueError("MYSQL_PASSWORD contains characters unsafe for an unquoted DSN") + return f"{MYSQL_USER}:{password}@tcp({host})/{MYSQL_DATABASE}?{DSN_PARAMS}" + + +def read_settings(path): + settings = {} + for line in path.read_text().splitlines(): + if line and not line.lstrip().startswith("#") and "=" in line: + name, value = line.split("=", 1) + settings[name.strip()] = value.strip() + return settings + + +def add_missing(destination, host): + """Upgrade an existing .env: only append derived keys that are missing or empty.""" + settings = read_settings(destination) + current = settings.get("WEBHOOK_MYSQL_ADDR", "") + if current: + print(f"{destination} already has WEBHOOK_MYSQL_ADDR; nothing changed. No secrets printed.") + return 0 + password = settings.get("MYSQL_PASSWORD", "") + if not password: + print("MYSQL_PASSWORD is missing or empty; cannot derive WEBHOOK_MYSQL_ADDR.", file=sys.stderr) + return 1 + if "WEBHOOK_MYSQL_ADDR" in settings: + # 存在空占位行时不改写原行（避免重写整个文件），提示人工删除空行后重试 + print("WEBHOOK_MYSQL_ADDR= exists but is empty; remove that line and rerun with --add-missing.", file=sys.stderr) + return 1 + try: + line = f"WEBHOOK_MYSQL_ADDR={webhook_dsn(password, host)}\n" + except ValueError as error: + print(f"{error}; set WEBHOOK_MYSQL_ADDR manually.", file=sys.stderr) + return 1 + raw = destination.read_bytes() + needs_newline = bool(raw) and not raw.endswith(b"\n") + descriptor = os.open(destination, os.O_WRONLY | os.O_APPEND) + with os.fdopen(descriptor, "w") as output: + output.write(("\n" if needs_newline else "") + "# Webhook (cmd/webhook) MySQL DSN, derived from MYSQL_PASSWORD.\n" + line) + os.chmod(destination, 0o600) + print(f"Appended WEBHOOK_MYSQL_ADDR to {destination} (mode 0600). No secrets printed.") + return 0 def main(): - destination = Path(__file__).resolve().parents[1] / "deploy/release/.env" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--env-file", type=Path, + default=Path(__file__).resolve().parents[1] / "deploy/release/.env", + help="destination .env (default: deploy/release/.env)") + parser.add_argument("--mysql-host", default=DEFAULT_MYSQL_HOST, + help="host:port used in WEBHOOK_MYSQL_ADDR (default: mysql:3306, the Compose service)") + parser.add_argument("--add-missing", action="store_true", + help="for an existing .env, append missing derived keys (WEBHOOK_MYSQL_ADDR) without touching other values") + args = parser.parse_args() + destination = args.env_file + if not SAFE_HOST.match(args.mysql_host): + print("--mysql-host must look like host:port", file=sys.stderr) + return 2 + + if args.add_missing: + if not destination.exists(): + print(f"{destination} does not exist; run without --add-missing first.", file=sys.stderr) + return 1 + return add_missing(destination, args.mysql_host) + names = ( "MYSQL_PASSWORD", "MYSQL_ROOT_PASSWORD", "REDIS_PASSWORD", "JWT_KEY1", "JWT_KEY2", "TREE_PASSWORD_ENCRYPTION_KEY", "BOOTSTRAP_ADMIN_PASSWORD", @@ -17,17 +95,25 @@ def main(): values = {name: secrets.token_hex(24) for name in names} # Existing AES encryption requires exactly 32 bytes; preserve this key on upgrades. values["TREE_PASSWORD_ENCRYPTION_KEY"] = secrets.token_hex(16) + # Webhook reads only WEBHOOK_MYSQL_ADDR (never the backend MYSQL_ADDR); same account and database. + values["WEBHOOK_MYSQL_ADDR"] = webhook_dsn(values["MYSQL_PASSWORD"], args.mysql_host) content = "# Keep private. Back up encryption keys with the database.\n" content += "".join(f"{name}={value}\n" for name, value in values.items()) content += "PUBLIC_ORIGIN=http://localhost:18080\nHTTP_PORT=18080\n" content += "BIND_ADDRESS=127.0.0.1\nRELEASE_VERSION=local\n" content += "# Optional AIOps custom bot; keep the webhook credential private.\n" content += "NOTIFICATION_ENABLED=true\nFEISHU_WEBHOOK_URL=\n" - descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + print(f"{destination} already exists; refusing to overwrite. " + "Use --add-missing to append only missing derived keys.", file=sys.stderr) + return 1 with os.fdopen(descriptor, "w") as output: output.write(content) print(f"Created {destination} with mode 0600. No secrets printed.") + return 0 if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/setup-git-hooks.sh b/scripts/setup-git-hooks.sh index 405c9a3e..abef355b 100755 --- a/scripts/setup-git-hooks.sh +++ b/scripts/setup-git-hooks.sh @@ -1,62 +1,79 @@ -#!/bin/bash +#!/usr/bin/env bash +# 配置本仓库使用 .githooks/ 下的 Git 钩子，并检查 gitleaks 是否已安装。 +# 用法: scripts/setup-git-hooks.sh +set -euo pipefail -set -e +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } +log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; } -# 颜色定义 -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# 项目根目录 PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$PROJECT_ROOT" -# 日志函数 -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} +if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + log_error "当前目录不是 Git 仓库: $PROJECT_ROOT" + exit 1 +fi -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} +# 只给钩子脚本加可执行位（不改动 README 等文档的文件模式） +find .githooks -maxdepth 1 -type f ! -name "*.md" -exec chmod +x {} + 2>/dev/null || true -# 检查 git 仓库 -check_git_repo() { - if [ ! -d ".git" ]; then - log_error "当前目录不是 Git 仓库" +# 在 linked worktree 中写入 worktree 级配置，避免修改主仓库/其他 worktree 共享的 .git/config。 +git_dir="$(cd "$(git rev-parse --git-dir)" && pwd)" +common_dir="$(cd "$(git rev-parse --git-common-dir)" && pwd)" +if [ "$git_dir" != "$common_dir" ]; then + if [ "$(git config --type=bool --get extensions.worktreeConfig 2>/dev/null || echo false)" != "true" ]; then + log_error "当前是 linked worktree，且未启用 extensions.worktreeConfig。" + log_error "为避免改动共享配置，请先执行: git config extensions.worktreeConfig true 后重试。" exit 1 fi -} + git config --worktree core.hooksPath .githooks + log_info "已设置 (worktree 级) core.hooksPath=.githooks" +else + git config core.hooksPath .githooks + log_info "已设置 core.hooksPath=.githooks" +fi -# 显示设置结果 -show_setup_result() { - echo "" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "🎉 Git Hooks 设置完成！" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "" - echo "你可以现在开始愉快地开发啦 🚀" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +# gitleaks 最低版本 8.25.0（review3g N6）：.gitleaks.toml 使用多个 [[allowlists]]（含 targetRules/condition）， +# 该写法自 8.25.0 起才支持；更低版本的策略语义不同，且 8.19 之前没有 `gitleaks git --staged`。 +# 只接受 gitleaks version 输出的严格形式 X.Y.Z（可带前缀 v；每段为不带前导零、最多 6 位的十进制数字），无法解析一律视为不满足。 +GITLEAKS_MIN_MAJOR=8 +GITLEAKS_MIN_MINOR=25 +gitleaks_version_check() { + gl_ver_raw="$(gitleaks version 2>/dev/null | head -n 1)" + gl_ver="${gl_ver_raw#v}" + case "$gl_ver" in *.*.*) ;; *) return 2 ;; esac + gl_major="${gl_ver%%.*}"; gl_rest="${gl_ver#*.}" + gl_minor="${gl_rest%%.*}"; gl_patch="${gl_rest#*.}" + # review3h P3-i：三段都只能是十进制数字，不允许前导零（单独的 0 除外），每段最多 6 位； + # 过长的数字直接视为无法解析，不交给 [ ] 做整数比较（溢出时 [ ] 报错，会被误判成“版本过低”） + for gl_c in "$gl_major" "$gl_minor" "$gl_patch"; do + case "$gl_c" in ''|*[!0-9]*|0?*|???????*) return 2 ;; esac + done + if [ "$gl_major" -gt "$GITLEAKS_MIN_MAJOR" ]; then return 0; fi + if [ "$gl_major" -eq "$GITLEAKS_MIN_MAJOR" ] && [ "$gl_minor" -ge "$GITLEAKS_MIN_MINOR" ]; then return 0; fi + return 1 } +if command -v gitleaks >/dev/null 2>&1; then + gl_vrc=0; gitleaks_version_check || gl_vrc=$? + if [ "$gl_vrc" -eq 2 ]; then + log_error "无法解析 gitleaks 版本（gitleaks version 输出: ${gl_ver_raw:-<空>}），需要 >= ${GITLEAKS_MIN_MAJOR}.${GITLEAKS_MIN_MINOR}.0；pre-commit 钩子会因此阻止提交。" + exit 1 + elif [ "$gl_vrc" -ne 0 ]; then + log_error "gitleaks 版本 ${gl_ver_raw} 低于 ${GITLEAKS_MIN_MAJOR}.${GITLEAKS_MIN_MINOR}.0；pre-commit 钩子会因此阻止提交。请从 https://github.com/gitleaks/gitleaks/releases 升级。" + exit 1 + fi + log_success "gitleaks 已安装: ${gl_ver_raw}（>= ${GITLEAKS_MIN_MAJOR}.${GITLEAKS_MIN_MINOR}.0）" +else + log_error "未找到 gitleaks。pre-commit 钩子会因此阻止提交。" + log_error "请从 https://github.com/gitleaks/gitleaks/releases 下载官方二进制，校验 sha256 后放入 PATH（如 ~/.local/bin）。" + exit 1 +fi -# 主函数 -main() { - log_info "🚀 设置 AI-CloudOps Git Hooks..." - - check_git_repo - - show_setup_result -} +if [ -f package.json ] && grep -q '"husky"' package.json; then + log_warning "本仓库依赖 husky：执行 pnpm install 时 husky 可能把 core.hooksPath 改回 .husky/_，之后请重新运行本脚本。" +fi -# 执行主函数 -main "$@" +log_success "Git hooks 设置完成（提交前将运行 gitleaks 密钥扫描）。" diff --git a/scripts/tests/security/test-check-webhook-env.sh b/scripts/tests/security/test-check-webhook-env.sh new file mode 100755 index 00000000..869c2c96 --- /dev/null +++ b/scripts/tests/security/test-check-webhook-env.sh @@ -0,0 +1,257 @@ +#!/usr/bin/env bash +# review3f/review3g/review3h：scripts/check-webhook-env.sh 的回归测试（规则对应当前 Go loader 校验）。 +# 每个用例分别用 sh、bash、bash --posix 运行被测脚本（机器上没有的解释器跳过），断言退出码与指定字段的结论； +# 另断言输出里不出现任何取值（合成的哨兵数字 / 合成密钥）。只读：样本写在 mktemp 目录；config/ 只读取。 +# 用法: scripts/tests/security/test-check-webhook-env.sh 兼容 macOS（bash 3.2 / BSD grep、sed、awk）。 +set -eu +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(git -C "$HERE" rev-parse --show-toplevel)" +CHK="$REPO/scripts/check-webhook-env.sh" +TMP="$(mktemp -d "${TMPDIR:-/tmp}/test-check-webhook-env.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT +FAILS=0; N=0 +ok() { echo "PASS [$1]"; } +bad() { echo "FAIL [$1]"; FAILS=$((FAILS+1)); } +SHELLS="" +for s in sh bash; do command -v "$s" >/dev/null 2>&1 && SHELLS="$SHELLS $s"; done +# 合成哨兵：运行时生成，断言不出现在输出里（不是真实密钥） +SEC="sk$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" +NUM=$((700 + $(od -An -N2 -tu2 /dev/urandom | tr -d ' \n') % 300)) + +y() { printf '%s\n' "$2" >"$TMP/$1.yaml"; } # y <名字> <内容> +y repo "webhook: + common_map_renew_interval_seconds: 10 # 注释 + port: \"8889\" + default_upgrade_minutes: 30 + im_feishu: + app_secret: \"$SEC\" + request_timeout_seconds: 10 +mysql: + addr: \"u:$SEC@tcp(127.0.0.1:3306)/d\"" +y flat "port: \"8889\" +fixed_workers: 10 +mysql: + addr: \"\"" +y flatPORT "PORT: \"8889\"" +y noroot "mysql: + addr: \"\"" +y whnull "webhook: +mysql: + addr: \"\"" +: >"$TMP/empty.yaml" +y q100 "webhook: + default_upgrade_minutes: \"10.0\"" +y uq100 "webhook: + default_upgrade_minutes: 10.0" +y qsp "webhook: + fixed_workers: \" 10\"" +y neg "webhook: + common_map_renew_interval_seconds: -5" +y zero "webhook: + common_map_renew_interval_seconds: 0" +y p0080 "webhook: + port: \"0080\"" +y pboth "webhook: + port: \"8889\" + PORT: \"8890\"" +y tmo0 "webhook: + im_feishu: + request_timeout_seconds: 0" +# review3g：任意仅大小写不同 / 完全相同的同名键 +y dupfront "webhook: + front_domain: a + port: \"8889\" + FRONT_DOMAIN: b" +y dupim "webhook: + im_feishu: + private_robot_app_id: \"$SEC\" + request_timeout_seconds: 10 + PRIVATE_ROBOT_APP_ID: \"$SEC\"" +y dupimblock "webhook: + im_feishu: + request_timeout_seconds: 10 + IM_FEISHU: + group_message_api: x" +y dupsame "webhook: + min_workers: 1 + min_workers: 2" +y duproot "webhook: + port: \"8889\" +Webhook: + port: \"8890\"" +y dupmysql "webhook: + port: \"8889\" +mysql: + addr: \"u:$SEC@tcp(h:1)/d\" + ADDR: \"u:$SEC@tcp(h:1)/d\"" +y nodup "webhook: + port: \"8889\" + im_feishu: + port: 1 + request_timeout_seconds: 10 +mysql: + port: 3306 +extra: + - name: a + port: 1 + - name: b + port: 2 + NAME: c" +y emptyval "webhook: + port: \"\"" +y sentinel "webhook: + fixed_workers: $NUM" +printf 'webhook:\r\n port: "8889"\r\n fixed_workers: 3\r\n' >"$TMP/crlf.yaml" +# review3h P3-d：先去注释再判断（制表符 + 注释、行尾注释），多文档、块标量里的 port 不误报；不支持的写法判不合规/请人工检查 +printf 'webhook:\t# 注释\n port: "8889"\n' >"$TMP/tabc.yaml" +y spc "webhook: # 注释 + port: 8889 # 端口 + im_feishu: # 飞书 + request_timeout_seconds: 10 # 秒" +y multidoc "webhook: + port: \"8889\" +--- +webhook: + port: abc + PORT: 1 +port: 1" +y leaddoc "--- +webhook: + port: 8889" +y blockscalar "doc: | + webhook: + port: 1 +webhook: + port: \"8889\" + note: | + port: abc + PORT: 1 + + fixed_workers: x + fixed_workers: 3 +mysql: + desc: >- + port: 1 + port: 2" +y qhash "webhook: + front_domain: \"http://x/#/a\" # 注释 + backend_domain: 'it''s # 不是注释' + alert_manager_api: http://x/a#b + port: 8889" +y uflow "webhook: + im_feishu: {request_timeout_seconds: 0}" +y uflowseq "webhook: + port: [1]" +y uanchor "base: &b + port: 1 +webhook: + <<: *b" +y ualias "webhook: + port: *p" +y uesc "webhook: + \"po\\x72t\": 1" +y uq "webhook: + ? port + : 1" +printf 'webhook:\n\tport: 1\n' >"$TMP/utab.yaml" +printf 'WEBHOOK_FIXED_WORKERS="12"\nexport WEBHOOK_PORT=9001\nWEBHOOK_MYSQL_ADDR=u:%s@tcp(h:1)/d\nWEBHOOK_IM_FEISHU_PRIVATE_ROBOT_APP_SECRET=%s\n' "$SEC" "$SEC" >"$TMP/a.env" + +# case_ <名字> <期望退出码> <期望行（“字段: 结论前缀”，可多条用 | 分隔）> -- [一个 VAR=值] -- <参数…> +# VAR=值 整体作为一个参数传给 env，值里的空格原样保留 +case_() { + name="$1"; want_rc="$2"; want="$3"; shift 3 + E1=""; [ "$1" = "--" ] && shift + if [ "$1" != "--" ]; then E1="$1"; shift; fi + shift + for s in $SHELLS "bash --posix"; do + case "$s" in "bash --posix") command -v bash >/dev/null 2>&1 || continue ;; esac + N=$((N+1)) + # 清掉调用方环境里的 WEBHOOK_*，只带用例指定的变量（另带合成密钥变量，验证不被读出） + rc=0 + env -i PATH="$PATH" WEBHOOK_MYSQL_ADDR="u:$SEC@tcp(h:1)/d" WEBHOOK_IM_FEISHU_PRIVATE_ROBOT_APP_SECRET="$SEC" ${E1:+"$E1"} \ + $s "$CHK" "$@" >"$TMP/out" 2>&1 || rc=$? + msg="${name}（${s}）" + [ "$rc" -eq "$want_rc" ] || { bad "$msg 退出码 $rc，期望 $want_rc"; sed 's/^/ /' "$TMP/out"; continue; } + if grep -F -q -e "$SEC" -e "$NUM" "$TMP/out"; then bad "$msg 输出里出现了取值"; continue; fi + miss="" + old_ifs="$IFS"; IFS='|' + for w in $want; do grep -F -q -e "$w" "$TMP/out" || miss="$miss [$w]"; done + IFS="$old_ifs" + [ -z "$miss" ] && ok "$msg" || { bad "$msg 缺:$miss"; sed 's/^/ /' "$TMP/out"; } + done +} +C="--config" +case_ "仓库形态 yaml、无环境变量" 0 "配置文件根节点: 合规|配置文件重复键: 合规|webhook.port: 合规（配置文件）|webhook.fixed_workers: 合规（内置默认值）|webhook.im_feishu.request_timeout_seconds: 合规（配置文件）" -- -- $C "$TMP/repo.yaml" +case_ "仓库 config/webhook.yaml + env.example" 0 "配置文件根节点: 合规|配置文件重复键: 合规|结果: 合规（告警 0 项）" -- -- $C "$REPO/config/webhook.yaml" --env-file "$REPO/env.example" +case_ "平铺 webhook 字段" 1 "配置文件根节点: 不合规（根节点下有 webhook 字段 port, fixed_workers" -- -- $C "$TMP/flat.yaml" +case_ "平铺大写 PORT" 1 "配置文件根节点: 不合规（根节点下有 webhook 字段 port" -- -- $C "$TMP/flatPORT.yaml" +case_ "无根节点只有 mysql" 0 "配置文件根节点: 告警|结果: 合规（告警 1 项）" -- -- $C "$TMP/noroot.yaml" +case_ "webhook: 为空" 0 "配置文件根节点: 告警" -- -- $C "$TMP/whnull.yaml" +case_ "空文件" 0 "配置文件根节点: 告警" -- -- $C "$TMP/empty.yaml" +for v in 10.0 " 10" "10 " 10.5 1e1 -1 +10 0x10; do + case_ "WEBHOOK_FIXED_WORKERS='$v'" 1 "webhook.fixed_workers: 不合规（环境变量 WEBHOOK_FIXED_WORKERS：不是十进制整数" -- "WEBHOOK_FIXED_WORKERS=$v" -- +done +case_ "WEBHOOK_FIXED_WORKERS=010" 1 "webhook.fixed_workers: 不合规（环境变量 WEBHOOK_FIXED_WORKERS：有前导零" -- WEBHOOK_FIXED_WORKERS=010 -- +case_ "WEBHOOK_FIXED_WORKERS=0" 1 "webhook.fixed_workers: 不合规（环境变量 WEBHOOK_FIXED_WORKERS：小于 1" -- WEBHOOK_FIXED_WORKERS=0 -- +case_ "WEBHOOK_DEFAULT_UPGRADE_MINUTES=0" 1 "webhook.default_upgrade_minutes: 不合规（环境变量 WEBHOOK_DEFAULT_UPGRADE_MINUTES：小于 1" -- WEBHOOK_DEFAULT_UPGRADE_MINUTES=0 -- +case_ "WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS=0" 1 "webhook.common_map_renew_interval_seconds: 不合规" -- WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS=0 -- +case_ "WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS=0" 1 "webhook.im_feishu.request_timeout_seconds: 不合规（环境变量" -- WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS=0 -- +for bound in WEBHOOK_FIXED_WORKERS:fixed_workers:1024 WEBHOOK_DEFAULT_UPGRADE_MINUTES:default_upgrade_minutes:10080 WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS:common_map_renew_interval_seconds:86400 WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS:im_feishu.request_timeout_seconds:600; do + ev="${bound%%:*}"; rest="${bound#*:}"; field="${rest%:*}"; upper="${rest##*:}" + case_ "${ev}=upper boundary" 0 "webhook.${field}: 合规（环境变量" -- "${ev}=${upper}" -- + case_ "${ev}=above upper boundary" 1 "webhook.${field}: 不合规（环境变量" -- "${ev}=$((upper+1))" -- +done +case_ "WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS=-1" 1 "webhook.im_feishu.request_timeout_seconds: 不合规" -- WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS=-1 -- +case_ "yaml 加引号 10.0" 1 "webhook.default_upgrade_minutes: 不合规（配置文件：不是十进制整数" -- -- $C "$TMP/q100.yaml" +case_ "yaml 不加引号 10.0" 1 "webhook.default_upgrade_minutes: 不合规（配置文件：不是十进制整数" -- -- $C "$TMP/uq100.yaml" +case_ "yaml 加引号带空格" 1 "webhook.fixed_workers: 不合规（配置文件" -- -- $C "$TMP/qsp.yaml" +case_ "yaml 负数" 1 "webhook.common_map_renew_interval_seconds: 不合规（配置文件" -- -- $C "$TMP/neg.yaml" +case_ "yaml 0" 1 "webhook.common_map_renew_interval_seconds: 不合规（配置文件：小于 1" -- -- $C "$TMP/zero.yaml" +case_ "yaml timeout 0" 1 "webhook.im_feishu.request_timeout_seconds: 不合规（配置文件" -- -- $C "$TMP/tmo0.yaml" +case_ "yaml port 0080" 1 "webhook.port: 不合规（配置文件：有前导零" -- -- $C "$TMP/p0080.yaml" +for v in 0080 0 65536 abc " 9998 "; do + case_ "WEBHOOK_PORT='$v'" 1 "webhook.port: 不合规（环境变量 WEBHOOK_PORT" -- "WEBHOOK_PORT=$v" -- +done +case_ "WEBHOOK_PORT=65535" 0 "webhook.port: 合规（环境变量 WEBHOOK_PORT）" -- WEBHOOK_PORT=65535 -- +case_ "port 与 PORT 同写" 1 "webhook.port: 不合规（配置文件同一层重复写了 port|配置文件重复键: 不合规（同一层有仅大小写不同或完全相同的键：webhook.port；" -- -- $C "$TMP/pboth.yaml" +case_ "port 与 PORT 同写 + WEBHOOK_PORT 已设" 1 "webhook.port: 不合规（配置文件同一层重复写了 port|配置文件重复键: 不合规" -- WEBHOOK_PORT=9001 -- $C "$TMP/pboth.yaml" +case_ "front_domain 与 FRONT_DOMAIN 同写" 1 "配置文件重复键: 不合规（同一层有仅大小写不同或完全相同的键：webhook.front_domain；|webhook.port: 合规（配置文件）" -- -- $C "$TMP/dupfront.yaml" +case_ "im_feishu 下 app_id 大小写重复（值不回显）" 1 "配置文件重复键: 不合规（同一层有仅大小写不同或完全相同的键：webhook.im_feishu.private_robot_app_id；" -- -- $C "$TMP/dupim.yaml" +case_ "im_feishu 与 IM_FEISHU 两个块" 1 "配置文件重复键: 不合规（同一层有仅大小写不同或完全相同的键：webhook.im_feishu；" -- -- $C "$TMP/dupimblock.yaml" +case_ "完全相同的键写两次" 1 "配置文件重复键: 不合规（同一层有仅大小写不同或完全相同的键：webhook.min_workers；" -- -- $C "$TMP/dupsame.yaml" +case_ "webhook 与 Webhook 两个根" 1 "配置文件根节点: 不合规（webhook 根节点重复）|配置文件重复键: 不合规（同一层有仅大小写不同或完全相同的键：webhook, webhook.port；" -- -- $C "$TMP/duproot.yaml" +case_ "mysql 下 addr 与 ADDR（值不回显）" 1 "配置文件重复键: 不合规（同一层有仅大小写不同或完全相同的键：mysql.addr；" -- -- $C "$TMP/dupmysql.yaml" +case_ "不同父节点同名、列表项同名不算重复" 0 "配置文件重复键: 合规|结果: 合规" -- -- $C "$TMP/nodup.yaml" +case_ "yaml 空值" 1 "webhook.port: 不合规（配置文件里是空值" -- -- $C "$TMP/emptyval.yaml" +case_ "环境变量覆盖文件" 0 "webhook.common_map_renew_interval_seconds: 合规（环境变量" -- WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS=5 -- $C "$TMP/zero.yaml" +case_ "空环境变量视为未设置" 1 "webhook.common_map_renew_interval_seconds: 不合规（配置文件：小于 1" -- WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS= -- $C "$TMP/zero.yaml" +case_ "哨兵值不回显" 0 "webhook.fixed_workers: 合规（配置文件）" -- -- $C "$TMP/sentinel.yaml" +case_ "CRLF 文件" 0 "webhook.port: 合规（配置文件）|webhook.fixed_workers: 合规（配置文件）" -- -- $C "$TMP/crlf.yaml" +case_ "--env-file（引号、export、密钥行）" 0 "webhook.fixed_workers: 合规（环境变量 WEBHOOK_FIXED_WORKERS）|webhook.port: 合规（环境变量 WEBHOOK_PORT）" -- -- --env-file "$TMP/a.env" +case_ "--config 文件不存在" 2 "配置文件不可读" -- -- $C "$TMP/nope.yaml" +# review3h P3-d +case_ "webhook:# 注释" 0 "配置文件根节点: 合规|配置文件写法: 合规|webhook.port: 合规（配置文件）" -- -- $C "$TMP/tabc.yaml" +case_ "webhook: # 注释 与行尾注释" 0 "配置文件根节点: 合规|webhook.port: 合规（配置文件）|webhook.im_feishu.request_timeout_seconds: 合规（配置文件）" -- -- $C "$TMP/spc.yaml" +case_ "多文档 --- 拒绝启动" 1 "配置文件根节点: 合规|配置文件重复键: 合规|webhook.port: 合规（配置文件）|配置文件写法: 不合规（多文档" -- -- $C "$TMP/multidoc.yaml" +case_ "开头的 --- 不算多文档" 0 "配置文件根节点: 合规|配置文件写法: 合规|webhook.port: 合规（配置文件）|结果: 合规（告警 0 项）" -- -- $C "$TMP/leaddoc.yaml" +case_ "块标量里的 port/webhook 不算键" 0 "配置文件根节点: 合规|配置文件重复键: 合规|配置文件写法: 合规|webhook.port: 合规（配置文件）|webhook.fixed_workers: 合规（配置文件）" -- -- $C "$TMP/blockscalar.yaml" +case_ "引号内与非空白后的 # 不算注释（对照）" 0 "配置文件写法: 合规|配置文件重复键: 合规|webhook.port: 合规（配置文件）|结果: 合规（告警 0 项）" -- -- $C "$TMP/qhash.yaml" +case_ "内联表 im_feishu: {...}" 1 "配置文件写法: 不合规（含本脚本不支持的 YAML 写法：内联表/数组 第 2 行；请人工检查）" -- -- $C "$TMP/uflow.yaml" +case_ "内联数组 port: [1]" 1 "配置文件写法: 不合规（含本脚本不支持的 YAML 写法：内联表/数组 第 2 行；请人工检查）" -- -- $C "$TMP/uflowseq.yaml" +case_ "锚点 + 合并键 <<: *b" 1 "配置文件写法: 不合规（含本脚本不支持的 YAML 写法：锚点/别名 第 1 行, 合并键 << 第 4 行, 锚点/别名 第 4 行；请人工检查）" -- -- $C "$TMP/uanchor.yaml" +case_ "别名 port: *p" 1 "配置文件写法: 不合规（含本脚本不支持的 YAML 写法：锚点/别名 第 2 行；请人工检查）" -- -- $C "$TMP/ualias.yaml" +case_ "转义键 \"po\\x72t\"" 1 "配置文件写法: 不合规（含本脚本不支持的 YAML 写法：转义键 第 2 行；请人工检查）" -- -- $C "$TMP/uesc.yaml" +case_ "复杂键 ? port" 1 "配置文件写法: 不合规（含本脚本不支持的 YAML 写法：复杂键 ? 第 2 行；请人工检查）" -- -- $C "$TMP/uq.yaml" +case_ "制表符缩进" 1 "配置文件写法: 不合规（含本脚本不支持的 YAML 写法：制表符缩进 第 2 行；请人工检查）" -- -- $C "$TMP/utab.yaml" +# review3h P3-e：临时文件用 mktemp 创建并检查；TMPDIR 不可用时报错退出 2，不得继续出结论；运行后不留临时文件 +case_ "TMPDIR 不存在" 2 "无法创建临时文件" -- "TMPDIR=$TMP/no-such-dir" -- $C "$TMP/repo.yaml" +case_ "TMPDIR 不存在（无 --config）" 2 "无法创建临时文件" -- "TMPDIR=$TMP/no-such-dir" -- +mkdir "$TMP/td" +for s in $SHELLS "bash --posix"; do + N=$((N+1)); rc=0 + env -i PATH="$PATH" TMPDIR="$TMP/td" $s "$CHK" $C "$TMP/repo.yaml" >"$TMP/out" 2>&1 || rc=$? + left="$(ls -A "$TMP/td" | wc -l | tr -d ' ')" + if [ "$rc" -eq 0 ] && [ "$left" = 0 ]; then ok "运行后 TMPDIR 无残留（${s}）"; else bad "运行后 TMPDIR 残留 ${left} 个 / 退出码 ${rc}（${s}）"; fi +done +echo "共 ${N} 次运行（解释器:${SHELLS} bash --posix）；失败 ${FAILS} 项" +[ "$FAILS" -eq 0 ] diff --git a/scripts/tests/security/test-env-example.sh b/scripts/tests/security/test-env-example.sh new file mode 100755 index 00000000..2ac9056a --- /dev/null +++ b/scripts/tests/security/test-env-example.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# SEC-01A 第三轮补修（review3b）：env.example 与端口 / 认证相关的说明必须和代码一致。 +# 端口 PROMETHEUS_HTTP_SD_API、WEBHOOK_BACKEND_DOMAIN 指向主后端端口（SERVER_PORT = config 中 server.port）； +# WEBHOOK_PORT 若出现，须留空或等于 config/webhook.yaml 的 webhook.port，注释分别说明合入前（不生效）/ +# 合入 sched/w1-mantou 后（bb410eb 起生效）的行为与改端口方法；PROMETHEUS_ALERT_WEBHOOK_ADDR 指向 webhook 端口； +# 本人负责的范围（env.example、deploy/、docs/、scripts/、.github/、根目录 README* / compose / Makefile） +# 不得再以“值”的形式出现旧端口 8888（注释里解释历史默认值可以；登记的例外见 ALLOW_8888）。 +# JWT JWT_EXPIRATION=30（单位分钟），注释写明单位与合入前/后默认值，并与 deploy/release/compose.yaml 一致； +# 本人负责的范围内提到 JWT_EXPIRATION 的每个文档都写明单位为分钟，且不再出现 JWT_EXPIRATION=3600。 +# 合入 也可在合入 sched/w1-mantou 后的工作区运行：馒头新增的 w1-*.md 只登记；“8888 改为 8889”同行说明不算旧值。 +# Redis REDIS_PASSWORD 值为空且注释标明 compose 必填。 +# 整数 WEBHOOK_* 整数字段的注释与取值符合 sched/w1-mantou 0146077 的范围（review3f）。 +# +# 用法: scripts/tests/security/test-env-example.sh [--rev ] （不带 --rev 检查工作区；对修复前版本运行应报 FAIL） +# 只读：不修改任何文件；config/ 仅读取作对照。兼容 macOS（bash 3.2 / BSD grep、sed、awk）。 +set -eu +# pipefail：bash（含 --posix 和 macOS 自带 3.2）、zsh、box 上的 dash 0.5.12 都实测支持；更老的 dash 或其它 sh 不支持， +# 所以先在子 shell 里试一下，不支持就打印提示并在没有 pipefail 的情况下继续。没有 pipefail 时，下面 git grep 的 +# 出错码（>1）会被管道后面的 sed 盖住，出错时命中列表为空，“禁止出现”类检查可能漏报（与加 pipefail 之前的行为相同）。 +if (set -o pipefail) 2>/dev/null; then set -o pipefail +else echo "注意: 当前 shell 不支持 set -o pipefail，按不带 pipefail 运行" >&2; fi + +REV="" +while [ $# -gt 0 ]; do + case "$1" in + --rev) [ $# -ge 2 ] || { echo "--rev 需要参数" >&2; exit 2; }; REV="$2"; shift 2 ;; + -h|--help) sed -n '2,10p' "$0"; exit 0 ;; + *) echo "未知参数: $1" >&2; exit 2 ;; + esac +done + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(git -C "$HERE" rev-parse --show-toplevel)" +if [ -n "$REV" ]; then git -C "$REPO" rev-parse --verify -q "$REV^{commit}" >/dev/null || { echo "无效的 rev: $REV" >&2; exit 2; }; fi +TMP="$(mktemp -d "${TMPDIR:-/tmp}/test-env-example.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT +FAILS=0 +ok() { echo "PASS [$1]"; } +bad() { echo "FAIL [$1]"; FAILS=$((FAILS+1)); } +# gg_rc ：git grep 没有命中时退出码为 1，属正常；其它非 0 是真出错（如 rev 或路径规格不对），直接中止 +gg_rc() { [ "$1" -eq 1 ] || { echo "git grep 出错（退出码 $1）" >&2; exit 2; }; } + +# src <路径>：从 --rev 或工作区取文件内容 +src() { if [ -n "$REV" ]; then git -C "$REPO" show "$REV:$1"; else cat "$REPO/$1"; fi; } +src env.example >"$TMP/env" +src config/config.development.yaml >"$TMP/dev.yaml" +src config/webhook.yaml >"$TMP/webhook.yaml" + +# val <变量名>：env.example 中该变量的值（只看未注释的行） +val() { sed -n "s/^$1=//p" "$TMP/env" | tail -n 1; } +# block <变量名>：该变量赋值行之前紧挨着的连续注释行 +block() { awk -v k="$1=" '/^#/{b=b $0 "\n"; next} index($0,k)==1{printf "%s", b; exit} {b=""}' "$TMP/env"; } + +yaml_srv="$(awk '/^server:/{f=1;next} f&&/^[^ #]/{f=0} f&&/^ port:/{sub(/#.*/,"");gsub(/[^0-9]/,"");print;exit}' "$TMP/dev.yaml")" +yaml_wh="$(awk '/^webhook:/{f=1;next} f&&/^[^ #]/{f=0} f&&/^ port:/{sub(/#.*/,"");gsub(/[^0-9]/,"");print;exit}' "$TMP/webhook.yaml")" +srv="$(val SERVER_PORT)" +echo "目标: ${REV:-工作区} SERVER_PORT=${srv} config.development.yaml server.port=${yaml_srv} config/webhook.yaml webhook.port=${yaml_wh}" + +# ---------- 主后端端口 ---------- +[ -n "$srv" ] && [ "$srv" = "$yaml_srv" ] && ok "SERVER_PORT=${srv} 与 config.development.yaml 一致" \ + || bad "SERVER_PORT=${srv}，config.development.yaml server.port=${yaml_srv}" +sd="$(val PROMETHEUS_HTTP_SD_API)" +case "$sd" in + "http://localhost:${srv}/"*) ok "PROMETHEUS_HTTP_SD_API 指向主后端端口 ${srv}" ;; + *) bad "PROMETHEUS_HTTP_SD_API=${sd} 未指向主后端端口 ${srv}" ;; +esac +bd="$(val WEBHOOK_BACKEND_DOMAIN)" +case "$bd" in + "http://localhost:${srv}"|"http://localhost:${srv}/"*) ok "WEBHOOK_BACKEND_DOMAIN 指向主后端端口 ${srv}" ;; + *) bad "WEBHOOK_BACKEND_DOMAIN=${bd} 未指向主后端端口 ${srv}" ;; +esac + +# ---------- WEBHOOK_PORT / PROMETHEUS_ALERT_WEBHOOK_ADDR ---------- +# 合入前 WEBHOOK_PORT 不生效、端口取自 config/webhook.yaml；合入 sched/w1-mantou（bb410eb 起）后 WEBHOOK_PORT > 文件 > 默认 8889 +wh_eff="$yaml_wh" +if grep -q '^WEBHOOK_PORT=' "$TMP/env"; then + wp="$(val WEBHOOK_PORT)" + if [ -z "$wp" ] || [ "$wp" = "$yaml_wh" ]; then ok "WEBHOOK_PORT='${wp}'（留空或与 config/webhook.yaml 一致）" + else bad "WEBHOOK_PORT=${wp}，config/webhook.yaml webhook.port=${yaml_wh}（合入前不生效、合入后会改变监听端口）"; wh_eff="$wp"; fi + block WEBHOOK_PORT >"$TMP/wp.txt" + for kw in 'config/webhook.yaml' '--config' '根节点' '同一台机器' '优先级' 'PROMETHEUS_ALERT_WEBHOOK_ADDR' '拒绝启动' '[配置告警]' '继续启动' '前导零' '重复键' '多文档'; do + if grep -F -q -e "$kw" "$TMP/wp.txt"; then ok "WEBHOOK_PORT 注释含「${kw}」" + else bad "WEBHOOK_PORT 注释缺「${kw}」"; fi + done +else + ok "env.example 未设置 WEBHOOK_PORT" +fi +if grep -q '^WEBHOOK_MYSQL_ADDR=' "$TMP/env"; then + block WEBHOOK_MYSQL_ADDR >"$TMP/wm.txt" + if grep -F -q '必填' "$TMP/wm.txt" && grep -F -q '不读 MYSQL_ADDR' "$TMP/wm.txt" && grep -F -q 'scripts/release-env.py' "$TMP/wm.txt"; then + ok "WEBHOOK_MYSQL_ADDR 说明完整 DSN 必填、独立读取与发布生成方式" + else bad "WEBHOOK_MYSQL_ADDR 缺少必填、独立读取或生成器说明"; fi +fi +# 当前 Go loader 的范围；测试真实边界，避免过时的分支说明代替有效规则。 +if grep -q '^WEBHOOK_FIXED_WORKERS=' "$TMP/env"; then + block WEBHOOK_FIXED_WORKERS >"$TMP/wi.txt" + for kw in '十进制整数' '1-1024' '1-10080' '1-86400' '1-600' '0 不允许' '拒绝启动'; do + grep -F -q -e "$kw" "$TMP/wi.txt" && ok "整数字段注释含「${kw}」" || bad "整数字段注释缺「${kw}」" + done + for k in WEBHOOK_FIXED_WORKERS:1024 WEBHOOK_DEFAULT_UPGRADE_MINUTES:10080 WEBHOOK_COMMON_MAP_RENEW_INTERVAL_SECONDS:86400 WEBHOOK_IM_FEISHU_REQUEST_TIMEOUT_SECONDS:600; do + n="${k%%:*}"; hi="${k#*:}"; v="$(val "$n")" + case "$v" in + '') ok "${n} 留空（取配置文件/默认值）" ;; + 0|*[!0-9]*|0*) bad "${n} 不是正十进制整数" ;; + *) if [ "${#v}" -le 9 ] && [ "$v" -le "$hi" ]; then ok "${n} 位于 1-${hi}"; else bad "${n} 超出 1-${hi}"; fi ;; + esac + done +fi +aw="$(val PROMETHEUS_ALERT_WEBHOOK_ADDR)" +case "$aw" in + "http://localhost:${wh_eff}/api/v1/alerts/receive") ok "PROMETHEUS_ALERT_WEBHOOK_ADDR 指向 webhook 端口 ${wh_eff}" ;; + *) bad "PROMETHEUS_ALERT_WEBHOOK_ADDR=${aw} 未指向 webhook 端口 ${wh_eff}" ;; +esac +block PROMETHEUS_ALERT_WEBHOOK_ADDR >"$TMP/aw.txt" +grep -F -q 'WEBHOOK_PORT' "$TMP/aw.txt" && ok "PROMETHEUS_ALERT_WEBHOOK_ADDR 注释说明与 WEBHOOK_PORT 联动" \ + || bad "PROMETHEUS_ALERT_WEBHOOK_ADDR 缺注释：指向 webhook 进程、端口须与 WEBHOOK_PORT 一致" + +# ---------- 旧端口 8888 ---------- +# 登记的例外（不属本人范围或另有归属，见 README-ops）： +# docs/release/prometheus-scrape-auth.md —— 该行在 sched/w1-mantou 中已由馒头改写，合并时以其版本为准 +# MANTOU_DOCS —— 馒头 sched/w1-mantou 新增的发布说明（合入后才出现），其中引用旧 env.example 的 8888/3600 作为 +# 迁移说明，归馒头，只登记不检查 +ALLOW_8888='docs/release/prometheus-scrape-auth.md' +MANTOU_DOCS='docs/release/w1-jwt-config.md docs/release/w1-webhook-env.md' +SELF='scripts/tests/security/test-env-example.sh' +if [ -n "$REV" ]; then + git -C "$REPO" grep -n -I -F 8888 "$REV" -- . ':!internal/' ':!config/' ':!go.sum' 2>/dev/null | sed "s|^$REV:||" >"$TMP/hits" || gg_rc $? +else + git -C "$REPO" grep -n -I -F 8888 -- . ':!internal/' ':!config/' ':!go.sum' 2>/dev/null >"$TMP/hits" || gg_rc $? +fi +: >"$TMP/bad8888" +while IFS= read -r h; do + f="${h%%:*}"; rest="${h#*:}"; text="${rest#*:}" + [ "$f" = "$SELF" ] && continue + case " $ALLOW_8888 $MANTOU_DOCS " in *" $f "*) echo " 登记例外: ${f}:${rest%%:*}"; continue ;; esac + # 同一行同时写了 8889 的，是“8888 改为 8889”一类迁移说明，不算旧值 + case "$text" in *8889*) continue ;; esac + # 注释行（# 或 // 开头）只是解释历史默认值，不算“值” + stripped="$(printf '%s' "$text" | sed 's/^[[:space:]]*//')" + case "$stripped" in '#'*|'//'*) continue ;; esac + echo "$h" >>"$TMP/bad8888" +done <"$TMP/hits" +if [ -s "$TMP/bad8888" ]; then bad "本人范围内仍有非注释的 8888：$(tr '\n' ' ' <"$TMP/bad8888")" +else ok "本人范围内无非注释的 8888（internal/、config/ 不属本人，只登记）"; fi + +# ---------- JWT_EXPIRATION ---------- +je="$(val JWT_EXPIRATION)" +[ "$je" = 30 ] && ok "JWT_EXPIRATION=30" || bad "JWT_EXPIRATION=${je}（应为 30 分钟；3600 实为 60 小时，合入后生产上限 1440）" +block JWT_EXPIRATION >"$TMP/je.txt" +for kw in '单位是分钟' '合入 sched/w1-mantou 前' '合入 sched/w1-mantou 后' '1440'; do + if grep -F -q -e "$kw" "$TMP/je.txt"; then ok "JWT_EXPIRATION 注释含「${kw}」" + else bad "JWT_EXPIRATION 注释缺「${kw}」"; fi +done +src deploy/release/compose.yaml >"$TMP/compose.yaml" 2>/dev/null || : >"$TMP/compose.yaml" +cj="$(sed -n '/^[[:space:]]*JWT_EXPIRATION:/{s/^[[:space:]]*JWT_EXPIRATION:[[:space:]]*"\{0,1\}\([0-9]*\)"\{0,1\}.*/\1/p;q;}' "$TMP/compose.yaml")" +[ "$cj" = "$je" ] && ok "deploy/release/compose.yaml JWT_EXPIRATION=${cj} 与 env.example 一致" \ + || bad "deploy/release/compose.yaml JWT_EXPIRATION=${cj}，env.example 为 ${je}" +if [ -n "$REV" ]; then + git -C "$REPO" grep -n -I -F JWT_EXPIRATION "$REV" -- . ':!internal/' ':!config/' 2>/dev/null | sed "s|^$REV:||" >"$TMP/jhits" || gg_rc $? +else + git -C "$REPO" grep -n -I -F JWT_EXPIRATION -- . ':!internal/' ':!config/' 2>/dev/null >"$TMP/jhits" || gg_rc $? +fi +: >"$TMP/badjwt" +# 单位按文件判断：一个 .md 提到 JWT_EXPIRATION 时，其中至少一处提到它的行写明分钟/minute +# 逐行读取文件名（文件名含空格也不会被拆开）；用 awk 按第一个字段精确比较文件名（不当正则，不做前缀匹配） +cut -d: -f1 "$TMP/jhits" | sort -u >"$TMP/jfiles" +while IFS= read -r f; do + case "$f" in *.md) ;; *) continue ;; esac + case " $MANTOU_DOCS " in *" $f "*) continue ;; esac + F="$f" awk -F: '$1 == ENVIRON["F"] && /分钟|minute/ { found = 1; exit } END { exit !found }' "$TMP/jhits" \ + || echo "${f}（提到 JWT_EXPIRATION 但未写单位）" >>"$TMP/badjwt" +done <"$TMP/jfiles" +while IFS= read -r h; do + f="${h%%:*}" + case "$f" in "$SELF"|env.example|deploy/release/compose.yaml|*.go) continue ;; esac + case " $MANTOU_DOCS " in *" $f "*) echo " 登记（馒头）: ${f}:$(printf '%s' "${h#*:}" | cut -d: -f1)"; continue ;; esac + if printf '%s' "$h" | grep -E -q 'JWT_EXPIRATION[=:][[:space:]]*"?3600'; then echo "${h}（3600）" >>"$TMP/badjwt"; fi +done <"$TMP/jhits" +if [ -s "$TMP/badjwt" ]; then bad "本人范围内 JWT_EXPIRATION 说明不符：$(tr '\n' ' ' <"$TMP/badjwt")" +else ok "本人范围内提到 JWT_EXPIRATION 的文档均写明分钟、无 3600（internal/、config/ 只登记）"; fi + +# ---------- Redis ---------- +rp="$(val REDIS_PASSWORD)" +[ -z "$rp" ] && ok "REDIS_PASSWORD 值为空" || bad "REDIS_PASSWORD 不为空" +block REDIS_PASSWORD >"$TMP/rp.txt" +grep -F -q '必填' "$TMP/rp.txt" && ok "REDIS_PASSWORD 注释标明 compose 必填" || bad "REDIS_PASSWORD 注释未标明必填" +if grep -F -q '默认空（无认证）' "$TMP/rp.txt"; then bad "REDIS_PASSWORD 仍写着“可选：默认空”"; fi + +echo "目标: ${REV:-工作区}；失败 ${FAILS} 项" +[ "$FAILS" -eq 0 ] diff --git a/scripts/tests/security/test-gitignore-example.sh b/scripts/tests/security/test-gitignore-example.sh new file mode 100755 index 00000000..9a5cc0ef --- /dev/null +++ b/scripts/tests/security/test-gitignore-example.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# SEC-01A 审查 P3：.gitignore 中宽泛的 "!*.example" 会把被 ".env.*" 等规则忽略的文件重新放出来 +# （如 .env.prod.example、config/.env.local.example 这类常被拿来存真实值的文件名）。 +# 修复后只对被跟踪的模板逐个放行。 +# 用法: <本脚本> [--rev ] （对修复前版本运行应报 FAIL） +set -euo pipefail +REPO="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)" +REV="" +while [ $# -gt 0 ]; do case "$1" in --rev) REV="$2"; shift 2;; *) echo "未知参数: $1" >&2; exit 2;; esac; done +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +export GIT_CONFIG_GLOBAL="$TMP/gitconfig" GIT_CONFIG_NOSYSTEM=1; : >"$GIT_CONFIG_GLOBAL" +git init -q "$TMP/r" +if [ -n "$REV" ]; then git -C "$REPO" show "$REV:.gitignore"; else cat "$REPO/.gitignore"; fi >"$TMP/r/.gitignore" +FAILS=0 +ign() { git -C "$TMP/r" check-ignore -q --no-index -- "$1"; } +expect_tracked() { if ign "$1"; then echo "FAIL [模板应被跟踪] $1 被忽略"; FAILS=$((FAILS+1)); else echo "PASS [模板应被跟踪] $1"; fi; } +expect_ignored() { if ign "$1"; then echo "PASS [应被忽略] $1"; else echo "FAIL [应被忽略] $1 未被忽略（可被 git add 提交）"; FAILS=$((FAILS+1)); fi; } + +# 1) 仓库中当前被跟踪的所有 *.example 模板都不能被忽略 +# 不用进程替换 "< <(...)"：macOS bash 3.2 的 --posix 模式下是语法错误（review3h 遗留项） +git -C "$REPO" ls-files -- '*.example' | sort >"$TMP/examples.txt" +while IFS= read -r f; do expect_tracked "$f"; done <"$TMP/examples.txt" +# 2) 被 .env.* 覆盖的"伪模板"必须仍被忽略 +expect_ignored ".env.example" +expect_ignored ".env.prod.example" +expect_ignored "config/.env.local.example" +expect_ignored "deploy/.env.staging.example" +# 3) 对照：真实 env / 集群凭据仍被忽略 +expect_ignored ".env" +expect_ignored ".env.bak-20261023" +expect_ignored "deploy/kubernetes/config" +echo "目标版本: ${REV:-工作区}；失败 $FAILS 项"; [ "$FAILS" -eq 0 ] diff --git a/scripts/tests/security/test-precommit-hook.sh b/scripts/tests/security/test-precommit-hook.sh new file mode 100755 index 00000000..270752e5 --- /dev/null +++ b/scripts/tests/security/test-precommit-hook.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# SEC-01A 审查 P1 复现/回归测试：.githooks/pre-commit 不得把未经 gitleaks 扫描的内容带进提交。 +# +# G1 部分暂存的 .go 文件：暂存版本无密钥，工作区未暂存部分含密钥（暂存版本格式不规范 / 规范两种） +# G2 从未暂存过的 .go 文件（含密钥、格式不规范）不得被钩子自动加入提交 +# 对照 暂存密钥必须被拦；干净提交必须成功（正向对照：钩子坏掉、对所有提交都失败时上面各项也会“PASS”， +# 所以必须有一个应当成功的提交来证明钩子本身可用） +# +# 用法: scripts/tests/security/test-precommit-hook.sh [--rev ] +# --rev 测试指定版本的 .githooks/pre-commit（默认：当前工作区文件）。对修复前版本运行应报 FAIL。 +# 依赖: git、go/gofmt、gitleaks（PATH 或 ~/.local/bin）。在 mktemp -d 的临时仓库中运行，不改动本仓库。 +# 密钥为运行时拼出的合成 GitHub PAT（ghp_ + 36 位随机字符），不是真实凭据。 +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +REV="" +while [ $# -gt 0 ]; do + case "$1" in + --rev) REV="$2"; shift 2 ;; + *) echo "未知参数: $1" >&2; exit 2 ;; + esac +done +export PATH="$HOME/.local/bin:$PATH" +for c in git go gofmt gitleaks; do command -v "$c" >/dev/null || { echo "缺少依赖: $c" >&2; exit 2; }; done + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +export GIT_CONFIG_GLOBAL="$TMP/gitconfig" GIT_CONFIG_NOSYSTEM=1 GOTOOLCHAIN=local GOFLAGS=-mod=mod +export SKIP_VET=1 SKIP_BUILD=1 SKIP_GENERATE=1 # 只关心暂存/格式化逻辑 +git config --global user.name test; git config --global user.email test@example.invalid +git config --global init.defaultBranch main + +hook_src() { + if [ -n "$REV" ]; then git -C "$REPO" show "$REV:.githooks/pre-commit"; else cat "$REPO/.githooks/pre-commit"; fi +} +fake_token() { printf 'ghp_%s' "$(LC_ALL=C tr -dc 'A-Za-z0-9' .githooks/pre-commit; chmod +x .githooks/pre-commit + printf 'module example.com/t\n\ngo 1.21\n' >go.mod + printf 'package main\n\nfunc main() {}\n' >main.go + git add -A; git -c core.hooksPath=/dev/null commit -qm init + git config core.hooksPath .githooks +} + +FAILS=0 +# 判定：提交历史中出现合成密钥 = 漏（LEAK） +check() { + local name="$1" tok="$2" rc="$3" hits + hits="$(git log --all -p | grep -cF "$tok" || true)" + if [ "$hits" -gt 0 ]; then + echo "FAIL [$name] 密钥进入了提交（commit rc=${rc}, 命中 ${hits} 处） -> 漏"; FAILS=$((FAILS+1)) + else + echo "PASS [$name] 密钥未进入提交（commit rc=${rc}）" + fi +} + +# G1a：暂存版本格式不规范且无密钥；工作区追加未暂存的密钥 +new_repo g1a; TOK="$(fake_token)" +printf 'package main\n\nfunc main() {\n}\n' >main.go; git add main.go +printf 'package main\n\nvar token = "%s"\n\nfunc main() {\n}\n' "$TOK" >main.go +set +e; git commit -qm g1a >"$TMP/g1a.out" 2>&1; rc=$?; set -e +check G1a-部分暂存-暂存版不规范 "$TOK" "$rc" +[ "$rc" -ne 0 ] && grep -q 'gofmt' "$TMP/g1a.out" && echo " (提交被拒并提示 gofmt)" + +# G1b：暂存版本规范；工作区未暂存部分含密钥且不规范 +new_repo g1b; TOK="$(fake_token)" +printf 'package main\n\nfunc main() { println("x") }\n' >main.go; git add main.go +printf 'package main\n\nvar token = "%s"\n\nfunc main() { println("x") }\n' "$TOK" >main.go +set +e; git commit -qm g1b >"$TMP/g1b.out" 2>&1; rc=$?; set -e +check G1b-部分暂存-暂存版规范 "$TOK" "$rc" + +# G2：从未暂存的 .go 文件（含密钥、不规范）；同时暂存另一个规范的 .go 改动 +new_repo g2; TOK="$(fake_token)" +printf 'package main\n\nvar leaked = "%s"\n' "$TOK" >secret.go +printf 'package main\n\nfunc main() { println("y") }\n' >main.go; git add main.go +set +e; git commit -qm g2 >"$TMP/g2.out" 2>&1; rc=$?; set -e +check G2-未暂存文件被自动加入 "$TOK" "$rc" + +# 负向对照：暂存版本含密钥时必须被 gitleaks 拦下 +new_repo ctl; TOK="$(fake_token)" +printf 'package main\n\nvar token = "%s"\n\nfunc main() {}\n' "$TOK" >main.go; git add main.go +set +e; git commit -qm ctl >"$TMP/ctl.out" 2>&1; rc=$?; set -e +check 对照-暂存密钥被gitleaks拦截 "$TOK" "$rc" +if [ "$rc" -eq 0 ] || ! grep -q 'gitleaks' "$TMP/ctl.out"; then + echo "FAIL [对照-暂存密钥被gitleaks拦截] 提交应因 gitleaks 检出而失败（rc=${rc}）"; FAILS=$((FAILS+1)) +fi + +# 正向对照：干净、格式规范的 .go 改动必须能正常提交，且提交内容就是暂存内容 +new_repo ok +printf 'package main\n\nfunc main() { println("ok") }\n' >main.go; git add main.go +before="$(git rev-parse HEAD)" +set +e; git commit -qm ok >"$TMP/ok.out" 2>&1; rc=$?; set -e +after="$(git rev-parse HEAD)" +if [ "$rc" -eq 0 ] && [ "$after" != "$before" ] && git show HEAD:main.go | grep -qF 'println("ok")'; then + echo "PASS [对照-干净提交成功] commit rc=0，HEAD ${before:0:7} -> ${after:0:7}" +else + echo "FAIL [对照-干净提交成功] 干净提交未成功（commit rc=${rc}，HEAD ${before:0:7} -> ${after:0:7}）" + sed 's/^/ /' "$TMP/ok.out" | tail -5; FAILS=$((FAILS+1)) +fi + +echo "目标版本: ${REV:-工作区}；失败 $FAILS 项" +[ "$FAILS" -eq 0 ] diff --git a/scripts/tests/test_env_example.py b/scripts/tests/test_env_example.py new file mode 100644 index 00000000..ae0d7c92 --- /dev/null +++ b/scripts/tests/test_env_example.py @@ -0,0 +1,722 @@ +"""根目录 env.example 与本分支代码行为的一致性检查（r3inc P3-7 / P3-8）。 + +只读取仓库里的 env.example、config/webhook.yaml、deploy/release/compose.yaml 和 Go 源码中的默认值 / 拼接格式， +不连网、不启动进程；群机器人 token 用运行时生成的合成值。 +""" +import json +import os +import random +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[2] +_DEFAULT_PORTS = {"http": 80, "https": 443} + + +_GODOTENV_SPACE = "\t\v\f\r \x85\xa0" # parser.go isSpace：不含换行 +_EXPAND_VAR = re.compile(r"(\\)?(\$)(\()?\{?([A-Z0-9_]+)?\}?") + + +def _go_unicode_isspace(ch): + # unicode.IsSpace：Latin-1 内只有这 8 个；Python 的 isspace 还把 \x1c-\x1f 算空白，要排除 + return ch in "\t\n\v\f\r \x85\xa0" or (ord(ch) > 0xff and ch.isspace()) + + +def _godotenv_expand(value, seen): + def repl(m): + sub = _EXPAND_VAR.search(m.group(0)) + if sub.group(1) == "\\" or sub.group(2) == "(": # 原实现比较的是第 2 组（恒为 "$"），这一支只会因反斜杠命中 + return m.group(0)[1:] + if sub.group(4): + return seen.get(sub.group(4), "") + return m.group(0) + return _EXPAND_VAR.sub(repl, value) + + +def _godotenv_escapes(value): + value = re.sub(r"\\.", lambda m: {"n": "\n", "r": "\r"}.get(m.group(0)[1], m.group(0)), value) + return re.sub(r"\\([^$])", r"\1", value) + + +def _godotenv_parse(text): + """逐条移植 github.com/joho/godotenv v1.5.1 parser.go（main.go 用 godotenv.Load() 读 .env）。 + + 重复键后者生效；可带 export 前缀；分隔符 = 或 :；未加引号的值从行尾往前找第一个前面是空白的 #， + 其后为注释（a#b 不是注释）；单引号原样，双引号展开 \\n \\r 和其他反斜杠转义； + 未加引号和双引号的值展开 $NAME / ${NAME}，只看前面已解析的键，找不到为空串。 + 格式错误时抛 ValueError（godotenv.Load 同样报错）。 + """ + src = text.replace("\r\n", "\n") + out = {} + while True: + # getStatementStart：跳过空白（含换行）和 # 注释行 + while True: + pos = next((i for i, ch in enumerate(src) if not _go_unicode_isspace(ch)), -1) + if pos == -1: + return out + src = src[pos:] + if src[0] != "#": + break + nl = src.find("\n") + if nl == -1: + return out + src = src[nl:] + # locateKeyName：键名按字节逐个检查（与 Go 的 for i, char := range []byte 一致） + src = src.lstrip(_GODOTENV_SPACE) + if src.startswith("export") and src[6:7] and src[6] in _GODOTENV_SPACE: + src = src[6:].lstrip(_GODOTENV_SPACE) + raw = src.encode("utf-8") + key, offset = b"", 0 + for i, c in enumerate(raw): + ch = chr(c) # rune(byte)：按 Latin-1 解释单个字节 + if ch in _GODOTENV_SPACE: + continue + if ch in "=:": + key, offset = raw[:i], i + 1 + break + if ch == "_" or ch == "." or ch.isalpha() or ch.isnumeric(): + continue + raise ValueError(f"unexpected character {ch!r} in variable name") + if not raw: + raise ValueError("zero length string") + key = key.decode("utf-8", "replace").rstrip() + src = raw[offset:].decode("utf-8").lstrip(_GODOTENV_SPACE) + # extractVarValue + if src[:1] in ('"', "'"): + quote = src[0] + for i in range(1, len(src)): + if src[i] != quote or src[i - 1] == "\\": + continue + value = src[:i].strip(quote) + if quote == '"': + value = _godotenv_expand(_godotenv_escapes(value), out) + out[key] = value + src = src[i + 1:] + break + else: + raise ValueError("unterminated quoted value") + continue + eol = next((i for i, ch in enumerate(src) if ch in "\n\r"), len(src)) + line, end = src[:eol], eol + for i in range(end - 1, 0, -1): + if line[i] == "#" and line[i - 1] in _GODOTENV_SPACE: + end = i + break + out[key] = _godotenv_expand(line[:end].strip(_GODOTENV_SPACE), out) + src = src[eol:] + + +def _env_example(text=None): + """env.example 复制成 .env 后 backend 实际读到的值（godotenv v1.5.1 语义，见 _godotenv_parse）""" + return _godotenv_parse((ROOT / "env.example").read_text(encoding="utf-8") if text is None else text) + + +def _ascii_port(url_or_netloc_port): + """URL 里显式写出的端口必须是 ASCII 数字：Python 3.9/3.10 的 urllib 会把全角数字 ８８８９ 当成 8889。""" + import re + if url_or_netloc_port is not None and not re.fullmatch(r"[0-9]+", url_or_netloc_port): + raise ValueError(f"端口只能是 ASCII 数字: {url_or_netloc_port!r}") + + +def _netloc_port(netloc): + # netloc 里冒号后的端口原文（IPv6 字面量跳过方括号）；没写端口返回 None + hostport = netloc.rpartition("@")[2] + tail = hostport.rpartition("]")[2] if hostport.startswith("[") else hostport + return tail.rpartition(":")[2] if ":" in tail else None + + +def _has_http_scheme_fold(base): + """对齐 w1 119c332 alert_cache.go 的 hasPrefixFold(base, "http://") || hasPrefixFold(base, "https://")： + 按字节长度切出前缀再 strings.EqualFold。前缀全是 ASCII，等价于只做 ASCII 大小写折叠； + ſ（U+017F）、全角字母等占的字节数不同，不会被认成 http(s)://。""" + b = base.encode("utf-8") + return b[:7].lower() == b"http://" or b[:8].lower() == b"https://" + + +def _split_alert_url(url): + """按 alert_cache.go buildWebhookURL 的规则拆地址（w1 119c332 起）：TrimSpace 后以 http:// 或 https:// 开头 + （不区分大小写，HTTP://、Https:// 也算）才是完整 URL，不含 :// 时按 host:port 走 http；其他带 :// 的写法 + （ftp://、httpſ:// 等）在代码里会拼出 http://ftp://... 的错误地址，这里直接报错。 + 注：合入 w1 前本分支 db97f42 的代码仍用区分大小写的 strings.HasPrefix。""" + base = url.strip() + if _has_http_scheme_fold(base): + parts = urlsplit(base) # urlsplit 会把 scheme 转成小写 + elif "://" in base: + raise ValueError(f"scheme 只能是 http/https（不区分大小写）: {base.split('://', 1)[0]!r}") + else: + parts = urlsplit("http://" + base) + _ascii_port(_netloc_port(parts.netloc)) + return parts + + +def _url_port(url): + parts = _split_alert_url(url) + scheme = parts.scheme + port = parts.port if parts.port is not None else _DEFAULT_PORTS[scheme] + if not 1 <= port <= 65535: + raise ValueError(f"端口越界: {port}") + return port + + +_RECEIVE_PATH_GO = "internal/prometheus/webhook/api/handler.go" +_HTTP_SD_GO = "internal/not_auth/api/handler.go" + + +def _gin_routes(rel, method): + """只读解析 Gin 路由： := .Group("<前缀>") 与 .("<子路径>", ...)，返回完整路径列表。""" + text = (ROOT / rel).read_text(encoding="utf-8") + groups = dict(re.findall(r'\b([A-Za-z_][A-Za-z0-9_]*)\s*:?=\s*[A-Za-z_][A-Za-z0-9_.]*\.Group\(\s*"([^"]*)"', text)) + routes = [groups[g] + sub for g, sub in re.findall(r'\b([A-Za-z_][A-Za-z0-9_]*)\.' + method + r'\(\s*"([^"]*)"', text) + if g in groups] + if not routes: + raise AssertionError(f"{rel} 里没解析到 {method} 路由") + return routes + + +def _alert_receive_route(): + routes = [r for r in _gin_routes(_RECEIVE_PATH_GO, "POST") if r.endswith("/receive")] + if len(routes) != 1: + raise AssertionError(f"webhook 告警接收路由应恰好 1 个，实际 {routes}") + return routes[0] + + +def _alert_target(addr): + """按 alert_cache.go buildWebhookURL 算出 Alertmanager 实际推送的 (scheme, host, port, path)。 + + 完整 URL：TrimRight "/"，已含 /api/v1/alerts/receive 就原样使用，否则在末尾追加；host:port：http:///api/v1/alerts/receive。 + """ + receive = "/api/v1/alerts/receive" + base = addr.strip() + parts = _split_alert_url(base) # 校验 scheme 与端口 + if "://" in base: + u = base.rstrip("/") + full = u if receive in u else u + receive + else: + full = "http://" + base + receive + parts = urlsplit(full) + return parts.scheme, parts.hostname, _url_port(full), parts.path + + +def _release_default_alert_addr(): + """发布栈里的告警地址期望值：compose 的 ${PROMETHEUS_ALERT_WEBHOOK_ADDR:-默认}；本分支 compose 没有时用 alertmanager.yml。""" + compose = (ROOT / "deploy/release/compose.yaml").read_text(encoding="utf-8") + m = re.search(r"PROMETHEUS_ALERT_WEBHOOK_ADDR:\s*\$\{PROMETHEUS_ALERT_WEBHOOK_ADDR:-([^}]+)\}", compose) + if m: + return m.group(1).strip() + found = re.findall(r"^\s*-?\s*url:\s*(\S+)\s*$", (ROOT / "deploy/release/alertmanager.yml").read_text(encoding="utf-8"), re.M) + if len(found) != 1: + raise AssertionError(f"alertmanager.yml 的 webhook url 应恰好 1 个，实际 {len(found)}") + return found[0] + + +def _go_default(key): + # internal/config/loader.go 里 .SetDefault("", "") 的值（只读解析）； + # 接收者不限于 viper：sched/w1-mantou 的 webhook 默认值写成 v.SetDefault("webhook.port", "8889") + text = (ROOT / "internal/config/loader.go").read_text(encoding="utf-8") + found = re.findall(r'\b[A-Za-z_][A-Za-z0-9_]*\.SetDefault\(\s*"' + re.escape(key) + r'",\s*"([^"]*)"\s*\)', text) + if len(found) != 1: + raise AssertionError(f"loader.go 中 {key} 默认值应恰好 1 处，实际 {len(found)}") + return found[0] + + +# webhook.port 的值：纯 ASCII 数字（可加单/双引号）；行内注释的 # 前必须有空白（YAML 规则，8889#oops 是一个标量） +_PORT_VALUE = r"""(?:"([0-9]+)"|'([0-9]+)'|([0-9]+))(?:[ \t]+#.*)?[ \t]*""" + + +def _webhook_yaml_port(text=None): + # config/webhook.yaml 的 webhook.port（webhook: 根节点下第一层的 port）；text 用于合成 YAML 用例 + if text is None: + text = (ROOT / "config/webhook.yaml").read_text(encoding="utf-8") + level = None + in_root = False + roots = 0 + ports = [] + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + indent = len(line) - len(line.lstrip(" ")) + if indent == 0: + if re.match(r"""(["'])webhook\1[ \t]*:""", line): + raise AssertionError(f"webhook 根键带引号，测试解析不支持，请写成 webhook: ：{line!r}") + in_root = re.match(r"webhook[ \t]*:", line) is not None + roots += in_root + if roots > 1: + raise AssertionError("config/webhook.yaml 的 webhook 根节点重复") + level = None + continue + if not in_root: + continue + if level is None: + level = indent + if indent == level: + if re.match(r"""(["'])port\1[ \t]*:""", line.strip()): + raise AssertionError(f"webhook.port 键名带引号，测试解析不支持，请写成 port: ：{line.strip()!r}") + m = re.fullmatch(r"port[ \t]*:[ \t]*" + _PORT_VALUE, line.strip()) + if m: + ports.append(int(next(g for g in m.groups() if g))) + elif re.match(r"port\s*:", line.strip()): + raise AssertionError(f"webhook.port 不是纯数字: {line.strip()!r}") + if len(ports) != 1: + raise AssertionError(f"config/webhook.yaml 的 webhook.port 应恰好 1 个，实际 {len(ports)}") + return ports[0] + + +class WebhookPortConsistencyTests(unittest.TestCase): + """P3-7：按 env.example 启动时 webhook 监听端口须与 PROMETHEUS_ALERT_WEBHOOK_ADDR 的端口一致。""" + + def test_declared_webhook_port_matches_alert_addr_port(self): + env = _env_example() + addr_port = _url_port(env["PROMETHEUS_ALERT_WEBHOOK_ADDR"]) + # WEBHOOK_PORT 留空时不能依赖配置文件：取代码默认（本分支 loader.go 的 webhook.port 默认值） + declared = env.get("WEBHOOK_PORT", "") or _go_default("webhook.port") + self.assertRegex(declared, r"^[1-9][0-9]{0,4}$") + self.assertEqual(int(declared), addr_port, + "env.example 的 webhook 端口（WEBHOOK_PORT，留空时为代码默认）与告警推送地址端口不一致") + + def test_repo_webhook_yaml_port_matches_alert_addr_port(self): + # 合入 w1 前 WEBHOOK_PORT 不起作用，从仓库根目录启动时实际端口来自默认 --config config/webhook.yaml； + # w1 之后 WEBHOOK_PORT 优先，env.example 里两者都是 8889，与告警推送地址端口一致 + self.assertEqual(_webhook_yaml_port(), _url_port(_env_example()["PROMETHEUS_ALERT_WEBHOOK_ADDR"])) + + def test_release_compose_default_alert_addr_port_matches(self): + compose = (ROOT / "deploy/release/compose.yaml").read_text(encoding="utf-8") + m = re.search(r"PROMETHEUS_ALERT_WEBHOOK_ADDR:\s*\$\{PROMETHEUS_ALERT_WEBHOOK_ADDR:-([^}]+)\}", compose) + if m is None: + self.skipTest("本分支发布 compose 未设置可覆盖的 PROMETHEUS_ALERT_WEBHOOK_ADDR（REL-03A 合入后才有）") + self.assertEqual(_url_port(m.group(1)), _webhook_yaml_port()) + + def test_url_port_boundaries(self): + self.assertEqual(_url_port("http://h/x"), 80) + self.assertEqual(_url_port("https://h/x"), 443) + self.assertEqual(_url_port(" https://h:9443/x \n"), 9443) + self.assertEqual(_url_port("h:8889"), 8889) + # w1 119c332 起 scheme 不区分大小写（alert_cache.go hasPrefixFold），大写写法按 scheme 取端口 + self.assertEqual(_url_port("HTTPS://h/x"), 443) + self.assertEqual(_url_port("HTTP://h:8889/x"), 8889) + self.assertEqual(_url_port("Http://h:8889/x"), 8889) + self.assertEqual(_url_port("hTTps://h:9443/x"), 9443) + # "httpſ://"（U+017F，与 s 大小写折叠）字节长度不同，hasPrefixFold 不认，代码会按 host:port 拼错地址 + for bad in ("ftp://h:21/x", "http://h:0/x", "http://h:8889suffix/x", "http://h:70000/x", + "http://h:８８８９/x", "h:８８８９", "httpſ://h:8889/x", "ＨTTP://h:8889/x"): + with self.subTest(bad=bad), self.assertRaises(ValueError): + _url_port(bad) + + +_GROUP_API_CALL = r'viper\.GetString\(\s*"webhook\.im_feishu\.group_message_api"\s*\)' +_GO_TRIMS = ("TrimSpace", "TrimRight", "TrimLeft", "TrimSuffix", "TrimPrefix", "Trim") + + +def _go_string(lit): + # 只接受不含转义的 Go 解释型字符串字面量；其他写法直接报错，避免静默误判 + m = re.fullmatch(r'"([^"\\]*)"', lit.strip()) + if m is None: + raise AssertionError(f"不支持的 Go 字符串字面量: {lit!r}") + return m.group(1) + + +def _go_call_args(text, open_paren): + """从 text[open_paren] 的 '(' 起，按顶层逗号切分实参；跳过字符串/字符/反引号字面量。返回 (实参列表, 右括号后位置)。""" + if text[open_paren] != "(": + raise AssertionError("期望 '('") + depth, i, start, args = 0, open_paren, open_paren + 1, [] + while i < len(text): + c = text[i] + if c in "\"'": + i += 1 + while text[i] != c: + i += 2 if text[i] == "\\" else 1 + elif c == "`": + i = text.index("`", i + 1) + elif c in "([{": + depth += 1 + elif c in ")]}": + depth -= 1 + if depth == 0: + args.append(text[start:i].strip()) + return [a for a in args if a], i + 1 + elif c == "," and depth == 1: + args.append(text[start:i].strip()) + start = i + 1 + i += 1 + raise AssertionError("Go 调用括号不配对") + + +def _group_base_transform(expr): + """把 Sprintf 第一个 %s 的实参解释成对 group_message_api 取值的变换（只认 viper.GetString 与 strings.Trim* 包装）。""" + expr = expr.strip() + if re.fullmatch(_GROUP_API_CALL, expr): + return lambda s: s + m = re.match(r"strings\.(" + "|".join(_GO_TRIMS) + r")\s*(?=\()", expr) + if m is None: + raise AssertionError(f"group_message_api 外层包装不认识，需补充测试解析: {expr!r}") + args, end = _go_call_args(expr, m.end()) + if expr[end:].strip(): + raise AssertionError(f"包装调用后还有多余内容: {expr!r}") + fn = m.group(1) + inner = _group_base_transform(args[0]) + if fn == "TrimSpace": + if len(args) != 1: + raise AssertionError(f"strings.TrimSpace 参数个数不对: {expr!r}") + return lambda s: inner(s).strip() + if len(args) != 2: + raise AssertionError(f"strings.{fn} 参数个数不对: {expr!r}") + cut = _go_string(args[1]) + ops = { + "TrimRight": lambda s: s.rstrip(cut) if cut else s, + "TrimLeft": lambda s: s.lstrip(cut) if cut else s, + "Trim": lambda s: s.strip(cut) if cut else s, + "TrimSuffix": lambda s: s[:-len(cut)] if cut and s.endswith(cut) else s, + "TrimPrefix": lambda s: s[len(cut):] if cut and s.startswith(cut) else s, + } + op = ops[fn] + return lambda s: op(inner(s)) + + +def _group_format(text): + """content.go 中用 group_message_api 拼群消息 URL 的 fmt.Sprintf：返回 (格式串, 对 base 的变换)。 + + 兼容：fmt.Sprintf("%s/%s", viper.GetString(...), robotToken)（本分支）、 + fmt.Sprintf("%s/%s", strings.TrimRight(viper.GetString(...), "/"), robotToken)（sched/w1-mantou 2026deb） + 以及再套一层 strings.TrimSpace(...) 等 strings.Trim* 包装。 + """ + calls = [] + for m in re.finditer(r"\bfmt\.Sprintf\s*(?=\()", text): + args, _ = _go_call_args(text, m.end()) + if any("group_message_api" in a for a in args): + calls.append(args) + if len(calls) != 1: + raise AssertionError(f"content.go 中群消息 URL 拼接应恰好 1 处，实际 {len(calls)}") + args = calls[0] + if len(args) != 3 or args[2] != "robotToken": + raise AssertionError(f"群消息 URL 拼接实参不是 (格式, base, robotToken): {args!r}") + fmt = _go_string(args[0]) + if fmt.count("%s") != 2 or fmt.replace("%s", "").count("%") != 0: + raise AssertionError(f"群消息 URL 格式串应恰好两个 %s: {fmt!r}") + return fmt, _group_base_transform(args[1]) + + +class GroupFormatParserTests(unittest.TestCase): + """_group_format 对几种 Go 写法的解析（合成代码片段）。""" + + BASE = "https://open.feishu.cn/open-apis/bot/v2/hook" + API = 'viper.GetString("webhook.im_feishu.group_message_api")' + + def _build(self, snippet, base, token="t"): + fmt, transform = _group_format(snippet) + return fmt % (transform(base), token) + + def test_old_form(self): + snippet = f'url := fmt.Sprintf("%s/%s", {self.API}, robotToken)' + self.assertEqual(self._build(snippet, self.BASE), self.BASE + "/t") + self.assertEqual(self._build(snippet, self.BASE + "/"), self.BASE + "//t") + + def test_w1_trim_right_form(self): + snippet = f'url := fmt.Sprintf("%s/%s", strings.TrimRight({self.API}, "/"), robotToken)' + for base in (self.BASE, self.BASE + "/", self.BASE + "///"): + with self.subTest(base=base): + self.assertEqual(self._build(snippet, base), self.BASE + "/t") + + def test_extra_outer_wrapper(self): + snippet = ('url := fmt.Sprintf(\n\t"%s/%s",\n\tstrings.TrimSpace(strings.TrimRight(\n\t\t' + + self.API + ', "/")),\n\trobotToken,\n)') + self.assertEqual(self._build(snippet, " " + self.BASE + "/"), self.BASE + "/t") + snippet = f'url := fmt.Sprintf("%s/%s", strings.TrimRight(strings.TrimSpace({self.API}), "/"), robotToken)' + self.assertEqual(self._build(snippet, self.BASE + "/ \t"), self.BASE + "/t") + + def test_unknown_or_ambiguous_forms_fail_loudly(self): + for snippet in ( + f'u := fmt.Sprintf("%s/%s", strings.ToLower({self.API}), robotToken)', + f'u := fmt.Sprintf("%s/%s", {self.API}, otherToken)', + f'u := fmt.Sprintf("%s/%s/%s", {self.API}, robotToken)', + 'u := fmt.Sprintf("%s/%s", base, robotToken)', + f'a := fmt.Sprintf("%s/%s", {self.API}, robotToken)\nb := fmt.Sprintf("%s/%s", {self.API}, robotToken)', + ): + with self.subTest(snippet=snippet), self.assertRaises(AssertionError): + _group_format(snippet) + + +class AlertAddrSchemeDocTests(unittest.TestCase): + """env.example 写明告警地址 scheme 建议小写、不区分大小写（w1 119c332 alert_cache.go hasPrefixFold）； + 不再写“必须小写 / 代码区分大小写”这类合入 w1 后会过时的说法。""" + + SCHEME_DOCS = ("env.example", "deploy/release/env.example", "deploy/release/compose.yaml", "docs/release/README.md") + # 旧文案（7b47940）原句片段；新文案里“合入前的旧代码区分大小写”是如实的历史说明，不算过时 + STALE = ("scheme 必须是小写", "（代码区分大小写", "代码按区分大小写的前缀判断，HTTP://", + "会被当成 host:port 拼出", "会被当成 host:port，拼出") + + def test_release_docs_drop_stale_case_sensitive_wording(self): + for rel in self.SCHEME_DOCS: + text = (ROOT / rel).read_text(encoding="utf-8") + with self.subTest(file=rel): + for stale in self.STALE: + self.assertFalse(stale in text, f"{rel} 仍有合入 w1 后过时的说法 {stale!r}") + self.assertTrue("不区分大小写" in text, f"{rel} 没写 scheme 不区分大小写") + + def test_env_example_recommends_lowercase_scheme_case_insensitive(self): + lines = (ROOT / "env.example").read_text(encoding="utf-8").splitlines() + idx = next(i for i, line in enumerate(lines) if line.startswith("PROMETHEUS_ALERT_WEBHOOK_ADDR=")) + block = [] + for line in reversed(lines[:idx]): + if not line.startswith("#"): + break + block.append(line) + text = "\n".join(block) + self.assertIn("scheme 建议写小写 http:// 或 https://，不区分大小写", text) + self.assertNotIn("必须是小写", text) + value = lines[idx].split("=", 1)[1].strip() + self.assertTrue(value.startswith(("http://", "https://")) or "://" not in value, value) + + +class WebhookPortDocTests(unittest.TestCase): + """发布文档必须说明端口覆盖优先级和 Compose 未传入 WEBHOOK_PORT。""" + + PORT_DOCS = ("docs/release/README.md", "deploy/release/compose.yaml") + + def test_release_docs_document_webhook_port_precedence(self): + for rel in self.PORT_DOCS: + text = (ROOT / rel).read_text(encoding="utf-8") + with self.subTest(file=rel): + self.assertFalse("对监听端口不生效" in text, f"{rel} 仍写 WEBHOOK_PORT 对监听端口不生效") + self.assertFalse("不合并环境变量覆盖" in text, f"{rel} 仍写 loader 不合并环境变量") + self.assertTrue("compose 没有传这个变量" in text, f"{rel} 没写 compose 没有传 WEBHOOK_PORT") + self.assertTrue("WEBHOOK_PORT > 配置文件 > 默认 8889" in text, f"{rel} 没写 WEBHOOK_PORT 优先级") + + +class WebhookYamlPortValueTests(unittest.TestCase): + """webhook.port 取值的正负向（合成 YAML）。""" + + def test_accepts_ascii_digits_and_spaced_comment(self): + for value in ("8889", '"8889"', "'8889'", "8889 # c", '"8889"\t# c', "8889 "): + with self.subTest(value=value): + self.assertEqual(_webhook_yaml_port(f"webhook:\n port: {value}\n"), 8889) + + def test_rejects_duplicate_root_and_quoted_keys(self): + for text in ('webhook:\n port: 8889\nwebhook:\n min_workers: 1\n', + 'webhook:\n port: 8889\nserver:\n port: 1\nwebhook:\n port: 8890\n', + 'webhook:\n port: 8889\n "port": 9999\n', + "webhook:\n 'port': 9999\n", + '"webhook":\n port: 9999\n'): + with self.subTest(text=text), self.assertRaises(AssertionError): + _webhook_yaml_port(text) + + def test_rejects_fullwidth_digits_and_unspaced_hash(self): + for value in ("８８８９", '"８８８９"', "8889#oops", '"8889"#oops', "'8889'#oops"): + with self.subTest(value=value), self.assertRaises(AssertionError): + _webhook_yaml_port(f"webhook:\n port: {value}\n") + + +class AlertAddrFullUrlTests(unittest.TestCase): + """r4 A-4：env.example 的告警地址整体比较（scheme、host、path、port），HTTP SD 端口与 SERVER_PORT 一致。 + + 期望值来源：scheme 与 path 取发布 compose 默认值（本分支没有时取 deploy/release/alertmanager.yml），path 还须等于 + webhook 路由 internal/prometheus/webhook/api/handler.go 的 POST 接收路由；host 取代码默认 + prometheus.alert_webhook_addr（本机开发，localhost）；port 取 WEBHOOK_PORT（留空为代码默认）并与 config/webhook.yaml 一致。 + """ + + def _expected(self): + env = _env_example() + route = _alert_receive_route() + rel_scheme, _, _, rel_path = _alert_target(_release_default_alert_addr()) + code_scheme, code_host, _, code_path = _alert_target(_go_default("prometheus.alert_webhook_addr")) + self.assertEqual(rel_path, route, "发布默认告警地址的路径与 webhook 接收路由不一致") + self.assertEqual(code_path, route, "代码默认告警地址的路径与 webhook 接收路由不一致") + self.assertEqual(rel_scheme, code_scheme) + port = int(env.get("WEBHOOK_PORT", "") or _go_default("webhook.port")) + self.assertEqual(port, _webhook_yaml_port(), "WEBHOOK_PORT 与 config/webhook.yaml 的 webhook.port 不一致") + return rel_scheme, code_host, port, route + + def test_alert_addr_full_url(self): + addr = _env_example()["PROMETHEUS_ALERT_WEBHOOK_ADDR"] + scheme, host, port, path = _alert_target(addr) + e_scheme, e_host, e_port, e_path = self._expected() + self.assertEqual(scheme, e_scheme, "告警地址 scheme 不对") + self.assertEqual(host, e_host, "告警地址 host 不对（本机开发模板应为代码默认的 localhost）") + self.assertEqual(path, e_path, "告警地址实际推送路径与 webhook 接收路由不一致") + self.assertEqual(port, e_port, "告警地址端口与 webhook 监听端口不一致") + self.assertEqual(f"{scheme}://{host}:{port}{path}", f"{e_scheme}://{e_host}:{e_port}{e_path}") + + def test_http_sd_api_port_matches_server_port(self): + env = _env_example() + sd = env["PROMETHEUS_HTTP_SD_API"] + server_port = env.get("SERVER_PORT", "") or _go_default("server.port") + self.assertRegex(server_port, r"^[1-9][0-9]{0,4}$") + self.assertEqual(_url_port(sd), int(server_port), "PROMETHEUS_HTTP_SD_API 端口与 SERVER_PORT 不一致") + parts = _split_alert_url(sd) + self.assertEqual(parts.scheme, "http") + self.assertIn(parts.path, _gin_routes(_HTTP_SD_GO, "GET"), "PROMETHEUS_HTTP_SD_API 路径不是主后端的 HTTP SD 路由") + + def test_alert_target_follows_go_builder(self): + self.assertEqual(_alert_target("http://h:1/api/v1/alerts/receive/"), ("http", "h", 1, "/api/v1/alerts/receive")) + self.assertEqual(_alert_target("http://h:1"), ("http", "h", 1, "/api/v1/alerts/receive")) + self.assertEqual(_alert_target("h:2"), ("http", "h", 2, "/api/v1/alerts/receive")) + self.assertEqual(_alert_target("https://h/x"), ("https", "h", 443, "/x/api/v1/alerts/receive")) + # w1 119c332：大写 scheme 也是完整 URL（不再拼成 http://HTTP://...） + self.assertEqual(_alert_target("HTTP://h:1/api/v1/alerts/receive"), ("http", "h", 1, "/api/v1/alerts/receive")) + self.assertEqual(_alert_target(" Https://h/x/ "), ("https", "h", 443, "/x/api/v1/alerts/receive")) + with self.assertRaises(ValueError): + _alert_target("httpſ://h:1/api/v1/alerts/receive") + + +_GODOTENV_CASES = [ + # (内容, godotenv v1.5.1 解析结果)；合成值，与仓库 / 主机上的真实配置无关 + ("export FOO=1\n", {"FOO": "1"}), + ("FOO = 1\nBAR: two\n", {"FOO": "1", "BAR": "two"}), + ("FOO=a # 注释\nBAR=a#b\nBAZ=a #b #c\n", {"FOO": "a", "BAR": "a#b", "BAZ": "a #b"}), + ("FOO=\"a b\" # 注释\nBAR='x # y'\n", {"FOO": "a b", "BAR": "x # y"}), + ("FOO=\"l1\\nl2\"\nBAR='raw\\n'\n", {"FOO": "l1\nl2", "BAR": "raw\\n"}), + ("A=1\nB=${A}x\nC=$A\nD=\\$A\nE=${MISSING}\nF='$A'\n", {"A": "1", "B": "1x", "C": "1", "D": "$A", "E": "", "F": "$A"}), + ("X=${LATER}\nLATER=v\n", {"X": "", "LATER": "v"}), + ("A=1\nA=2\n", {"A": "2"}), + ("lower.case=1\n", {"lower.case": "1"}), + (" # 注释\n\tFOO=1 \r\nBAR=\n", {"FOO": "1", "BAR": ""}), +] + + +class GodotenvParserTests(unittest.TestCase): + """_env_example 与 backend 读 .env 的方式一致：main.go 调用 godotenv.Load()（go.mod 锁定 v1.5.1）""" + + def test_semantics(self): + for text, expected in _GODOTENV_CASES: + with self.subTest(text=text): + self.assertEqual(_env_example(text), expected) + + def test_rejects_malformed(self): + for text in ("FOO\n", 'FOO="unterminated\n', "FOO-BAR=1\n"): + with self.subTest(text=text), self.assertRaises(ValueError): + _env_example(text) + + def test_env_example_parses(self): + self.assertIn("PROMETHEUS_ALERT_WEBHOOK_ADDR", _env_example()) + + def test_matches_real_godotenv(self): + # 可选交叉核对：只用本机模块缓存（GOPROXY=off、不切换工具链、不改 go.mod）；缺 go 或缓存时跳过 + go = shutil.which("go") + if go is None: + self.skipTest("go 不可用，跳过与真实 godotenv 的交叉核对") + sums = [l for l in (ROOT / "go.sum").read_text(encoding="utf-8").splitlines() + if l.startswith("github.com/joho/godotenv v1.5.1")] + if len(sums) != 2: + self.skipTest("go.sum 里没有 godotenv v1.5.1") + rnd = random.Random(20261024) + alphabet = ["A", "B", "_", "x", "1", "=", ":", " ", "\t", "#", "\n", "\r", "\r\n", '"', "'", "\\", "$", + "{", "}", "(", "export ", "\\n", "${A}", "$B", "é", "\xa0", "\x85", "\u3000", ".", "-", "１"] + cases = [t for t, _ in _GODOTENV_CASES] + [(ROOT / "env.example").read_text(encoding="utf-8")] + cases += ["".join(rnd.choice(alphabet) for _ in range(rnd.randint(0, 24))) for _ in range(3000)] + with tempfile.TemporaryDirectory() as tmp: + mod = Path(tmp) / "godotenvcheck" + mod.mkdir() + (mod / "go.mod").write_text("module godotenvcheck\n\ngo 1.22\n\nrequire github.com/joho/godotenv v1.5.1\n") + (mod / "go.sum").write_text("\n".join(sums) + "\n") + (mod / "main.go").write_text(_GODOTENV_GO) + env = {**os.environ, "GOTOOLCHAIN": "local", "GOPROXY": "off", "GOSUMDB": "off", "GOFLAGS": "", + "GOCACHE": os.environ.get("GOCACHE") or str(Path(tmp) / "gocache")} + r = subprocess.run([go, "run", "."], cwd=mod, input=json.dumps(cases), env=env, + capture_output=True, text=True, timeout=300) + if r.returncode != 0: + self.skipTest("本机模块缓存缺少 github.com/joho/godotenv v1.5.1，跳过交叉核对") + mismatches = [] + for text, got in zip(cases, json.loads(r.stdout)): + want = {"err": True} if got.get("err") else {"ok": got.get("ok") or {}} + try: + mine = {"ok": _env_example(text)} + except ValueError: + mine = {"err": True} + if mine != want: + mismatches.append(text) + self.assertEqual(mismatches[:5], [], f"{len(mismatches)} / {len(cases)} 条与 godotenv 结果不同") + + +_GODOTENV_GO = """package main + +import ( + "encoding/json" + "os" + "strings" + + "github.com/joho/godotenv" +) + +type result struct { + OK map[string]string `json:"ok,omitempty"` + Err bool `json:"err,omitempty"` +} + +func main() { + var cases []string + if err := json.NewDecoder(os.Stdin).Decode(&cases); err != nil { + panic(err) + } + out := make([]result, 0, len(cases)) + for _, c := range cases { + m, err := godotenv.Parse(strings.NewReader(c)) + if err != nil { + out = append(out, result{Err: true}) + continue + } + out = append(out, result{OK: m}) + } + _ = json.NewEncoder(os.Stdout).Encode(out) +} +""" + + +class FeishuGroupMessageUrlTests(unittest.TestCase): + """P3-8：WEBHOOK_IM_FEISHU_GROUP_MESSAGE_API 按 content.go SentFeishuGroup 的拼接方式须得到群机器人 webhook 地址。""" + + CONTENT_GO = "internal/prometheus/webhook/content/content.go" + EVENT_GO = "internal/prometheus/service/alert/event_service.go" + + def _build(self, base, token): + fmt, transform = _group_format((ROOT / self.CONTENT_GO).read_text(encoding="utf-8")) + return fmt % (transform(base), token) + + def _strips_trailing_slash(self): + _, transform = _group_format((ROOT / self.CONTENT_GO).read_text(encoding="utf-8")) + return transform("x/") == "x" + + def _reference_format(self): + # 后端 event_service.go 用同一个群机器人 token 直接拼出的地址，作为期望值来源 + text = (ROOT / self.EVENT_GO).read_text(encoding="utf-8") + m = re.findall(r'fmt\.Sprintf\("(https://[^"]*/%s)",\s*sendGroup\.FeiShuQunRobotToken\)', text) + self.assertEqual(len(m), 1) + return m[0] + + def test_group_message_url_from_env_example(self): + token = f"r4-synthetic-{os.getpid()}" + base = _env_example()["WEBHOOK_IM_FEISHU_GROUP_MESSAGE_API"] + url = self._build(base, token) + expected = self._reference_format() % token + parts = urlsplit(url) + self.assertNotIn("//", parts.path, f"拼出的路径有重复斜杠: {url}") + self.assertEqual(url, expected, f"拼出的群消息地址不对: {url}") + self.assertEqual(parts.path.rsplit("/", 1)[-1], token) + + def test_wrong_group_message_bases_are_detected(self): + # 负向：旧示例值（im/v1/messages）、缺 hook 段在任何写法下都拼不出正确地址 + token = f"r4-synthetic-{os.getpid()}" + expected = self._reference_format() % token + for base in ("https://open.feishu.cn/open-apis/im/v1/messages", + "https://open.feishu.cn/open-apis/bot/v2"): + with self.subTest(base=base): + self.assertNotEqual(self._build(base, token), expected) + + def test_trailing_slash_base_follows_code_semantics(self): + # 带尾斜杠（config/webhook.yaml 现值）：本分支原样拼接得到 hook//（错误）； + # sched/w1-mantou 的 strings.TrimRight(..., "/") 会去掉尾斜杠，得到正确地址 + token = f"r4-synthetic-{os.getpid()}" + expected = self._reference_format() % token + url = self._build("https://open.feishu.cn/open-apis/bot/v2/hook/", token) + if self._strips_trailing_slash(): + self.assertEqual(url, expected) + else: + self.assertNotEqual(url, expected) + self.assertIn("//", urlsplit(url).path) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_release.py b/scripts/tests/test_release.py index c69ac424..1c15c478 100644 --- a/scripts/tests/test_release.py +++ b/scripts/tests/test_release.py @@ -137,5 +137,463 @@ def test_crash_smoke_guards_do_not_call_docker(self): self.assertIn(message, result.stdout + result.stderr) +# 告警地址只允许 http/https；没写端口时按 scheme 取默认端口 +_DEFAULT_PORTS = {"http": 80, "https": 443} + + +def _ascii_port(url_or_netloc_port): + """URL 里显式写出的端口必须是 ASCII 数字：Python 3.9/3.10 的 urllib 会把全角数字 ８８８９ 当成 8889。""" + import re + if url_or_netloc_port is not None and not re.fullmatch(r"[0-9]+", url_or_netloc_port): + raise ValueError(f"端口只能是 ASCII 数字: {url_or_netloc_port!r}") + + +def _netloc_port(netloc): + # netloc 里冒号后的端口原文（IPv6 字面量跳过方括号）；没写端口返回 None + hostport = netloc.rpartition("@")[2] + tail = hostport.rpartition("]")[2] if hostport.startswith("[") else hostport + return tail.rpartition(":")[2] if ":" in tail else None + + +def _alert_addr_target(addr): + """解析 PROMETHEUS_ALERT_WEBHOOK_ADDR，返回 (主机名, 端口, 路径)。 + + 对齐 alert_cache.go buildWebhookURL（w1 119c332 起）：先 TrimSpace；以 http:// 或 https:// 开头 + （hasPrefixFold，不区分大小写，HTTPS:// 也算）才是完整 URL，不含 :// 时按 host:port 走 http。 + 其他带 :// 的写法（ftp://、httpſ:// 等，hasPrefixFold 按字节切前缀，只折叠 ASCII）在代码里会拼成 + http://ftp://... 的错误地址，这里抛 ValueError；端口不是 1-65535 的 ASCII 数字（如 8889suffix、８８８９）同样抛 ValueError。 + 合入 w1 前本分支 db97f42 的代码仍区分大小写，所以文档统一建议写小写。 + """ + from urllib.parse import urlsplit + base = addr.strip() + head = base.encode("utf-8") + if head[:7].lower() == b"http://" or head[:8].lower() == b"https://": + url = urlsplit(base) # urlsplit 会把 scheme 转成小写 + elif "://" in base: + raise ValueError(f"告警地址 scheme 只能是 http/https（不区分大小写），实际 {base.split('://', 1)[0]!r}") + else: + url = urlsplit("http://" + base) + _ascii_port(_netloc_port(url.netloc)) + scheme = url.scheme + port = url.port + if port is None: + port = _DEFAULT_PORTS[scheme] + if not 1 <= port <= 65535: + raise ValueError(f"告警地址端口超出 1-65535: {port}") + return url.hostname, port, url.path + + +def _webhook_root_port(text): + """从 webhook.yaml 文本取 webhook 根节点下的 port，没有时返回 None。 + + 不依赖固定缩进：以 webhook: 下第一个子键的缩进为准，只认这一层的 port（更深层的嵌套键不算）。 + webhook 根节点重复、根键或 port 键名带引号（"webhook": / "port":）也抛 ValueError，不静默漏掉。 + 值只能是 ASCII 纯数字（可加单/双引号，后面只能跟空白或“空白 + # 注释”；8889#oops、全角 ８８８９ 不算）， + 8889suffix 这类写法、重复的 port、 + 超出 1-65535 都抛 ValueError。不依赖 PyYAML，macOS 自带 python3 也能跑。 + """ + import re + ports, in_root, child, roots = [], False, None, 0 + for raw in text.splitlines(): + body = raw.lstrip(" \t") + if not body or body.startswith("#"): + continue + indent = len(raw) - len(body) + if indent == 0: # 顶层键 + if re.match(r"""(["'])webhook\1[ \t]*:""", raw): + raise ValueError(f"webhook 根键带引号，测试解析不支持，请写成 webhook: ：{raw!r}") + in_root = re.match(r"webhook[ \t]*:[ \t]*(#.*)?$", raw) is not None + roots += in_root + if roots > 1: + # YAML 里重复的映射键：yaml.v3（viper）直接报错，其他解析器取后一个，都不能只看第一个 + raise ValueError("webhook.yaml 的 webhook 根节点重复") + child = None + continue + if not in_root: + continue + if "\t" in raw[:indent]: + raise ValueError("webhook.yaml 缩进里有 Tab") + if child is None: + child = indent + if indent == child and re.match(r"""(["'])port\1[ \t]*:""", body): + raise ValueError(f"webhook.port 键名带引号，测试解析不支持，请写成 port: ：{body!r}") + if indent != child or not re.match(r"port[ \t]*:", body): + continue + m = re.fullmatch(r"""port[ \t]*:[ \t]*(?:"([0-9]+)"|'([0-9]+)'|([0-9]+))(?:[ \t]+#.*)?[ \t]*""", body) + if not m: + raise ValueError(f"webhook.port 不是纯数字: {body!r}") + ports.append(int(next(g for g in m.groups() if g))) + if len(ports) > 1: + raise ValueError(f"webhook.port 重复 {len(ports)} 次") + if not ports: + return None + if not 1 <= ports[0] <= 65535: + raise ValueError(f"webhook.port 超出 1-65535: {ports[0]}") + return ports[0] + + +def _compose_env(home): + """compose 测试的隔离环境，探测（_compose_tools）和执行（ReleaseComposeWebhookAddrTests）共用。 + + 只保留 PATH，HOME 指向临时目录，不继承调用者导出的其他变量（避免覆盖合成值）。 + DOCKER_CONFIG 透传；未设置时指向隔离前的 ~/.docker：docker CLI 在 $DOCKER_CONFIG/cli-plugins + （默认 ~/.docker/cli-plugins）里找 compose 插件，Mac 上 OrbStack / Docker Desktop 就装在这里， + 只换 HOME 会让执行时找不到插件（退出码 125）。 + """ + real_home = os.environ.get("HOME") or os.path.expanduser("~") + docker_config = os.environ.get("DOCKER_CONFIG") or os.path.join(real_home, ".docker") + return {"PATH": os.environ.get("PATH", ""), "HOME": str(home), "DOCKER_CONFIG": docker_config} + + +def _compose_tools(): + # 在导入阶段调用：任何探测失败（超时、无法执行）都只让相关用例 skip，不能让模块导入报错 + if not (shutil.which("docker") and shutil.which("jq")): + return False + try: + with tempfile.TemporaryDirectory() as home: + return subprocess.run(["docker", "compose", "version"], env=_compose_env(home), + capture_output=True, timeout=30).returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +_COMPOSE_OK = _compose_tools() +_COMPOSE_MISSING = "需要 docker compose 和 jq（只做 compose config，不连 Docker 守护进程、不启动服务）" + + +def _compose_required(): + """CI 里（CI=true/1，GitHub Actions、GitLab CI 都会设置）或显式 RELEASE_TESTS_REQUIRE_COMPOSE=1 时，缺 compose 算失败。""" + return (os.environ.get("CI", "").strip().lower() in ("true", "1") + or os.environ.get("RELEASE_TESTS_REQUIRE_COMPOSE", "").strip() == "1") + + +class ReleaseComposeWebhookAddrTests(unittest.TestCase): + """backend 的 PROMETHEUS_ALERT_WEBHOOK_ADDR 必须指向 webhook 服务、端口与 webhook 实际监听端口一致""" + + COMPOSE = ROOT / "deploy/release/compose.yaml" + + def setUp(self): + if not _COMPOSE_OK: + if _compose_required(): + self.fail(_COMPOSE_MISSING + "；CI=true 或 RELEASE_TESTS_REQUIRE_COMPOSE=1 时不允许跳过") + self.skipTest(_COMPOSE_MISSING + "（本地跳过；设 RELEASE_TESTS_REQUIRE_COMPOSE=1 可强制执行）") + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + # 合成 .env：compose 中所有 ${X:?…} 必填变量都给运行时拼出的占位值，不读仓库或主机上的真实 .env + import re + names = sorted(set(re.findall(r"\$\{([A-Z0-9_]+):\?", self.COMPOSE.read_text()))) + self.env_file = Path(self.temp.name) / "synthetic.env" + self.env_file.write_text("".join(f"{n}={'fixture'}-{n.lower()}-{os.getpid()}\n" for n in names)) + # 干净环境：不继承调用者导出的变量，避免覆盖合成值；与探测用同一套隔离规则（见 _compose_env） + self.env = _compose_env(self.temp.name) + self.base = ["docker", "compose", "--project-directory", str(self.COMPOSE.parent), + "-f", str(self.COMPOSE), "--env-file", str(self.env_file)] + + def _jq(self, expr): + # 只经 jq 取指定字段，完整 config 输出不进入测试日志 + config = subprocess.run(self.base + ["config", "--format", "json"], env=self.env, + capture_output=True, timeout=60) + self.assertEqual(config.returncode, 0, "compose config 失败（输出已省略）") + result = subprocess.run(["jq", "-r", expr], input=config.stdout, capture_output=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr.decode(errors="replace")) + return result.stdout.decode().strip() + + def test_compose_config_quiet(self): + result = subprocess.run(self.base + ["config", "--quiet"], env=self.env, capture_output=True, timeout=60) + self.assertEqual(result.returncode, 0, "compose config --quiet 失败（输出已省略）") + + DEFAULT_ALERT_ADDR = "http://webhook:8889/api/v1/alerts/receive" + ALERT_ADDR_JQ = '.services.backend.environment.PROMETHEUS_ALERT_WEBHOOK_ADDR // ""' + + def test_alert_webhook_addr_default_when_unset_or_empty(self): + # r3inc P3-4：compose 写成 ${PROMETHEUS_ALERT_WEBHOOK_ADDR:-默认}；未设置和设为空串都取默认 + self.assertEqual(self._jq(self.ALERT_ADDR_JQ), self.DEFAULT_ALERT_ADDR) + with self.env_file.open("a") as f: + f.write("PROMETHEUS_ALERT_WEBHOOK_ADDR=\n") + self.assertEqual(self._jq(self.ALERT_ADDR_JQ), self.DEFAULT_ALERT_ADDR) + + def test_alert_webhook_addr_override_from_env_file(self): + # Compose 网络外的 Alertmanager：在 .env 里覆盖（合成地址，.test 保留域名） + custom = f"https://alerts-{os.getpid()}.example.test:9443/api/v1/alerts/receive" + with self.env_file.open("a") as f: + f.write(f"PROMETHEUS_ALERT_WEBHOOK_ADDR={custom}\n") + self.assertEqual(self._jq(self.ALERT_ADDR_JQ), custom) + self.assertEqual(_alert_addr_target(custom)[1:], (9443, "/api/v1/alerts/receive")) + + def test_alert_webhook_addr_override_from_shell_env(self): + # shell 里导出的同名变量同样生效（优先于 .env）：发布前要确认没有误带本机开发用的 localhost 值 + custom = f"http://198.51.100.{os.getpid() % 250 + 1}:18889/api/v1/alerts/receive" + self.env["PROMETHEUS_ALERT_WEBHOOK_ADDR"] = custom + self.assertEqual(self._jq(self.ALERT_ADDR_JQ), custom) + + def test_shell_value_beats_env_file(self): + # shell 与 .env 同时设置：shell 优先（只经 jq 取这一个字段） + from_file = f"http://env-file-{os.getpid()}.example.test:18889/api/v1/alerts/receive" + from_shell = f"http://shell-{os.getpid()}.example.test:28889/api/v1/alerts/receive" + with self.env_file.open("a") as f: + f.write(f"PROMETHEUS_ALERT_WEBHOOK_ADDR={from_file}\n") + self.env["PROMETHEUS_ALERT_WEBHOOK_ADDR"] = from_shell + self.assertEqual(self._jq(self.ALERT_ADDR_JQ), from_shell) + + def test_empty_shell_value_beats_env_file_then_default(self): + # shell 里导出空串、.env 里有值：空串照样压过 .env，再由 :- 回落到 compose 默认值（不是 .env 的值） + from_file = f"http://env-file-{os.getpid()}.example.test:18889/api/v1/alerts/receive" + with self.env_file.open("a") as f: + f.write(f"PROMETHEUS_ALERT_WEBHOOK_ADDR={from_file}\n") + self.env["PROMETHEUS_ALERT_WEBHOOK_ADDR"] = "" + self.assertEqual(self._jq(self.ALERT_ADDR_JQ), self.DEFAULT_ALERT_ADDR) + + def test_backend_alert_webhook_addr_targets_webhook_port(self): + addr = self._jq('.services.backend.environment.PROMETHEUS_ALERT_WEBHOOK_ADDR // ""') + self.assertTrue(addr, "backend 未设置 PROMETHEUS_ALERT_WEBHOOK_ADDR（代码默认 localhost:8889，Alertmanager 推送时指向它自己所在的容器或主机）") + host, port, path = _alert_addr_target(addr) + self.assertEqual(host, "webhook") + self.assertEqual(self._jq('.services.webhook | type'), "object", "compose 里没有 webhook 服务") + self.assertEqual(self._jq('.services.backend.network_mode // ""'), "", "backend 用了 network_mode，服务名不可解析") + self.assertEqual(self._jq('.services.webhook.environment.WEBHOOK_PORT // ""'), "", + "webhook 设了 WEBHOOK_PORT，需同步检查本测试") + # webhook 实际端口：镜像内 config/webhook.yaml 的 webhook.port + listen = _webhook_root_port((ROOT / "config/webhook.yaml").read_text()) + self.assertIsNotNone(listen, "config/webhook.yaml 里没找到 webhook.port（必须在 webhook: 根节点下）") + self.assertEqual(port, listen) + self.assertIn(path.rstrip("/"), ("", "/api/v1/alerts/receive")) + # 与 webhook 健康检查、alertmanager.yml 的端口一致 + health = self._jq('.services.webhook.healthcheck.test | join(" ")') + self.assertIn(f"127.0.0.1:{port}/", health) + self.assertIn(f"http://webhook:{port}/api/v1/alerts/receive", + (ROOT / "deploy/release/alertmanager.yml").read_text()) + +class AlertAddrLocalhostWordingTests(unittest.TestCase): + """告警地址由 Alertmanager 按它推送：localhost 指向 Alertmanager 所在的容器或主机，不是 backend。文档不能写反。""" + + FILES = ("docs/release/README.md", "deploy/release/compose.yaml", "env.example", "scripts/tests/test_release.py") + WRONG = "指向 backend " + "自己" # 拆开写，避免本文件自身命中 + + def test_no_points_to_backend_itself(self): + for rel in self.FILES: + with self.subTest(file=rel): + self.assertNotIn(self.WRONG, (ROOT / rel).read_text(encoding="utf-8")) + + def test_release_readme_says_alertmanager_host(self): + text = (ROOT / "docs/release/README.md").read_text(encoding="utf-8") + line = next(l for l in text.splitlines() if "PROMETHEUS_ALERT_WEBHOOK_ADDR" in l and "localhost:8889" in l) + self.assertIn("指向 Alertmanager 所在的容器或主机", line) + + +class AlertAddrBoundaryTests(unittest.TestCase): + """告警地址解析边界（r3inc P3-3）：scheme 白名单、按 scheme 取默认端口。不需要 Docker""" + + def test_accepts_http_https_and_host_port(self): + self.assertEqual(_alert_addr_target("http://webhook:8889/api/v1/alerts/receive"), + ("webhook", 8889, "/api/v1/alerts/receive")) + self.assertEqual(_alert_addr_target("https://webhook:9443/x")[1], 9443) + self.assertEqual(_alert_addr_target(" http://webhook:8889/x\n"), ("webhook", 8889, "/x")) + self.assertEqual(_alert_addr_target("webhook:8889"), ("webhook", 8889, "")) + + def test_rejects_non_http_scheme(self): + for addr in ("ftp://webhook:8889/api/v1/alerts/receive", "ws://webhook:8889/api/v1/alerts/receive", + "file://webhook:8889/api/v1/alerts/receive", "unix://webhook:8889/x"): + with self.subTest(addr=addr), self.assertRaises(ValueError): + _alert_addr_target(addr) + + def test_accepts_uppercase_scheme(self): + # w1 119c332 起 alert_cache.go 用 hasPrefixFold，scheme 不区分大小写：大写写法也是完整 URL + for addr, want in (("HTTPS://webhook:9443/x", ("webhook", 9443, "/x")), + ("HTTP://webhook:8889/api/v1/alerts/receive", ("webhook", 8889, "/api/v1/alerts/receive")), + ("Http://webhook:8889/api/v1/alerts/receive", ("webhook", 8889, "/api/v1/alerts/receive")), + ("hTTps://webhook/x", ("webhook", 443, "/x"))): + with self.subTest(addr=addr): + self.assertEqual(_alert_addr_target(addr), want) + + def test_rejects_non_ascii_lookalike_scheme(self): + # hasPrefixFold 按字节长度切前缀：ſ（U+017F）、全角 Ｈ 不会被认成 http(s)://，代码会按 host:port 拼错地址 + for addr in ("httpſ://webhook:8889/x", "ＨTTP://webhook:8889/x"): + with self.subTest(addr=addr), self.assertRaises(ValueError): + _alert_addr_target(addr) + + def test_default_port_follows_scheme(self): + self.assertEqual(_alert_addr_target("http://webhook/api/v1/alerts/receive")[1], 80) + self.assertEqual(_alert_addr_target("https://webhook/api/v1/alerts/receive")[1], 443) + + def test_https_without_port_is_not_80(self): + # 负向：https 不写端口时不能当成 80，否则 webhook 监听 80 时会被误判为一致 + self.assertNotEqual(_alert_addr_target("https://webhook/api/v1/alerts/receive")[1], 80) + + def test_rejects_bad_port(self): + for addr in ("http://webhook:8889suffix/x", "http://webhook:0/x", "http://webhook:65536/x", + "http://webhook:８８８９/x", "webhook:８８８９"): + with self.subTest(addr=addr), self.assertRaises(ValueError): + _alert_addr_target(addr) + + +class WebhookYamlPortBoundaryTests(unittest.TestCase): + """webhook.yaml 根节点端口解析边界（r3inc P3-3）。全部是运行时拼出的合成 YAML，不读主机文件""" + + def test_two_space_indent(self): + text = 'webhook:\n front_domain: "x"\n port: "8889" # HTTP 服务端口\n' + self.assertEqual(_webhook_root_port(text), 8889) + + def test_quote_styles_and_comment(self): + for value in ("8889", '"8889"', "'8889'", '"8889" # c', "8889\t# c"): + with self.subTest(value=value): + self.assertEqual(_webhook_root_port(f"webhook:\n port: {value}\n"), 8889) + + def test_four_space_indent(self): + # 负向：缩进不是两个空格时也必须找到 webhook.port + text = 'webhook:\n im_feishu:\n port: "1"\n port: "9999"\n' + self.assertEqual(_webhook_root_port(text), 9999) + + def test_nested_port_is_not_root_port(self): + text = 'webhook:\n im_feishu:\n port: "1"\n alert_manager_api: "x"\n' + self.assertIsNone(_webhook_root_port(text)) + + def test_port_outside_webhook_root_ignored(self): + text = 'server:\n port: "1234"\nwebhook:\n min_workers: 1\n' + self.assertIsNone(_webhook_root_port(text)) + + def test_rejects_suffix_after_digits(self): + # 负向：8889suffix 不能被截成 8889 + for value in ("8889suffix", '"8889"suffix', "8889 suffix", '"8889suffix"', "88 89", + "８８８９", '"８８８９"', "8889#oops", '"8889"#oops', "'8889'#oops"): + with self.subTest(value=value), self.assertRaises(ValueError): + _webhook_root_port(f"webhook:\n port: {value}\n") + + def test_rejects_duplicate_root(self): + # 负向：第二个 webhook 根里没有 port 时，旧实现只看到第一个根的 8889 + for text in ('webhook:\n port: "8889"\nwebhook:\n min_workers: 1\n', + 'webhook:\n port: "8889"\nserver:\n port: 1\nwebhook:\n front_domain: x\n', + 'webhook: # c\n port: 8889\nwebhook:\n port: 8889\n'): + with self.subTest(text=text), self.assertRaises(ValueError): + _webhook_root_port(text) + + def test_rejects_quoted_keys(self): + # 负向：带引号的键名在 YAML 里与 port 等价，旧实现会跳过它 + for text in ('webhook:\n "port": "9999"\n', "webhook:\n 'port': 9999\n", + 'webhook:\n port: 8889\n "port": 9999\n', '"webhook":\n port: 9999\n'): + with self.subTest(text=text), self.assertRaises(ValueError): + _webhook_root_port(text) + + def test_rejects_tab_indent(self): + with self.assertRaises(ValueError): + _webhook_root_port("webhook:\n\tport: 8889\n") + + def test_rejects_duplicate_and_out_of_range(self): + for text in ('webhook:\n port: "8889"\n port: "8890"\n', "webhook:\n port: 0\n", "webhook:\n port: 65536\n"): + with self.subTest(text=text), self.assertRaises(ValueError): + _webhook_root_port(text) + + +class ComposeProbeEnvTests(unittest.TestCase): + """P2-1 回归：compose 探测与执行必须用同一个隔离环境。 + + 模拟 Mac（OrbStack）布局：compose 插件只在 $HOME/.docker/cli-plugins，系统目录里没有。 + 假 docker 按真实 CLI 的规则只在 ${DOCKER_CONFIG:-$HOME/.docker}/cli-plugins 找插件，找不到时退出 125。 + 全部用运行时生成的 /bin/sh 替身，不需要真实 Docker，也不读取主机的 ~/.docker。 + """ + + def setUp(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + self.base = Path(temp.name) + self.bin = self.base / "bin" + self.bin.mkdir() + self.home = self.base / "real-home" # 隔离前的 HOME（相当于 Mac 上的 /Users/） + self.calls = self.base / "plugin-calls" + self.calls.touch() + self._script(self.bin / "docker", ( + '#!/bin/sh\n' + 'if [ "$1" = compose ]; then\n' + ' p="${DOCKER_CONFIG:-$HOME/.docker}/cli-plugins/docker-compose"\n' + ' if [ -x "$p" ]; then exec "$p" "$@"; fi\n' + ' echo "docker: \'compose\' is not a docker command." >&2\n' + ' exit 125\n' + 'fi\n' + 'exit 0\n')) + self._script(self.bin / "jq", '#!/bin/sh\ncat >/dev/null\necho "{}"\n') + self.environ = {"PATH": os.pathsep.join([str(self.bin), "/usr/bin", "/bin"]), "HOME": str(self.home)} + + @staticmethod + def _script(path, body): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body) + path.chmod(0o700) + + def _plugin(self, directory): + # 假插件：记录参数（不含任何值），version / config 一律成功 + self._script(directory / "docker-compose", ( + '#!/bin/sh\n' + f'printf "%s\\n" "$*" >> "{self.calls}"\n' + 'case "$*" in *"--format json"*) echo "{}";; esac\n' + 'exit 0\n')) + + def _probe_then_run(self, extra=None): + from unittest import mock + with mock.patch.dict(os.environ, {**self.environ, **(extra or {})}, clear=True): + self.assertTrue(_compose_tools(), "探测应找到 compose 插件") + case = ReleaseComposeWebhookAddrTests("test_compose_config_quiet") + # 直接调用 setUp；_COMPOSE_OK 换成本次探测结果，保证在没有真实 Docker 的机器上也会执行 + with mock.patch.dict(globals(), {"_COMPOSE_OK": True}): + case.setUp() + try: + case.test_compose_config_quiet() + finally: + case.doCleanups() + self.assertIn("config --quiet", self.calls.read_text()) + + def test_plugin_only_in_home_docker_cli_plugins(self): + self._plugin(self.home / ".docker/cli-plugins") + self._probe_then_run() + + def test_docker_config_is_passed_through(self): + custom = self.base / "custom-docker-config" + self._plugin(custom / "cli-plugins") + self._probe_then_run({"DOCKER_CONFIG": str(custom)}) + + def test_probe_oserror_skips_instead_of_raising(self): + from unittest import mock + # 有执行位、shutil.which 能找到，但解释器不存在：exec 失败，subprocess 抛 OSError（FileNotFoundError） + self._script(self.bin / "docker", "#!/nonexistent-r3inc/interpreter\n") + # PATH 只留替身目录：否则 subprocess 遇到 ENOENT 会继续试 PATH 里后面的真实 docker + with mock.patch.dict(os.environ, {**self.environ, "PATH": str(self.bin)}, clear=True): + self.assertFalse(_compose_tools()) + + def test_probe_timeout_skips_instead_of_raising(self): + from unittest import mock + timeout = subprocess.TimeoutExpired(["docker", "compose", "version"], 30) + with mock.patch.dict(os.environ, self.environ, clear=True), \ + mock.patch.object(subprocess, "run", side_effect=timeout): + self.assertFalse(_compose_tools()) + + +class ComposeRequiredGateTests(unittest.TestCase): + """缺 compose 时：CI=true / RELEASE_TESTS_REQUIRE_COMPOSE=1 判失败，本地 skip。不需要真实 Docker。""" + + def _run_gated(self, environ, ok=False): + from unittest import mock + result = unittest.TestResult() + with mock.patch.dict(os.environ, {"PATH": os.environ.get("PATH", ""), **environ}, clear=True), \ + mock.patch.dict(globals(), {"_COMPOSE_OK": ok}): + unittest.defaultTestLoader.loadTestsFromTestCase(ReleaseComposeWebhookAddrTests).run(result) + return result + + def test_missing_compose_fails_when_required(self): + for environ in ({"CI": "true"}, {"CI": "TRUE"}, {"CI": "1"}, {"RELEASE_TESTS_REQUIRE_COMPOSE": "1"}, + {"CI": "false", "RELEASE_TESTS_REQUIRE_COMPOSE": "1"}): + with self.subTest(environ=environ): + r = self._run_gated(environ) + self.assertEqual((len(r.skipped), len(r.errors)), (0, 0)) + self.assertEqual(len(r.failures), r.testsRun) + self.assertGreater(r.testsRun, 0) + self.assertIn("不允许跳过", r.failures[0][1]) + + def test_missing_compose_skips_locally(self): + for environ in ({}, {"CI": "false"}, {"CI": ""}, {"RELEASE_TESTS_REQUIRE_COMPOSE": "0"}): + with self.subTest(environ=environ): + r = self._run_gated(environ) + self.assertEqual((len(r.failures), len(r.errors)), (0, 0)) + self.assertEqual(len(r.skipped), r.testsRun) + self.assertGreater(r.testsRun, 0) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_release_env.py b/scripts/tests/test_release_env.py index 8096d498..98a94e4b 100644 --- a/scripts/tests/test_release_env.py +++ b/scripts/tests/test_release_env.py @@ -1,14 +1,87 @@ -"""Release container configuration regression without Docker or external services.""" +"""release-env.py 的 WEBHOOK_MYSQL_ADDR 生成与新参数回归：只用临时目录和合成口令，不访问 Docker 或数据库""" +import json +import os from pathlib import Path import re +import shutil +import stat +import subprocess +import sys +import tempfile import unittest ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts/release-env.py" COMPOSE = ROOT / "deploy/release/compose.yaml" +# 合成口令：仅用于测试，满足 SAFE_VALUE 字符集 +FIXTURE_PASSWORD = "fixture-only-pw" +# go-sql-driver DSN：user:passwd@net(addr)/dbname?params（webhook 经 gorm.io/driver/mysql 调用 mysql.ParseDSN） +DSN_PATTERN = re.compile(r"^(?P[^:@/]+):(?P[^@/]+)@(?Ptcp)\((?P[^)]+)\)/(?P[^?/]+)\?(?P.+)$") + +GO_PARSE_DSN = """package main + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/go-sql-driver/mysql" +) + +func main() { + line, _ := bufio.NewReader(os.Stdin).ReadString('\\n') + // 与 webhook 加载逻辑一致：strings.TrimSpace(os.Getenv("WEBHOOK_MYSQL_ADDR")) 后交给 mysql.Open + c, err := mysql.ParseDSN(strings.TrimSpace(line)) + if err != nil { + fmt.Fprintln(os.Stderr, "DSN_PARSE_FAILED") + os.Exit(3) + } + out, _ := json.Marshal(map[string]any{"user": c.User, "passwd": c.Passwd, "net": c.Net, "addr": c.Addr, + "db": c.DBName, "parseTime": c.ParseTime, "loc": c.Loc.String(), "charset": c.Params["charset"]}) + fmt.Println(string(out)) +} +""" + + +def read_env(path): + return dict(line.split("=", 1) for line in path.read_text().splitlines() + if line and not line.startswith("#") and "=" in line) + + +def compose_mysql_settings(): + """从发布 compose 读取 mysql 服务的 MYSQL_USER / MYSQL_DATABASE 与 backend 的 MYSQL_ADDR 模板（纯文本解析，无需 PyYAML）""" + text = COMPOSE.read_text() + user = re.search(r"^\s+MYSQL_USER:\s*(\S+)\s*$", text, re.M).group(1) + database = re.search(r"^\s+MYSQL_DATABASE:\s*(\S+)\s*$", text, re.M).group(1) + backend = re.search(r"^\s+MYSQL_ADDR:\s*(.+?)\s*$", text, re.M).group(1) + return user, database, backend class ReleaseEnvTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.dir = Path(self.temp.name) + + def run_script(self, *arguments): + return subprocess.run([sys.executable, str(SCRIPT), *arguments], + capture_output=True, text=True, timeout=30) + + def assert_no_value_leak(self, result, *values): + output = result.stdout + result.stderr + for value in values: + self.assertNotIn(value, output) + self.assertNotIn("@tcp(", output) + + def generate(self, *extra): + destination = self.dir / "gen.env" + result = self.run_script("--env-file", str(destination), *extra) + self.assertEqual(result.returncode, 0, result.stderr) + return destination, result + def test_release_container_issuer_matches_repository_configuration(self): # The runtime image only copies the binary: a YAML-only issuer would stop startup. compose = COMPOSE.read_text() @@ -21,3 +94,133 @@ def test_release_container_issuer_matches_repository_configuration(self): self.assertEqual(match.group(1), configured.group(1), "changing issuer on container upgrade would invalidate existing tokens") + def test_generated_dsn_matches_mysql_variables_and_backend_dsn(self): + destination, result = self.generate() + settings = read_env(destination) + dsn = settings["WEBHOOK_MYSQL_ADDR"] + match = DSN_PATTERN.match(dsn) + self.assertIsNotNone(match, "WEBHOOK_MYSQL_ADDR 不符合 go-sql-driver DSN 格式") + user, database, backend_template = compose_mysql_settings() + self.assertEqual(match["user"], user) + self.assertEqual(match["passwd"], settings["MYSQL_PASSWORD"]) + self.assertEqual(match["addr"], "mysql:3306") + self.assertEqual(match["db"], database) + self.assertEqual(dict(p.split("=", 1) for p in match["params"].split("&")), + {"charset": "utf8mb4", "parseTime": "True", "loc": "Local"}) + # 与 compose 中 backend 的 MYSQL_ADDR（用同一 MYSQL_PASSWORD 插值）逐字一致 + expanded = re.sub(r"\$\{MYSQL_PASSWORD:\?[^}]*\}", settings["MYSQL_PASSWORD"], backend_template) + self.assertEqual(dsn, expanded) + # .env 不加引号：值中不能出现空白、引号或 Compose 插值字符 + self.assertIsNone(re.search(r"[\s'\"$#]", dsn)) + self.assert_no_value_leak(result, dsn, settings["MYSQL_PASSWORD"]) + + def test_generated_dsn_parses_with_go_sql_driver(self): + go = shutil.which("go") + if go is None: + self.skipTest("go 不可用，跳过真实驱动解析（正则校验仍在上一用例执行）") + destination, _ = self.generate() + settings = read_env(destination) + module = self.dir / "dsncheck" + module.mkdir() + (module / "go.mod").write_text("module dsncheck\n\ngo 1.22\n\nrequire github.com/go-sql-driver/mysql v1.8.1\n") + (module / "main.go").write_text(GO_PARSE_DSN) + # 只用本机模块缓存，不联网、不自动切换工具链 + env = {**os.environ, "GOTOOLCHAIN": "local", "GOPROXY": "off", "GOFLAGS": "-mod=mod", "GOSUMDB": "off"} + result = subprocess.run([go, "run", "."], cwd=module, input=settings["WEBHOOK_MYSQL_ADDR"] + "\n", + env=env, capture_output=True, text=True, timeout=180) + if result.returncode != 0 and "DSN_PARSE_FAILED" not in result.stderr: + self.skipTest("本机模块缓存缺少 go-sql-driver/mysql v1.8.1，跳过真实驱动解析") + self.assertEqual(result.returncode, 0, "go-sql-driver 解析 WEBHOOK_MYSQL_ADDR 失败") + parsed = json.loads(result.stdout) + user, database, _ = compose_mysql_settings() + self.assertEqual(parsed, {"user": user, "passwd": settings["MYSQL_PASSWORD"], "net": "tcp", + "addr": "mysql:3306", "db": database, "parseTime": True, + "loc": "Local", "charset": "utf8mb4"}) + + def test_env_file_and_mysql_host_parameters(self): + destination, result = self.generate("--mysql-host", "127.0.0.1:13306") + self.assertEqual(stat.S_IMODE(destination.stat().st_mode), 0o600) + match = DSN_PATTERN.match(read_env(destination)["WEBHOOK_MYSQL_ADDR"]) + self.assertEqual(match["addr"], "127.0.0.1:13306") + self.assertIn(str(destination), result.stdout) + for host in ("mysql", "mysql:3306/x", "my sql:3306", "mysql:abc", "a@b:3306"): + with self.subTest(host=host): + target = self.dir / "rejected.env" + bad = self.run_script("--env-file", str(target), "--mysql-host", host) + self.assertEqual(bad.returncode, 2) + self.assertIn("--mysql-host", bad.stderr) + self.assertFalse(target.exists()) + + def test_existing_file_never_overwritten_without_add_missing(self): + destination = self.dir / "existing.env" + destination.write_text(f"MYSQL_PASSWORD={FIXTURE_PASSWORD}\nOTHER=keep\n") + before = destination.read_bytes() + result = self.run_script("--env-file", str(destination)) + self.assertEqual(result.returncode, 1) + self.assertIn("--add-missing", result.stderr) + self.assertEqual(destination.read_bytes(), before) + self.assert_no_value_leak(result, FIXTURE_PASSWORD) + + def test_add_missing_appends_only_dsn_and_sets_0600(self): + destination = self.dir / "upgrade.env" + # 无结尾换行 + 0644，验证追加时补换行且收紧权限 + destination.write_text(f"# note\nMYSQL_PASSWORD={FIXTURE_PASSWORD}\nOTHER=keep") + destination.chmod(0o644) + before = destination.read_text() + result = self.run_script("--env-file", str(destination), "--add-missing") + self.assertEqual(result.returncode, 0, result.stderr) + after = destination.read_text() + self.assertTrue(after.startswith(before + "\n"), "原有内容必须原样保留") + self.assertEqual(stat.S_IMODE(destination.stat().st_mode), 0o600) + settings = read_env(destination) + self.assertEqual(settings["OTHER"], "keep") + self.assertEqual(settings["MYSQL_PASSWORD"], FIXTURE_PASSWORD) + expected = f"cloudops:{FIXTURE_PASSWORD}@tcp(mysql:3306)/cloudops?charset=utf8mb4&parseTime=True&loc=Local" + self.assertEqual(settings["WEBHOOK_MYSQL_ADDR"], expected) + self.assertEqual(after.count("WEBHOOK_MYSQL_ADDR="), 1) + self.assert_no_value_leak(result, FIXTURE_PASSWORD, expected) + # 再次执行：已有值不覆盖、不重复追加 + again = self.run_script("--env-file", str(destination), "--add-missing", "--mysql-host", "other:3306") + self.assertEqual(again.returncode, 0, again.stderr) + self.assertEqual(destination.read_text(), after) + self.assert_no_value_leak(again, FIXTURE_PASSWORD, expected) + + def test_add_missing_keeps_existing_custom_value(self): + destination = self.dir / "custom.env" + custom = "hook:fixture-custom@tcp(db.internal:3306)/hookdb?parseTime=True" + destination.write_text(f"MYSQL_PASSWORD={FIXTURE_PASSWORD}\nWEBHOOK_MYSQL_ADDR={custom}\n") + destination.chmod(0o600) + before = destination.read_bytes() + result = self.run_script("--env-file", str(destination), "--add-missing") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(destination.read_bytes(), before) + self.assert_no_value_leak(result, FIXTURE_PASSWORD, custom) + + def test_add_missing_refusals_leave_file_unchanged(self): + cases = ( + ("missing-password", "OTHER=keep\n", "MYSQL_PASSWORD"), + ("empty-password", "MYSQL_PASSWORD=\n", "MYSQL_PASSWORD"), + ("empty-placeholder", f"MYSQL_PASSWORD={FIXTURE_PASSWORD}\nWEBHOOK_MYSQL_ADDR=\n", "remove that line"), + ("unsafe-password", "MYSQL_PASSWORD=fixture@pw/x\n", "unsafe"), + ) + for name, content, message in cases: + with self.subTest(case=name): + destination = self.dir / f"{name}.env" + destination.write_text(content) + before = destination.read_bytes() + result = self.run_script("--env-file", str(destination), "--add-missing") + self.assertEqual(result.returncode, 1) + self.assertIn(message, result.stderr) + self.assertEqual(destination.read_bytes(), before) + self.assert_no_value_leak(result, FIXTURE_PASSWORD, "fixture@pw/x") + + def test_add_missing_requires_existing_file(self): + destination = self.dir / "absent.env" + result = self.run_script("--env-file", str(destination), "--add-missing") + self.assertEqual(result.returncode, 1) + self.assertIn("does not exist", result.stderr) + self.assertFalse(destination.exists()) + + +if __name__ == "__main__": + unittest.main()