diff --git a/docs/deploy/key-management.md b/docs/deploy/key-management.md index dd18c11..5490228 100644 --- a/docs/deploy/key-management.md +++ b/docs/deploy/key-management.md @@ -7,7 +7,9 @@ This guide covers the secrets used by `result_server/app.py`. Production deployments must provide: - `FLASK_SECRET_KEY`: at least 32 characters, generated randomly. -- `RESULT_SERVER_KEYS`: one or more runner-scoped ingest keys. +- `RESULT_SERVER_KEYS`: one or more runner-scoped ingest keys, or + `RESULT_SERVER_TRUSTED_PROXY_AUTH=mtls` when nginx verifies client + certificates before proxying ingest/query API requests. Use runner-scoped server keys instead of the legacy server-side `RESULT_SERVER_KEY` fallback: @@ -17,20 +19,49 @@ RESULT_SERVER_KEYS=runner-a:,runner-b: ``` `RESULT_SERVER_KEYS` is the server-side registry of accepted posting/query -keys. It is intentionally broader than the current single-key CI setup so that -the portal can later accept results from multiple trusted CI sources, such as -the main BenchKit CI, site-managed runners, collaborator forks, or -estimator-only pipelines. - -Each client job still receives a single `RESULT_SERVER_KEY` secret for its own -uploads. This client-side key must match one entry in `RESULT_SERVER_KEYS`, and -it is typically injected through GitLab CI/CD variables or another CI secret -mechanism rather than stored on the runner host. +keys for deployments that still use shared API keys. Client jobs in mTLS mode +do not use `RESULT_SERVER_KEY` and do not send an `X-API-Key` header. Each key must be at least 32 characters and must not use known insecure examples such as `dev-api-key`, `changeme`, or `secret`. The production app refuses to start when these checks fail. +## Client Certificate Mode + +Deployments can avoid shared ingest keys by terminating TLS at a trusted reverse +proxy and requiring a client certificate for result API endpoints. Configure the +portal with: + +```text +RESULT_SERVER_TRUSTED_PROXY_AUTH=mtls +``` + +In this mode `RESULT_SERVER_KEYS` and the legacy `RESULT_SERVER_KEY` may be +empty, provided nginx verifies the client certificate and forwards these headers +only to the local Flask backend: + +```nginx +proxy_set_header X-Result-Server-Client-Verify $ssl_client_verify; +proxy_set_header X-Result-Server-Client-DN $ssl_client_s_dn; +proxy_set_header X-Result-Server-Client-Fingerprint $ssl_client_fingerprint; +``` + +The nginx location must reject requests unless `$ssl_client_verify` is +`SUCCESS`. Keep the backend bound to loopback or a Unix socket so clients cannot +bypass nginx and provide these headers themselves. + +CI jobs can use host-managed certificates instead of GitLab CI/CD secret +variables by mounting them read-only into a self-managed runner container and +setting: + +```text +RESULT_SERVER_CLIENT_CERT=/run/benchkit/result-server/client.crt +RESULT_SERVER_CLIENT_KEY=/run/benchkit/result-server/client.key +``` + +The upload/query helper scripts use these variables automatically and do not +send an `X-API-Key` header. + ## Generation Generate random values with a local secret generator, for example: diff --git a/docs/guides/developer-reference.md b/docs/guides/developer-reference.md index 99dc505..1dbc523 100644 --- a/docs/guides/developer-reference.md +++ b/docs/guides/developer-reference.md @@ -250,7 +250,7 @@ For production portal deployments: - `app.py` binds to `127.0.0.1:8800` by default; set `RESULT_SERVER_HOST` and `RESULT_SERVER_PORT` explicitly when the deployment requires a different bind address. - Set runner-scoped ingest keys with `RESULT_SERVER_KEYS=runner-a:,runner-b:`. - `RESULT_SERVER_KEYS` is the server-side registry of accepted posting/query keys. It is intentionally broader than the current single-key CI setup so the portal can later accept results from multiple trusted CI sources such as main BenchKit CI, site-managed runners, collaborator forks, or estimator-only pipelines. -- Each client job still receives a single `RESULT_SERVER_KEY` secret for its own upload/query operations, usually through GitLab CI/CD variables or another CI secret store. That client-side key must match one entry in server-side `RESULT_SERVER_KEYS`. +- Client jobs on mTLS-protected deployments use `RESULT_SERVER_CLIENT_CERT` and `RESULT_SERVER_CLIENT_KEY` instead of `RESULT_SERVER_KEY`; they do not send an `X-API-Key` header. - `FLASK_SECRET_KEY` and each ingest key must be at least 32 characters and must not use known insecure examples such as `dev-api-key`, `changeme`, or `secret`; production startup refuses these values. - The legacy server-side `RESULT_SERVER_KEY` variable is still accepted as runner `default` for compatibility, but production portal deployments should rotate the accepted-key registry to `RESULT_SERVER_KEYS`. - See `docs/deploy/key-management.md` for generation and rotation guidance. diff --git a/result_server/app.py b/result_server/app.py index 2b13617..a88aa98 100644 --- a/result_server/app.py +++ b/result_server/app.py @@ -54,8 +54,14 @@ def _configure_redis(app, prefix): redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0") app.config["REDIS_CONN"] = redis.from_url(redis_url, decode_responses=True) - app.config["REDIS_PREFIX"] = "dev:" if prefix == "/dev" else "main:" - app.config["SESSION_COOKIE_NAME"] = "session_dev" if prefix == "/dev" else "session_main" + app.config["REDIS_PREFIX"] = os.environ.get( + "RESULT_SERVER_REDIS_PREFIX", + "dev:" if prefix == "/dev" else "main:", + ) + app.config["SESSION_COOKIE_NAME"] = os.environ.get( + "RESULT_SERVER_SESSION_COOKIE_NAME", + "session_dev" if prefix == "/dev" else "session_main", + ) app.config["AUTH_REQUIRES_REDIS"] = True @@ -109,6 +115,14 @@ def _configure_admin_policy(app): ) +def _configure_api_auth(app): + """Configure API authentication modes accepted behind the reverse proxy.""" + app.config["TRUSTED_PROXY_AUTH"] = os.environ.get( + "RESULT_SERVER_TRUSTED_PROXY_AUTH", + "", + ).strip() + + def _configure_execution_profiles(app, base_dir): """Configure the site-local execution profile database path.""" app.config["EXECUTION_PROFILE_DB_PATH"] = os.environ.get( @@ -153,6 +167,7 @@ def create_app(prefix="", base_dir=None): _configure_result_directories(app, base_dir) _configure_upload_limits(app) _configure_admin_policy(app) + _configure_api_auth(app) _configure_execution_profiles(app, base_dir) init_csrf(app, exempt_blueprints=(api_bp,)) diff --git a/result_server/routes/admin.py b/result_server/routes/admin.py index 7636da1..e5992e8 100644 --- a/result_server/routes/admin.py +++ b/result_server/routes/admin.py @@ -26,6 +26,7 @@ load_execution_profiles, normalize_profile, ) +from utils.gitlab_pipeline import build_pipeline_plan, configured_gitlab_repo from utils.rate_limit import rate_limited from utils.user_store import get_user_store @@ -122,6 +123,10 @@ def _parse_execution_profile_form(): return raw_profile, errors +def _parse_bool_form(name): + return request.form.get(name) == "on" + + def _user_affiliations(store, email): """Return the affiliations for a user, handling missing records uniformly.""" if hasattr(store, "get_user"): @@ -173,6 +178,7 @@ def execution_profiles(): return render_template( "admin_execution_profiles.html", profile_result=profile_result, + dry_run_result=None, ) @@ -230,6 +236,88 @@ def upsert_execution_profile(): return redirect(url_for("admin.execution_profiles")) +@admin_bp.route("/execution-profiles/dry-run-submit", methods=["POST"]) +@admin_required +@rate_limited(max_per_minute=20, key_fn=_admin_rate_key, scope="admin_write") +def dry_run_execution_profile_submit(): + """Resolve an execution profile and render a GitLab Pipeline API dry run.""" + db_path = current_app.config.get("EXECUTION_PROFILE_DB_PATH") + store = ExecutionProfileStore(db_path) + target_ref = request.form.get("target_ref", "").strip() or "develop" + profile_id = request.form.get("profile_id", "").strip() + code = request.form.get("code", "").strip() + system = request.form.get("system", "").strip() + exp = request.form.get("exp", "").strip() + app = request.form.get("app", "").strip() + benchpark = _parse_bool_form("benchpark") + park_only = _parse_bool_form("park_only") + park_send = _parse_bool_form("park_send") + + resolve_result = store.resolve_profile( + profile_id=profile_id, + code=code, + system=system, + exp=exp, + ) + profile = resolve_result.profile + plan = build_pipeline_plan( + gitlab_repo=configured_gitlab_repo(), + target_ref=target_ref, + code=code, + system=system, + app=app, + benchpark=benchpark, + park_only=park_only, + park_send=park_send, + scheduler_extra_args=resolve_result.scheduler_extra_args, + ) + errors = resolve_result.errors + plan.errors + status = "dry_run_ready" if not errors else "dry_run_blocked" + request_id = store.create_execution_request( + request_type="gitlab_pipeline", + status=status, + dry_run=True, + profile_id=profile["id"] if profile else profile_id, + target_ref=target_ref, + code=code, + system=system, + exp=exp, + payload={"api_url": plan.api_url, "payload": plan.payload}, + errors=errors, + actor=session.get("user_email", ""), + ) + + audit_event( + "admin_execution_profile_submit_dry_run", + actor=session.get("user_email"), + target=profile["id"] if profile else profile_id, + result="success" if not errors else "failure", + details={ + "request_id": request_id, + "target_ref": target_ref, + "code": code, + "system": system, + "exp": exp, + "errors": errors, + }, + ) + + profile_result = load_execution_profiles(db_path) + return render_template( + "admin_execution_profiles.html", + profile_result=profile_result, + dry_run_result={ + "request_id": request_id, + "status": status, + "profile": profile, + "api_url": plan.api_url, + "payload_json": json.dumps(plan.payload, indent=2, sort_keys=True), + "errors": errors, + "warnings": plan.warnings, + }, + ) + + @admin_bp.route("/users/add", methods=["POST"]) @admin_required @rate_limited(max_per_minute=20, key_fn=_admin_rate_key, scope="admin_write") diff --git a/result_server/routes/api.py b/result_server/routes/api.py index 1c6b255..d74cfb9 100644 --- a/result_server/routes/api.py +++ b/result_server/routes/api.py @@ -12,7 +12,7 @@ import tempfile from datetime import datetime -from utils.auth import verify_ingest_key +from utils.auth import verify_ingest_key, verify_trusted_proxy_auth from utils.audit_logging import audit_event from utils.rate_limit import rate_limited @@ -28,19 +28,29 @@ def require_api_key(): """Validate the request API key and return the authenticated runner id.""" runner_id = verify_ingest_key(request.headers.get("X-API-Key", "")) + auth_method = "api_key" + if not runner_id: + runner_id = verify_trusted_proxy_auth(request.headers) + auth_method = "trusted_proxy" if not runner_id: audit_event( "api_auth_failed", result="failure", level=logging.WARNING, - details={"reason": "invalid_api_key"}, + details={"reason": "invalid_api_key_or_proxy_auth"}, ) abort(401, description="Invalid API Key") - audit_event("api_auth_success", actor=runner_id, result="success") + audit_event( + "api_auth_success", + actor=runner_id, + result="success", + details={"auth_method": auth_method}, + ) current_app.logger.info( - "api key accepted", + "api auth accepted", extra={ "runner_id": runner_id, + "auth_method": auth_method, "endpoint": request.path, "ip": request.remote_addr, }, @@ -50,7 +60,11 @@ def require_api_key(): def _api_rate_key(req): """Return the runner-scoped API rate-limit key for a request.""" - runner_id = verify_ingest_key(req.headers.get("X-API-Key", "")) or "unknown" + runner_id = ( + verify_ingest_key(req.headers.get("X-API-Key", "")) + or verify_trusted_proxy_auth(req.headers) + or "unknown" + ) return f"runner:{runner_id}" diff --git a/result_server/templates/admin_execution_profiles.html b/result_server/templates/admin_execution_profiles.html index 774bd0a..be80333 100644 --- a/result_server/templates/admin_execution_profiles.html +++ b/result_server/templates/admin_execution_profiles.html @@ -93,6 +93,17 @@ } .btn-primary { background-color: #0f766e; color: #fff; border-color: #0f766e; } .btn-primary:hover { background-color: #0b5f59; color: #fff; } + .btn-secondary { background-color: #1f2937; color: #fff; border-color: #1f2937; } + .btn-secondary:hover { background-color: #111827; color: #fff; } + .profile-dry-run-output { + margin-top: 14px; + padding: 12px; + overflow-x: auto; + border: 1px solid #d8e2e8; + border-radius: 8px; + background: #0f172a; + color: #e2e8f0; + } {% with messages = get_flashed_messages() %} @@ -237,6 +248,88 @@

Create / Update Profile

+
+

GitLab Pipeline Dry Run

+

+ Resolve an approved profile for a target scope and preview the GitLab + Pipeline API request. This does not submit a pipeline. +

+
+ {% if csrf_token is defined %}{% endif %} + + + + + + +
+ + + + +
+
+ + {% if dry_run_result %} +
+ Dry-run request #{{ dry_run_result.request_id }}: + {{ dry_run_result.status }} + {% if dry_run_result.profile %} + using profile {{ dry_run_result.profile.id }} + {% endif %} +
+ {% if dry_run_result.errors %} +
+ Dry-run blockers: +
    + {% for error in dry_run_result.errors %} +
  • {{ error }}
    • + {% endfor %} +
+
+ {% endif %} + {% if dry_run_result.warnings %} +
+ Dry-run warnings: +
    + {% for warning in dry_run_result.warnings %} +
  • {{ warning }}
    • + {% endfor %} +
+
+ {% endif %} +

API URL: {{ dry_run_result.api_url or 'not configured' }}

+ 
{{ dry_run_result.payload_json }}
+ {% endif %} +
+

Registered Profiles

diff --git a/result_server/tests/test_api_routes.py b/result_server/tests/test_api_routes.py index b33dd61..c16c09e 100644 --- a/result_server/tests/test_api_routes.py +++ b/result_server/tests/test_api_routes.py @@ -96,12 +96,37 @@ def test_valid_key_logs_runner_id(self, client, caplog): assert resp.status_code == 200 assert any( - record.message == "api key accepted" + record.message == "api auth accepted" and getattr(record, "runner_id", None) == "test-runner" + and getattr(record, "auth_method", None) == "api_key" and getattr(record, "endpoint", None) == "/api/ingest/result" for record in caplog.records ) + def test_trusted_proxy_mtls_auth_is_accepted_without_api_key(self, app, caplog): + """nginx-verified mTLS can authenticate API requests without shared keys.""" + app.config["INGEST_KEYS"] = {} + app.config["TRUSTED_PROXY_AUTH"] = "mtls" + + with app.test_client() as client, caplog.at_level(logging.INFO): + resp = client.post( + "/api/ingest/result", + data=b'{"code":"mtls"}', + headers={ + "X-Result-Server-Client-Verify": "SUCCESS", + "X-Result-Server-Client-Fingerprint": "AA:BB:CC", + "Content-Type": "application/json", + }, + ) + + assert resp.status_code == 200 + assert any( + record.message == "api auth accepted" + and getattr(record, "runner_id", None) == "mtls:AA:BB:CC" + and getattr(record, "auth_method", None) == "trusted_proxy" + for record in caplog.records + ) + def test_multiple_ingest_keys_accept_individual_runner_keys(self, app): """RESULT_SERVER_KEYS-style config should accept each runner key.""" app.config["INGEST_KEYS"] = { diff --git a/result_server/tests/test_execution_profiles.py b/result_server/tests/test_execution_profiles.py index 0f9a320..5214e35 100644 --- a/result_server/tests/test_execution_profiles.py +++ b/result_server/tests/test_execution_profiles.py @@ -5,6 +5,7 @@ import json import os import shutil +import sqlite3 import sys import tempfile @@ -193,6 +194,31 @@ def test_execution_profile_store_validates_requested_profile_scope(tmp_path): assert result.errors == ["execution profile is not approved for target: rikyu-qws"] +def test_execution_profile_store_records_dry_run_request(tmp_path): + db_path = tmp_path / "cx_portal.sqlite3" + store = ExecutionProfileStore(str(db_path)) + + request_id = store.create_execution_request( + request_type="gitlab_pipeline", + status="dry_run_ready", + dry_run=True, + profile_id="rikyu-qws-nightly", + target_ref="develop", + code="qws", + system="RIKYU", + exp="case0", + payload={"payload": {"ref": "develop", "variables": []}}, + actor="admin@test.com", + ) + + assert request_id == 1 + with sqlite3.connect(db_path) as conn: + row = conn.execute( + "SELECT status, dry_run, profile_id, target_ref FROM execution_requests" + ).fetchone() + assert row == ("dry_run_ready", 1, "rikyu-qws-nightly", "develop") + + def test_import_execution_profiles_json_seeds_sqlite_registry(tmp_path): db_path = tmp_path / "cx_portal.sqlite3" seed_path = tmp_path / "execution_profiles.json" @@ -344,3 +370,68 @@ def test_admin_execution_profiles_rejects_invalid_metadata_json(tmp_path): assert result.profiles == [] finally: _cleanup(temp_dirs) + + +def test_admin_execution_profiles_dry_run_submit_renders_payload(tmp_path, monkeypatch): + monkeypatch.setenv("RESULT_SERVER_GITLAB_REPO", "gitlab.example.org/group/benchkit.git") + db_path = tmp_path / "cx_portal.sqlite3" + ExecutionProfileStore(str(db_path)).upsert_profile(_profile(), actor="admin") + app, temp_dirs = _admin_app(db_path) + try: + with app.test_client() as client: + _login_admin(client) + resp = client.post( + "/admin/execution-profiles/dry-run-submit", + data={ + "target_ref": "develop", + "code": "qws", + "system": "RIKYU", + "exp": "case0", + }, + ) + + html = resp.data.decode() + assert resp.status_code == 200 + assert "Dry-run request #1" in html + assert "dry_run_ready" in html + assert "https://gitlab.example.org/api/v4/projects/group%2Fbenchkit/pipeline" in html + assert "--account=site-local" in html + + with sqlite3.connect(db_path) as conn: + row = conn.execute( + "SELECT status, profile_id, code, system, payload_json FROM execution_requests" + ).fetchone() + assert row[:4] == ("dry_run_ready", "rikyu-qws-nightly", "qws", "RIKYU") + payload_record = json.loads(row[4]) + variables = payload_record["payload"]["variables"] + assert {"key": "code", "value": "qws", "variable_type": "env_var"} in variables + assert { + "key": "BK_SCHEDULER_EXTRA_ARGS_RIKYU", + "value": "--account=site-local", + "variable_type": "env_var", + } in variables + finally: + _cleanup(temp_dirs) + + +def test_admin_execution_profiles_dry_run_blocks_without_matching_profile( + tmp_path, + monkeypatch, +): + monkeypatch.setenv("RESULT_SERVER_GITLAB_REPO", "gitlab.example.org/group/benchkit.git") + db_path = tmp_path / "cx_portal.sqlite3" + app, temp_dirs = _admin_app(db_path) + try: + with app.test_client() as client: + _login_admin(client) + resp = client.post( + "/admin/execution-profiles/dry-run-submit", + data={"target_ref": "develop", "code": "qws", "system": "RIKYU"}, + ) + + html = resp.data.decode() + assert resp.status_code == 200 + assert "dry_run_blocked" in html + assert "no approved execution profile matches target" in html + finally: + _cleanup(temp_dirs) diff --git a/result_server/tests/test_preflight.py b/result_server/tests/test_preflight.py index 76694a6..9d24269 100644 --- a/result_server/tests/test_preflight.py +++ b/result_server/tests/test_preflight.py @@ -27,6 +27,10 @@ def test_accepts_parallel_rotation_keys_for_same_runner(): assert validate_ingest_keys({old_key: "runner-a", new_key: "runner-a"}) == [] +def test_accepts_empty_ingest_keys_when_mtls_proxy_auth_is_configured(): + assert validate_ingest_keys({}, trusted_proxy_auth="mtls") == [] + + def test_rejects_short_flask_secret_key(): env = {"FLASK_SECRET_KEY": "short"} errors = validate_production_config(env, {"runner-key-012345678901234567890": "runner-a"}) @@ -39,3 +43,12 @@ def test_accepts_strong_production_config(): ingest_keys = {"runner-key-012345678901234567890": "runner-a"} assert validate_production_config(env, ingest_keys) == [] + + +def test_accepts_production_config_with_mtls_proxy_auth_without_ingest_keys(): + env = { + "FLASK_SECRET_KEY": "flask-secret-012345678901234567890", + "RESULT_SERVER_TRUSTED_PROXY_AUTH": "mtls", + } + + assert validate_production_config(env, {}) == [] diff --git a/result_server/utils/auth.py b/result_server/utils/auth.py index 8b16ad1..093b701 100644 --- a/result_server/utils/auth.py +++ b/result_server/utils/auth.py @@ -11,6 +11,11 @@ from flask import current_app +MTLS_VERIFY_HEADER = "X-Result-Server-Client-Verify" +MTLS_FINGERPRINT_HEADER = "X-Result-Server-Client-Fingerprint" +MTLS_DN_HEADER = "X-Result-Server-Client-DN" + + def parse_ingest_keys(env: Mapping[str, str] | None = None) -> dict[str, str]: """Parse RESULT_SERVER_KEYS/RESULT_SERVER_KEY into {api_key: runner_id}.""" env = env or os.environ @@ -63,3 +68,20 @@ def verify_ingest_key(presented: str | None) -> Optional[str]: if hmac.compare_digest(presented, configured_key): return runner_id return None + + +def verify_trusted_proxy_auth(headers: Mapping[str, str]) -> Optional[str]: + """Return a runner id when nginx has verified a configured proxy auth method.""" + if current_app.config.get("TRUSTED_PROXY_AUTH") != "mtls": + return None + + if headers.get(MTLS_VERIFY_HEADER, "") != "SUCCESS": + return None + + fingerprint = headers.get(MTLS_FINGERPRINT_HEADER, "").strip() + subject = headers.get(MTLS_DN_HEADER, "").strip() + if fingerprint: + return f"mtls:{fingerprint}" + if subject: + return f"mtls:{subject}" + return "mtls:verified-client" diff --git a/result_server/utils/execution_profiles.py b/result_server/utils/execution_profiles.py index 614873f..976e41d 100644 --- a/result_server/utils/execution_profiles.py +++ b/result_server/utils/execution_profiles.py @@ -13,7 +13,7 @@ PROFILE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$") -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 @dataclass(frozen=True) @@ -159,6 +159,9 @@ def migrate(self) -> None: ).fetchone()["version"] if current < 1: self._apply_v1(conn) + current = 1 + if current < 2: + self._apply_v2(conn) def _apply_v1(self, conn: sqlite3.Connection) -> None: now = _utc_now_iso() @@ -203,7 +206,34 @@ def _apply_v1(self, conn: sqlite3.Connection) -> None: ) conn.execute( "INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)", - (SCHEMA_VERSION, now), + (1, now), + ) + + def _apply_v2(self, conn: sqlite3.Connection) -> None: + now = _utc_now_iso() + conn.executescript( + """ + CREATE TABLE execution_requests ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + request_type TEXT NOT NULL, + status TEXT NOT NULL, + dry_run INTEGER NOT NULL DEFAULT 1, + profile_id TEXT NOT NULL DEFAULT '', + target_ref TEXT NOT NULL DEFAULT '', + code TEXT NOT NULL DEFAULT '', + system TEXT NOT NULL DEFAULT '', + exp TEXT NOT NULL DEFAULT '', + payload_json TEXT NOT NULL DEFAULT '{}', + errors_json TEXT NOT NULL DEFAULT '[]', + actor TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + """ + ) + conn.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (?, ?)", + (2, now), ) def upsert_profile(self, profile: dict[str, Any], *, actor: str = "") -> None: @@ -405,6 +435,51 @@ def resolve_profile( ) return ExecutionProfileResolveResult(profile=matches[0], errors=[]) + def create_execution_request( + self, + *, + request_type: str, + status: str, + dry_run: bool, + profile_id: str = "", + target_ref: str = "", + code: str = "", + system: str = "", + exp: str = "", + payload: dict[str, Any] | None = None, + errors: list[str] | None = None, + actor: str = "", + ) -> int: + """Record a Portal-triggered execution request or dry-run preview.""" + self.migrate() + now = _utc_now_iso() + with self.connect() as conn: + cur = conn.execute( + """ + INSERT INTO execution_requests ( + request_type, status, dry_run, profile_id, target_ref, + code, system, exp, payload_json, errors_json, actor, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + request_type, + status, + 1 if dry_run else 0, + profile_id, + target_ref, + code, + system, + exp, + _json_dump(payload or {}), + json.dumps(errors or [], ensure_ascii=False), + actor, + now, + now, + ), + ) + return int(cur.lastrowid) + def load_execution_profiles(db_path: str | None) -> ExecutionProfileLoadResult: """Load execution profiles from the site-local SQLite registry.""" diff --git a/result_server/utils/gitlab_pipeline.py b/result_server/utils/gitlab_pipeline.py new file mode 100644 index 0000000..d2ac219 --- /dev/null +++ b/result_server/utils/gitlab_pipeline.py @@ -0,0 +1,108 @@ +"""GitLab Pipeline API request planning helpers for Portal-triggered runs.""" + +from __future__ import annotations + +import os +import re +import urllib.parse +from dataclasses import dataclass +from typing import Any + + +GITLAB_REPO_RE = re.compile(r"^[A-Za-z0-9_.:-]+/[A-Za-z0-9_.~/:-]+(?:\\.git)?$") + + +@dataclass(frozen=True) +class GitLabPipelinePlan: + """A dry-run representation of a GitLab Pipeline API request.""" + + api_url: str + payload: dict[str, Any] + errors: list[str] + warnings: list[str] + + +def configured_gitlab_repo(env: dict[str, str] | None = None) -> str: + """Return the configured host/path GitLab repo for Portal submits.""" + source = env if env is not None else os.environ + return ( + source.get("RESULT_SERVER_GITLAB_REPO", "").strip() + or source.get("GITLAB_REPO", "").strip() + ) + + +def _split_gitlab_repo(repo: str) -> tuple[str, str] | None: + if not repo or "://" in repo or not GITLAB_REPO_RE.match(repo): + return None + normalized = repo.removesuffix(".git") + host, project_path = normalized.split("/", 1) + if not host or not project_path: + return None + return host, project_path + + +def _add_variable(variables: list[dict[str, str]], key: str, value: str) -> None: + text = str(value or "").strip() + if not text: + return + variables.append({"key": key, "value": text, "variable_type": "env_var"}) + + +def _scheduler_extra_args_key(system: str) -> str: + system_key = re.sub(r"[^A-Za-z0-9_]", "_", system) + return f"BK_SCHEDULER_EXTRA_ARGS_{system_key}" if system_key else "BK_SCHEDULER_EXTRA_ARGS" + + +def build_pipeline_plan( + *, + gitlab_repo: str, + target_ref: str, + code: str = "", + system: str = "", + app: str = "", + benchpark: bool = False, + park_only: bool = False, + park_send: bool = False, + scheduler_extra_args: str = "", +) -> GitLabPipelinePlan: + """Build the GitLab Pipeline API URL and JSON payload without sending it.""" + errors: list[str] = [] + warnings: list[str] = [] + ref = target_ref.strip() or "develop" + split_repo = _split_gitlab_repo(gitlab_repo) + api_url = "" + if split_repo is None: + errors.append( + "RESULT_SERVER_GITLAB_REPO or GITLAB_REPO must be host/path format" + ) + else: + host, project_path = split_repo + encoded_project = urllib.parse.quote(project_path, safe="") + api_url = f"https://{host}/api/v4/projects/{encoded_project}/pipeline" + + variables: list[dict[str, str]] = [] + _add_variable(variables, "code", code) + _add_variable(variables, "system", system) + _add_variable(variables, "app", app) + if benchpark: + _add_variable(variables, "benchpark", "true") + if park_only: + _add_variable(variables, "park_only", "true") + if park_send: + _add_variable(variables, "park_send", "true") + if scheduler_extra_args: + if system and "," not in system: + _add_variable(variables, _scheduler_extra_args_key(system), scheduler_extra_args) + else: + _add_variable(variables, "BK_SCHEDULER_EXTRA_ARGS", scheduler_extra_args) + if not system: + warnings.append( + "scheduler extra args are not scoped to a single system" + ) + + return GitLabPipelinePlan( + api_url=api_url, + payload={"ref": ref, "variables": variables}, + errors=errors, + warnings=warnings, + ) diff --git a/result_server/utils/preflight.py b/result_server/utils/preflight.py index 723f5e1..026e872 100644 --- a/result_server/utils/preflight.py +++ b/result_server/utils/preflight.py @@ -33,10 +33,16 @@ def _validate_secret(name: str, value: str | None) -> list[str]: return errors -def validate_ingest_keys(ingest_keys: Mapping[str, str]) -> list[str]: +def validate_ingest_keys( + ingest_keys: Mapping[str, str], + *, + trusted_proxy_auth: str = "", +) -> list[str]: """Return validation errors for runner-scoped ingest keys.""" errors: list[str] = [] if not ingest_keys: + if trusted_proxy_auth == "mtls": + return [] return ["RESULT_SERVER_KEYS or RESULT_SERVER_KEY is not set"] for key, runner_id in ingest_keys.items(): @@ -53,7 +59,12 @@ def validate_production_config( ) -> list[str]: """Return production startup errors for insecure result_server config.""" errors = _validate_secret("FLASK_SECRET_KEY", env.get("FLASK_SECRET_KEY")) - errors.extend(validate_ingest_keys(ingest_keys)) + errors.extend( + validate_ingest_keys( + ingest_keys, + trusted_proxy_auth=env.get("RESULT_SERVER_TRUSTED_PROXY_AUTH", "").strip(), + ) + ) if env.get("FLASK_DEBUG") in {"1", "true", "True"}: errors.append("FLASK_DEBUG must not be enabled for app.py") return errors diff --git a/scripts/estimation/result_query.sh b/scripts/estimation/result_query.sh index 09c8da8..f660edd 100644 --- a/scripts/estimation/result_query.sh +++ b/scripts/estimation/result_query.sh @@ -9,7 +9,8 @@ # $2 code (e.g. qws) # $3 exp (optional, e.g. default) # -# Requires: RESULT_SERVER, RESULT_SERVER_KEY environment variables +# Requires: RESULT_SERVER, RESULT_SERVER_CLIENT_CERT, and RESULT_SERVER_CLIENT_KEY +# environment variables. # Sets: est_current_fom (FOM value from the selected baseline-system result) # Exits with 1 on failure. # --------------------------------------------------------------------------- diff --git a/scripts/estimation/test_reestimate.sh b/scripts/estimation/test_reestimate.sh index c8b710d..caba93c 100644 --- a/scripts/estimation/test_reestimate.sh +++ b/scripts/estimation/test_reestimate.sh @@ -24,8 +24,8 @@ if [[ ! -d "programs/$code" ]]; then exit 1 fi -if [[ -z "${RESULT_SERVER:-}" || -z "${RESULT_SERVER_KEY:-}" ]]; then - echo "ERROR: RESULT_SERVER and RESULT_SERVER_KEY must be set" >&2 +if [[ -z "${RESULT_SERVER:-}" || -z "${RESULT_SERVER_CLIENT_CERT:-}" || -z "${RESULT_SERVER_CLIENT_KEY:-}" ]]; then + echo "ERROR: RESULT_SERVER, RESULT_SERVER_CLIENT_CERT, and RESULT_SERVER_CLIENT_KEY must be set" >&2 exit 1 fi diff --git a/scripts/result_server/api.sh b/scripts/result_server/api.sh index 9582e0c..4d83e58 100644 --- a/scripts/result_server/api.sh +++ b/scripts/result_server/api.sh @@ -8,16 +8,22 @@ bk_result_server_require_env() { echo "ERROR: RESULT_SERVER is not set" >&2 exit 1 fi - if [[ -z "${RESULT_SERVER_KEY:-}" ]]; then - echo "ERROR: RESULT_SERVER_KEY is not set" >&2 + if [[ -z "${RESULT_SERVER_CLIENT_CERT:-}" || -z "${RESULT_SERVER_CLIENT_KEY:-}" ]]; then + echo "ERROR: RESULT_SERVER_CLIENT_CERT and RESULT_SERVER_CLIENT_KEY must be set" >&2 exit 1 fi } +bk_result_server_set_curl_args() { + curl_auth_args=(--cert "$RESULT_SERVER_CLIENT_CERT" --key "$RESULT_SERVER_CLIENT_KEY") +} + bk_result_server_get_json() { local path_and_query="$1" bk_result_server_require_env - curl --fail -L -sS -H "X-API-Key: ${RESULT_SERVER_KEY}" \ + local curl_auth_args=() + bk_result_server_set_curl_args + curl --fail -L -sS "${curl_auth_args[@]}" \ "${RESULT_SERVER}${path_and_query}" } @@ -25,7 +31,9 @@ bk_result_server_get_json_to_file() { local path_and_query="$1" local output_path="$2" bk_result_server_require_env - curl --fail -L -sS -H "X-API-Key: ${RESULT_SERVER_KEY}" \ + local curl_auth_args=() + bk_result_server_set_curl_args + curl --fail -L -sS "${curl_auth_args[@]}" \ -o "$output_path" \ "${RESULT_SERVER}${path_and_query}" } @@ -34,7 +42,9 @@ bk_result_server_download_to_file() { local path_and_query="$1" local output_path="$2" bk_result_server_require_env - curl --fail -L -sS -H "X-API-Key: ${RESULT_SERVER_KEY}" \ + local curl_auth_args=() + bk_result_server_set_curl_args + curl --fail -L -sS "${curl_auth_args[@]}" \ -o "$output_path" \ "${RESULT_SERVER}${path_and_query}" } diff --git a/scripts/result_server/send_estimate.sh b/scripts/result_server/send_estimate.sh index 9d20fdc..f643830 100644 --- a/scripts/result_server/send_estimate.sh +++ b/scripts/result_server/send_estimate.sh @@ -6,6 +6,14 @@ set -euo pipefail echo "Sending estimate results to server" +result_server_set_curl_args() { + if [[ -z "${RESULT_SERVER_CLIENT_CERT:-}" || -z "${RESULT_SERVER_CLIENT_KEY:-}" ]]; then + echo "ERROR: RESULT_SERVER_CLIENT_CERT and RESULT_SERVER_CLIENT_KEY must be set" >&2 + exit 1 + fi + curl_auth_args=(--cert "$RESULT_SERVER_CLIENT_CERT" --key "$RESULT_SERVER_CLIENT_KEY") +} + upload_estimation_artifacts() { local json_file="$1" local source_uuid="$2" @@ -41,8 +49,9 @@ upload_estimation_artifacts() { endpoints=("/api/ingest/estimation-artifacts" "/api/ingest/estimation-inputs") for endpoint in "${endpoints[@]}"; do - if response=$(curl --fail -sS -X POST "${RESULT_SERVER}${endpoint}" \ - -H "X-API-Key: ${RESULT_SERVER_KEY}" \ + local curl_auth_args=() + result_server_set_curl_args + if response=$(curl --fail -sS "${curl_auth_args[@]}" -X POST "${RESULT_SERVER}${endpoint}" \ -F "id=${source_uuid}" \ -F "file=@${archive}" 2>&1); then upload_ok=1 @@ -89,8 +98,9 @@ for json_file in results/estimate*.json; do // empty ' "$json_file") echo "Posting $json_file to ${RESULT_SERVER}/api/ingest/estimate" - curl --fail -sS -X POST "${RESULT_SERVER}/api/ingest/estimate" \ - -H "X-API-Key: ${RESULT_SERVER_KEY}" \ + curl_auth_args=() + result_server_set_curl_args + curl --fail -sS "${curl_auth_args[@]}" -X POST "${RESULT_SERVER}/api/ingest/estimate" \ -H "Content-Type: application/json" \ --data-binary @"$json_file" echo "" diff --git a/scripts/result_server/send_results.sh b/scripts/result_server/send_results.sh index bc36c17..37efe55 100644 --- a/scripts/result_server/send_results.sh +++ b/scripts/result_server/send_results.sh @@ -8,6 +8,14 @@ ls results/ meta_file="results/server_result_meta.json" echo "{}" > "$meta_file" +result_server_set_curl_args() { + if [[ -z "${RESULT_SERVER_CLIENT_CERT:-}" || -z "${RESULT_SERVER_CLIENT_KEY:-}" ]]; then + echo "ERROR: RESULT_SERVER_CLIENT_CERT and RESULT_SERVER_CLIENT_KEY must be set" >&2 + exit 1 + fi + curl_auth_args=(--cert "$RESULT_SERVER_CLIENT_CERT" --key "$RESULT_SERVER_CLIENT_KEY") +} + # Backfill profile_data for older result JSONs that were produced before # result.sh learned to embed profiler summaries. The summary comes from # bk_profiler_artifact/meta.json inside the matching padata archive; raw @@ -65,8 +73,9 @@ upload_padata_archive() { local response echo "Uploading $tgz_file with UUID $uuid" - if response=$(curl --fail -sS -X POST "${RESULT_SERVER}/api/ingest/padata" \ - -H "X-API-Key: ${RESULT_SERVER_KEY}" \ + local curl_auth_args=() + result_server_set_curl_args + if response=$(curl --fail -sS "${curl_auth_args[@]}" -X POST "${RESULT_SERVER}/api/ingest/padata" \ -F "id=${uuid}" \ -F "timestamp=${timestamp}" \ -F "file=@${tgz_file}" 2>&1); then @@ -120,8 +129,9 @@ for json_file in results/result*.json; do echo "Posting $json_file to ${RESULT_SERVER}/api/ingest/result" # Post JSON and capture response - response=$(curl --fail -sS -X POST "${RESULT_SERVER}/api/ingest/result" \ - -H "X-API-Key: ${RESULT_SERVER_KEY}" \ + curl_auth_args=() + result_server_set_curl_args + response=$(curl --fail -sS "${curl_auth_args[@]}" -X POST "${RESULT_SERVER}/api/ingest/result" \ -H "Content-Type: application/json" \ --data-binary @"$json_file") diff --git a/scripts/tests/test_process_and_send_results.sh b/scripts/tests/test_process_and_send_results.sh index ab6874a..71f1197 100644 --- a/scripts/tests/test_process_and_send_results.sh +++ b/scripts/tests/test_process_and_send_results.sh @@ -43,7 +43,8 @@ chmod -R a+rX "${TMP_DIR}/project/results" export PATH="${TMP_DIR}/bin:${PATH}" export RESULT_SERVER="https://example.invalid" -export RESULT_SERVER_KEY="dummy" +export RESULT_SERVER_CLIENT_CERT="${TMP_DIR}/client.crt" +export RESULT_SERVER_CLIENT_KEY="${TMP_DIR}/client.key" pushd "${TMP_DIR}/project" >/dev/null bash scripts/result_server/process_and_send_results.sh qws Fugaku cross qws_Fugaku_build qws_Fugaku_N1_P2_T3_run 12345 diff --git a/scripts/tests/test_send_estimate_artifacts.sh b/scripts/tests/test_send_estimate_artifacts.sh index 3d81a96..f44f5d5 100644 --- a/scripts/tests/test_send_estimate_artifacts.sh +++ b/scripts/tests/test_send_estimate_artifacts.sh @@ -75,11 +75,14 @@ export PATH="${TMP_DIR}/bin:${PATH}" export CURL_LOG="${TMP_DIR}/curl.log" export ESTIMATION_ARTIFACTS_TAR_LIST="${TMP_DIR}/estimation_artifacts_tar_list.txt" export RESULT_SERVER="https://result.example.test" -export RESULT_SERVER_KEY="dummy-key" +export RESULT_SERVER_CLIENT_CERT="${TMP_DIR}/client.crt" +export RESULT_SERVER_CLIENT_KEY="${TMP_DIR}/client.key" cd "$TMP_DIR" bash "${REPO_DIR}/scripts/result_server/send_estimate.sh" +grep -q -- '--cert' "$CURL_LOG" +grep -q -- '--key' "$CURL_LOG" grep -q '/api/ingest/estimate' "$CURL_LOG" grep -q '/api/ingest/estimation-artifacts' "$CURL_LOG" grep -q 'id=11111111-2222-3333-4444-555555555555' "$CURL_LOG" diff --git a/scripts/tests/test_send_results_profile_data.sh b/scripts/tests/test_send_results_profile_data.sh index ba0d8be..d911a39 100644 --- a/scripts/tests/test_send_results_profile_data.sh +++ b/scripts/tests/test_send_results_profile_data.sh @@ -201,7 +201,8 @@ EOF chmod +x "${TMP_DIR}/bin/curl" "${TMP_DIR}/bin/jq" "${TMP_DIR}/bin/python" "${TMP_DIR}/bin/python3" export PATH="${TMP_DIR}/bin:${PATH}" export RESULT_SERVER="https://example.invalid" -export RESULT_SERVER_KEY="dummy" +export RESULT_SERVER_CLIENT_CERT="${TMP_DIR}/client.crt" +export RESULT_SERVER_CLIENT_KEY="${TMP_DIR}/client.key" pushd "${TMP_DIR}" >/dev/null bash "${REPO_DIR}/scripts/result_server/send_results.sh" >/dev/null