diff --git a/docs/guides/portal-execution-profiles-handoff.md b/docs/guides/portal-execution-profiles-handoff.md index 97a3ef8..3b78778 100644 --- a/docs/guides/portal-execution-profiles-handoff.md +++ b/docs/guides/portal-execution-profiles-handoff.md @@ -28,11 +28,11 @@ Recommended order: 2. Confirm the admin page can create and update profiles in the live SQLite DB. 3. Add an execution request table for Portal-triggered runs. 4. Add a dry-run submit view that resolves a profile and shows the GitLab - Pipeline API payload without sending it. -5. Add the real GitLab Pipeline API trigger only after the dry-run path is + pipeline trigger payload without sending it. +5. Add the real GitLab pipeline trigger only after the dry-run path is reviewed. Keep the trigger token in the site-local service environment as - `RESULT_SERVER_GITLAB_TOKEN`; do not store it in SQLite, logs, or the OSS - repository. + `RESULT_SERVER_GITLAB_TRIGGER_TOKEN`; do not store it in SQLite, logs, or + the OSS repository. 6. Index received benchmark and estimation JSON metadata into SQLite while keeping JSON/tgz artifacts as raw records. 7. Add environment snapshot storage after deciding which host/runtime metadata @@ -42,7 +42,7 @@ GitLab schedules should not be the primary governance point. The Portal should own periodic and event-triggered execution decisions, then trigger GitLab CI with resolved site-local variables. -## GitLab Pipeline API Configuration +## GitLab Pipeline Trigger Configuration Dry-run payload rendering requires: @@ -53,27 +53,27 @@ RESULT_SERVER_GITLAB_REPO=gitlab.example.org/group/project Actual submission also requires: ```text -RESULT_SERVER_GITLAB_TOKEN= +RESULT_SERVER_GITLAB_TRIGGER_TOKEN= ``` -`RESULT_SERVER_GITLAB_REPO` is a scheme-less `host/path` value. The token must -have permission to create pipelines in that GitLab project. The Portal records -the request payload, GitLab response metadata, status, and errors in -`execution_requests`; it must not record the token value. +`RESULT_SERVER_GITLAB_REPO` is a scheme-less `host/path` value. The trigger +token must be created for that GitLab project. The Portal records the request +payload, GitLab response metadata, status, and errors in `execution_requests`; +it must not record the token value. For multiple destinations, configure named targets instead of the single-repo fallback: ```text RESULT_SERVER_GITLAB_TARGETS=swc=gitlab.swc.example.org/group/project,gitlab_com=gitlab.com/group/project -RESULT_SERVER_GITLAB_TOKEN_SWC= -RESULT_SERVER_GITLAB_TOKEN_GITLAB_COM= +RESULT_SERVER_GITLAB_TRIGGER_TOKEN_SWC= +RESULT_SERVER_GITLAB_TRIGGER_TOKEN_GITLAB_COM= ``` Target IDs may contain letters, digits, `_`, `.`, and `-`. The token variable is -`RESULT_SERVER_GITLAB_TOKEN_` with non-alphanumeric characters -converted to `_` and uppercased. Store these variables in the Portal systemd -`EnvironmentFile`, not in the repository. +`RESULT_SERVER_GITLAB_TRIGGER_TOKEN_` with non-alphanumeric +characters converted to `_` and uppercased. Store these variables in the Portal +systemd `EnvironmentFile`, not in the repository. ## Compatibility Expectations diff --git a/result_server/routes/admin.py b/result_server/routes/admin.py index 11a706c..d5fa851 100644 --- a/result_server/routes/admin.py +++ b/result_server/routes/admin.py @@ -30,7 +30,7 @@ build_pipeline_plan, configured_gitlab_target, configured_gitlab_targets, - configured_gitlab_token, + configured_gitlab_trigger_token, submit_pipeline_plan, ) from utils.rate_limit import rate_limited @@ -295,7 +295,7 @@ def upsert_execution_profile(): @admin_required @rate_limited(max_per_minute=20, key_fn=_admin_rate_key, scope="admin_write") def dry_run_execution_profile_submit(): - """Resolve an execution profile and render a GitLab Pipeline API dry run.""" + """Resolve an execution profile and render a GitLab trigger dry run.""" db_path = current_app.config.get("EXECUTION_PROFILE_DB_PATH") store = ExecutionProfileStore(db_path) submit_plan = _build_execution_pipeline_plan(store) @@ -388,7 +388,7 @@ def submit_execution_profile_pipeline(): if not errors: submit_result = submit_pipeline_plan( plan, - token=configured_gitlab_token(gitlab_target), + token=configured_gitlab_trigger_token(gitlab_target), ) errors.extend(submit_result.errors) diff --git a/result_server/templates/admin_execution_profiles.html b/result_server/templates/admin_execution_profiles.html index 70ff6e3..9f2dde0 100644 --- a/result_server/templates/admin_execution_profiles.html +++ b/result_server/templates/admin_execution_profiles.html @@ -249,10 +249,10 @@

Create / Update Profile

-

GitLab Pipeline Submit

+

GitLab Pipeline Trigger

Resolve an approved profile for a target scope and preview the GitLab - Pipeline API request before submitting it. + pipeline trigger request before submitting it.

{% if csrf_token is defined %}{% endif %} @@ -340,7 +340,7 @@

GitLab Pipeline Submit

{% endif %} -

API URL: {{ dry_run_result.api_url or 'not configured' }}

+

Trigger API URL: {{ dry_run_result.api_url or 'not configured' }}

{{ dry_run_result.payload_json }}
{% endif %} @@ -399,7 +399,7 @@

GitLab Pipeline Submit

confirm submit - +
diff --git a/result_server/tests/test_execution_profiles.py b/result_server/tests/test_execution_profiles.py index d225e04..78d8c19 100644 --- a/result_server/tests/test_execution_profiles.py +++ b/result_server/tests/test_execution_profiles.py @@ -8,6 +8,7 @@ import sqlite3 import sys import tempfile +import urllib.parse sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) @@ -26,7 +27,7 @@ build_pipeline_plan, configured_gitlab_target, configured_gitlab_targets, - configured_gitlab_token, + configured_gitlab_trigger_token, submit_pipeline_plan, ) @@ -402,7 +403,7 @@ def test_admin_execution_profiles_dry_run_submit_renders_payload(tmp_path, monke assert resp.status_code == 200 assert "Dry-run request #1" in html assert "dry_run_ready" in html - assert "https://gitlab.example.org/api/v4/projects/group%2Fbenchkit/pipeline" in html + assert "https://gitlab.example.org/api/v4/projects/group%2Fbenchkit/trigger/pipeline" in html assert "--account=site-local" in html with sqlite3.connect(db_path) as conn: @@ -412,12 +413,8 @@ def test_admin_execution_profiles_dry_run_submit_renders_payload(tmp_path, monke assert row[:4] == ("dry_run_ready", "rikyu-qws-nightly", "qws", "RIKYU") payload_record = json.loads(row[4]) variables = payload_record["payload"]["variables"] - assert {"key": "code", "value": "qws", "variable_type": "env_var"} in variables - assert { - "key": "BK_SCHEDULER_EXTRA_ARGS_RIKYU", - "value": "--account=site-local", - "variable_type": "env_var", - } in variables + assert variables["code"] == "qws" + assert variables["BK_SCHEDULER_EXTRA_ARGS_RIKYU"] == "--account=site-local" finally: _cleanup(temp_dirs) @@ -445,7 +442,7 @@ def test_admin_execution_profiles_dry_run_blocks_without_matching_profile( _cleanup(temp_dirs) -def test_gitlab_pipeline_submit_posts_private_token_without_storing_it(monkeypatch): +def test_gitlab_pipeline_submit_posts_trigger_token_without_storing_it(monkeypatch): plan = build_pipeline_plan( gitlab_repo="gitlab.example.org/group/benchkit.git", target_ref="develop", @@ -478,9 +475,13 @@ def fake_urlopen(request, timeout): assert result.ok is True assert result.status_code == 201 assert result.response["id"] == 123 - assert captured["url"] == "https://gitlab.example.org/api/v4/projects/group%2Fbenchkit/pipeline" - assert captured["headers"]["Private-token"] == "secret-token" - assert json.loads(captured["data"])["ref"] == "develop" + assert captured["url"] == "https://gitlab.example.org/api/v4/projects/group%2Fbenchkit/trigger/pipeline" + assert "Private-token" not in captured["headers"] + assert captured["headers"]["Content-type"] == "application/x-www-form-urlencoded" + fields = urllib.parse.parse_qs(captured["data"]) + assert fields["token"] == ["secret-token"] + assert fields["ref"] == ["develop"] + assert fields["variables[code]"] == ["qws"] def test_gitlab_pipeline_submit_blocks_without_token(): @@ -494,7 +495,7 @@ def test_gitlab_pipeline_submit_blocks_without_token(): assert result.ok is False assert result.status_code == 0 - assert result.errors == ["RESULT_SERVER_GITLAB_TOKEN is not set"] + assert result.errors == ["RESULT_SERVER_GITLAB_TRIGGER_TOKEN is not set"] def test_gitlab_pipeline_targets_parse_multiple_destinations(): @@ -503,8 +504,8 @@ def test_gitlab_pipeline_targets_parse_multiple_destinations(): "swc=gitlab.swc.example.org/fugakunext/benchmark/benchkit," "gitlab_com=gitlab.com/yoshifuminakamura/benchkit" ), - "RESULT_SERVER_GITLAB_TOKEN_SWC": "swc-token", - "RESULT_SERVER_GITLAB_TOKEN_GITLAB_COM": "com-token", + "RESULT_SERVER_GITLAB_TRIGGER_TOKEN_SWC": "swc-token", + "RESULT_SERVER_GITLAB_TRIGGER_TOKEN_GITLAB_COM": "com-token", } targets, errors = configured_gitlab_targets(env) @@ -512,10 +513,10 @@ def test_gitlab_pipeline_targets_parse_multiple_destinations(): assert errors == [] assert [target.id for target in targets] == ["swc", "gitlab_com"] - assert targets[0].token_env == "RESULT_SERVER_GITLAB_TOKEN_SWC" + assert targets[0].token_env == "RESULT_SERVER_GITLAB_TRIGGER_TOKEN_SWC" assert selected_errors == [] assert selected.repo == "gitlab.com/yoshifuminakamura/benchkit" - assert configured_gitlab_token(selected, env) == "com-token" + assert configured_gitlab_trigger_token(selected, env) == "com-token" def test_admin_execution_profiles_submit_posts_pipeline_and_records_request( @@ -523,14 +524,14 @@ def test_admin_execution_profiles_submit_posts_pipeline_and_records_request( monkeypatch, ): monkeypatch.setenv("RESULT_SERVER_GITLAB_REPO", "gitlab.example.org/group/benchkit.git") - monkeypatch.setenv("RESULT_SERVER_GITLAB_TOKEN", "secret-token") + monkeypatch.setenv("RESULT_SERVER_GITLAB_TRIGGER_TOKEN", "secret-token") db_path = tmp_path / "cx_portal.sqlite3" ExecutionProfileStore(str(db_path)).upsert_profile(_profile(), actor="admin") app, temp_dirs = _admin_app(db_path) def fake_submit(plan, *, token): assert token == "secret-token" - assert plan.api_url == "https://gitlab.example.org/api/v4/projects/group%2Fbenchkit/pipeline" + assert plan.api_url == "https://gitlab.example.org/api/v4/projects/group%2Fbenchkit/trigger/pipeline" return GitLabPipelineSubmitResult( status_code=201, response={"id": 123, "web_url": "https://gitlab.example.org/p/123"}, @@ -582,13 +583,13 @@ def test_admin_execution_profiles_submit_uses_selected_gitlab_target( monkeypatch, ): monkeypatch.delenv("RESULT_SERVER_GITLAB_REPO", raising=False) - monkeypatch.delenv("RESULT_SERVER_GITLAB_TOKEN", raising=False) + monkeypatch.delenv("RESULT_SERVER_GITLAB_TRIGGER_TOKEN", raising=False) monkeypatch.setenv( "RESULT_SERVER_GITLAB_TARGETS", "swc=gitlab.swc.example.org/fugakunext/benchmark/benchkit," "gitlab_com=gitlab.com/yoshifuminakamura/benchkit", ) - monkeypatch.setenv("RESULT_SERVER_GITLAB_TOKEN_GITLAB_COM", "com-token") + monkeypatch.setenv("RESULT_SERVER_GITLAB_TRIGGER_TOKEN_GITLAB_COM", "com-token") db_path = tmp_path / "cx_portal.sqlite3" ExecutionProfileStore(str(db_path)).upsert_profile(_profile(), actor="admin") app, temp_dirs = _admin_app(db_path) @@ -596,7 +597,7 @@ def test_admin_execution_profiles_submit_uses_selected_gitlab_target( def fake_submit(plan, *, token): assert token == "com-token" assert plan.target_id == "gitlab_com" - assert plan.api_url == "https://gitlab.com/api/v4/projects/yoshifuminakamura%2Fbenchkit/pipeline" + assert plan.api_url == "https://gitlab.com/api/v4/projects/yoshifuminakamura%2Fbenchkit/trigger/pipeline" return GitLabPipelineSubmitResult( status_code=201, response={"id": 456, "web_url": "https://gitlab.com/p/456"}, @@ -637,7 +638,7 @@ def fake_submit(plan, *, token): def test_admin_execution_profiles_submit_requires_confirmation(tmp_path, monkeypatch): monkeypatch.setenv("RESULT_SERVER_GITLAB_REPO", "gitlab.example.org/group/benchkit.git") - monkeypatch.setenv("RESULT_SERVER_GITLAB_TOKEN", "secret-token") + monkeypatch.setenv("RESULT_SERVER_GITLAB_TRIGGER_TOKEN", "secret-token") db_path = tmp_path / "cx_portal.sqlite3" ExecutionProfileStore(str(db_path)).upsert_profile(_profile(), actor="admin") app, temp_dirs = _admin_app(db_path) diff --git a/result_server/utils/gitlab_pipeline.py b/result_server/utils/gitlab_pipeline.py index 40d8e17..b33efe1 100644 --- a/result_server/utils/gitlab_pipeline.py +++ b/result_server/utils/gitlab_pipeline.py @@ -1,4 +1,4 @@ -"""GitLab Pipeline API request planning helpers for Portal-triggered runs.""" +"""GitLab pipeline trigger request planning helpers for Portal-triggered runs.""" from __future__ import annotations @@ -26,7 +26,7 @@ class GitLabPipelineTarget: @dataclass(frozen=True) class GitLabPipelinePlan: - """A dry-run representation of a GitLab Pipeline API request.""" + """A dry-run representation of a GitLab pipeline trigger request.""" api_url: str payload: dict[str, Any] @@ -37,7 +37,7 @@ class GitLabPipelinePlan: @dataclass(frozen=True) class GitLabPipelineSubmitResult: - """Result of submitting a GitLab Pipeline API request.""" + """Result of submitting a GitLab pipeline trigger request.""" status_code: int response: dict[str, Any] @@ -59,7 +59,7 @@ def configured_gitlab_repo(env: dict[str, str] | None = None) -> str: def _target_token_env(target_id: str) -> str: normalized = re.sub(r"[^A-Za-z0-9]", "_", target_id).upper() - return f"RESULT_SERVER_GITLAB_TOKEN_{normalized}" + return f"RESULT_SERVER_GITLAB_TRIGGER_TOKEN_{normalized}" def configured_gitlab_targets( @@ -105,7 +105,7 @@ def configured_gitlab_targets( GitLabPipelineTarget( id="default", repo=repo, - token_env="RESULT_SERVER_GITLAB_TOKEN", + token_env="RESULT_SERVER_GITLAB_TRIGGER_TOKEN", ) ) return targets, errors @@ -132,19 +132,19 @@ def configured_gitlab_target( return None, [f"unknown GitLab target: {selected}"] -def configured_gitlab_token( +def configured_gitlab_trigger_token( target: GitLabPipelineTarget | None = None, env: dict[str, str] | None = None, ) -> str: - """Return the configured GitLab API token for Portal submits.""" + """Return the configured GitLab trigger token for Portal submits.""" source = env if env is not None else os.environ if target is not None: token = source.get(target.token_env, "").strip() if token: return token - if target.token_env != "RESULT_SERVER_GITLAB_TOKEN": + if target.token_env != "RESULT_SERVER_GITLAB_TRIGGER_TOKEN": return "" - return source.get("RESULT_SERVER_GITLAB_TOKEN", "").strip() + return source.get("RESULT_SERVER_GITLAB_TRIGGER_TOKEN", "").strip() def _split_gitlab_repo(repo: str) -> tuple[str, str] | None: @@ -157,11 +157,11 @@ def _split_gitlab_repo(repo: str) -> tuple[str, str] | None: return host, project_path -def _add_variable(variables: list[dict[str, str]], key: str, value: str) -> None: +def _add_variable(variables: dict[str, str], key: str, value: str) -> None: text = str(value or "").strip() if not text: return - variables.append({"key": key, "value": text, "variable_type": "env_var"}) + variables[key] = text def _scheduler_extra_args_key(system: str) -> str: @@ -182,7 +182,7 @@ def build_pipeline_plan( scheduler_extra_args: str = "", target_id: str = "", ) -> GitLabPipelinePlan: - """Build the GitLab Pipeline API URL and JSON payload without sending it.""" + """Build the GitLab trigger API URL and form-like payload without sending it.""" errors: list[str] = [] warnings: list[str] = [] ref = target_ref.strip() or "develop" @@ -195,9 +195,9 @@ def build_pipeline_plan( else: host, project_path = split_repo encoded_project = urllib.parse.quote(project_path, safe="") - api_url = f"https://{host}/api/v4/projects/{encoded_project}/pipeline" + api_url = f"https://{host}/api/v4/projects/{encoded_project}/trigger/pipeline" - variables: list[dict[str, str]] = [] + variables: dict[str, str] = {} _add_variable(variables, "code", code) _add_variable(variables, "system", system) _add_variable(variables, "app", app) @@ -233,25 +233,25 @@ def submit_pipeline_plan( timeout: float = 20.0, urlopen=urllib.request.urlopen, ) -> GitLabPipelineSubmitResult: - """Submit a planned GitLab Pipeline API request.""" + """Submit a planned GitLab trigger API request.""" errors = list(plan.errors) if not token: if plan.target_id: - errors.append(f"GitLab token is not set for target: {plan.target_id}") + errors.append(f"GitLab trigger token is not set for target: {plan.target_id}") else: - errors.append("RESULT_SERVER_GITLAB_TOKEN is not set") + errors.append("RESULT_SERVER_GITLAB_TRIGGER_TOKEN is not set") if not plan.api_url: - errors.append("GitLab Pipeline API URL is not configured") + errors.append("GitLab trigger API URL is not configured") if errors: return GitLabPipelineSubmitResult(status_code=0, response={}, errors=errors) - body = json_dumps_bytes(plan.payload) + form_fields = _trigger_form_fields(plan.payload, token) + body = urllib.parse.urlencode(form_fields).encode("utf-8") request = urllib.request.Request( plan.api_url, data=body, headers={ - "PRIVATE-TOKEN": token, - "Content-Type": "application/json", + "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json", }, method="POST", @@ -266,24 +266,24 @@ def submit_pipeline_plan( return GitLabPipelineSubmitResult( status_code=status_code, response=payload, - errors=[f"GitLab Pipeline API returned HTTP {status_code}"], + errors=[f"GitLab trigger API returned HTTP {status_code}"], ) except urllib.error.URLError as exc: return GitLabPipelineSubmitResult( status_code=0, response={}, - errors=[f"GitLab Pipeline API request failed: {exc.reason}"], + errors=[f"GitLab trigger API request failed: {exc.reason}"], ) except TimeoutError: return GitLabPipelineSubmitResult( status_code=0, response={}, - errors=["GitLab Pipeline API request timed out"], + errors=["GitLab trigger API request timed out"], ) errors = [] if not 200 <= status_code < 300: - errors.append(f"GitLab Pipeline API returned HTTP {status_code}") + errors.append(f"GitLab trigger API returned HTTP {status_code}") return GitLabPipelineSubmitResult( status_code=status_code, response=payload, @@ -291,11 +291,17 @@ def submit_pipeline_plan( ) -def json_dumps_bytes(payload: dict[str, Any]) -> bytes: - """Encode a JSON payload for urllib.""" - import json - - return json.dumps(payload).encode("utf-8") +def _trigger_form_fields(payload: dict[str, Any], token: str) -> dict[str, str]: + """Convert a dry-run payload into GitLab trigger API form fields.""" + fields = { + "token": token, + "ref": str(payload.get("ref", "")), + } + variables = payload.get("variables", {}) + if isinstance(variables, dict): + for key, value in variables.items(): + fields[f"variables[{key}]"] = str(value) + return fields def _decode_json_response(raw: bytes) -> dict[str, Any]: