Skip to content

Commit d79dbec

Browse files
ALikesToCodeALikesToCode
committed
fix(runner): isolate generator with privilege broker
1 parent 48b8cef commit d79dbec

30 files changed

Lines changed: 2792 additions & 567 deletions

‎Dockerfile.runner‎

Lines changed: 45 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,23 @@ RUN pnpm install --frozen-lockfile --filter @counterlab/hosted-runner... \
4040
--format=esm \
4141
--target=node22 \
4242
--outfile=/out/runner.mjs \
43+
&& node_modules/.pnpm/esbuild@0.28.1/node_modules/esbuild/bin/esbuild \
44+
services/hosted-runner/src/privsep-broker.ts \
45+
--bundle \
46+
--platform=node \
47+
--format=esm \
48+
--target=node22 \
49+
--outfile=/out/privsep.mjs \
50+
&& node_modules/.pnpm/esbuild@0.28.1/node_modules/esbuild/bin/esbuild \
51+
services/hosted-runner/src/privsep-client.ts \
52+
--bundle \
53+
--platform=node \
54+
--format=esm \
55+
--target=node22 \
56+
--outfile=/out/privsep-client.mjs \
4357
&& node --check /out/runner.mjs \
58+
&& node --check /out/privsep.mjs \
59+
&& node --check /out/privsep-client.mjs \
4460
&& mkdir -p /out/codex \
4561
&& cp -a /build/node_modules/.pnpm/@openai+codex@0.144.4-linux-x64/node_modules/@openai/codex/vendor/x86_64-unknown-linux-musl/. /out/codex/ \
4662
&& mkdir -p \
@@ -84,8 +100,10 @@ RUN chmod 0555 /usr/local/bin/node /opt/counterlab/landlock_launcher.py \
84100
&& test -x /usr/sbin/groupadd \
85101
&& test -x /usr/sbin/useradd \
86102
&& test -x /usr/bin/setpriv \
87-
&& groupadd --gid 10001 counterlab-codex \
88-
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin counterlab-codex \
103+
&& groupadd --gid 10001 counterlab-runner \
104+
&& useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin counterlab-runner \
105+
&& groupadd --gid 10002 counterlab-generator \
106+
&& useradd --uid 10002 --gid 10002 --no-create-home --shell /usr/sbin/nologin counterlab-generator \
89107
&& python -m venv /opt/counterlab-venv
90108

91109
WORKDIR /app
@@ -95,6 +113,8 @@ COPY fixtures/public/customer_churn.csv ./fixtures/public/customer_churn.csv
95113
COPY fixtures/public/fraud_rare_event.csv ./fixtures/public/fraud_rare_event.csv
96114
COPY --from=python-builder /wheelhouse/ /opt/counterlab-wheelhouse/
97115
COPY --from=node-builder /out/runner.mjs /app/runner.mjs
116+
COPY --from=node-builder /out/privsep.mjs /app/privsep.mjs
117+
COPY --from=node-builder /out/privsep-client.mjs /app/privsep-client.mjs
98118
COPY --from=node-builder /out/codex/ /opt/codex/
99119
COPY --from=node-builder /out/runtime-rootfs/ /
100120

@@ -110,11 +130,17 @@ RUN chmod -R a+rX /opt/codex \
110130
/repo/reachability-review.json \
111131
/repo/scripts/verify_scientific_runtime.py \
112132
/repo/scripts/probe_cpython_htmlparser_reachability.py \
113-
&& chown -R root:root /repo \
114-
&& chmod -R a=rX /repo \
115-
&& chown -R root:root /app \
116-
&& chmod -R a=rX /app \
117-
&& chmod 555 /app/runner.mjs \
133+
&& chown -R root:counterlab-runner /repo \
134+
&& chmod -R u=rX,g=rX,o= /repo \
135+
&& chown -R root:counterlab-runner /opt/counterlab-venv /opt/counterlab-wheelhouse \
136+
&& chmod -R u=rX,g=rX,o= /opt/counterlab-venv /opt/counterlab-wheelhouse \
137+
&& chown -R root:counterlab-generator /opt/codex \
138+
&& chmod -R u=rX,g=rX,o= /opt/codex \
139+
&& chown -R root:counterlab-runner /app \
140+
&& chmod -R u=rX,g=rX,o= /app \
141+
&& chown root:root /app/privsep.mjs \
142+
&& chmod 0500 /app/privsep.mjs \
143+
&& chmod 0440 /app/runner.mjs /app/privsep-client.mjs \
118144
&& chown root:root \
119145
/usr/lib \
120146
/usr/lib64 \
@@ -133,11 +159,18 @@ RUN chmod -R a+rX /opt/codex \
133159
/dev/shm \
134160
/dev/mqueue \
135161
/counterlab-runtime \
136-
&& chown -R counterlab-codex:counterlab-codex /work/jobs /run/counterlab-codex \
137-
&& chmod 700 /work/jobs /run/counterlab-codex
138-
139-
USER 10001:10001
162+
&& mkdir -p /run/counterlab-privsep \
163+
&& chown 10001:10002 /work/jobs \
164+
&& chmod 2710 /work/jobs \
165+
&& chown root:counterlab-generator /run/counterlab-codex \
166+
&& chmod 0710 /run/counterlab-codex \
167+
&& chown root:counterlab-runner /run/counterlab-privsep \
168+
&& chmod 0750 /run/counterlab-privsep \
169+
&& chown root:root /tmp \
170+
&& chmod 0555 /tmp
171+
172+
USER 0:0
140173

141174
EXPOSE 8080
142175

143-
ENTRYPOINT ["/usr/local/bin/node", "/app/runner.mjs"]
176+
ENTRYPOINT ["/usr/local/bin/node", "/app/privsep.mjs"]

‎apps/web/worker/deployment-config.test.ts‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -662,23 +662,26 @@ describe("Cloudflare static asset routing", () => {
662662
);
663663

664664
expect(dockerfile).toContain(
665-
'ENTRYPOINT ["/usr/local/bin/node", "/app/runner.mjs"]',
665+
'ENTRYPOINT ["/usr/local/bin/node", "/app/privsep.mjs"]',
666666
);
667667
expect(dockerfile).not.toContain('CMD ["node", "/app/runner.mjs"]');
668668
});
669669

670-
it("keeps immutable application files readable by the declared non-root user", () => {
670+
it("separates the root broker, non-root runner, and non-root generator paths", () => {
671671
const dockerfile = readFileSync(
672672
resolve(process.cwd(), "../../Dockerfile.runner"),
673673
"utf8",
674674
);
675675

676-
expect(dockerfile).toContain("chown -R root:root /app");
677-
expect(dockerfile).toContain("chmod -R a=rX /app");
678-
expect(dockerfile).toContain("chmod 555 /app/runner.mjs");
676+
expect(dockerfile).toContain("chown -R root:counterlab-runner /app");
677+
expect(dockerfile).toContain("chmod -R u=rX,g=rX,o= /app");
678+
expect(dockerfile).toContain("chmod 0500 /app/privsep.mjs");
679+
expect(dockerfile).toContain(
680+
"chmod 0440 /app/runner.mjs /app/privsep-client.mjs",
681+
);
679682
expect(dockerfile).toContain("/repo/fixtures/notebooks");
680683
expect(dockerfile).toContain("/repo/requirements.runner.lock.txt");
681-
expect(dockerfile).toContain("chmod -R a=rX /repo");
684+
expect(dockerfile).toContain("chmod -R u=rX,g=rX,o= /repo");
682685
expect(dockerfile).toContain("chmod 0555 /usr/local/bin/node");
683686
expect(dockerfile).toContain(
684687
"/usr/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2",
@@ -702,16 +705,13 @@ describe("Cloudflare static asset routing", () => {
702705
expect(dockerfile).toContain(
703706
"chmod 0555 /usr/local/bin/node /opt/counterlab/landlock_launcher.py",
704707
);
705-
expect(
706-
dockerfile.indexOf("/opt/counterlab/landlock_launcher.py"),
707-
).toBeLessThan(dockerfile.indexOf("USER 10001:10001"));
708-
expect(dockerfile.indexOf("chmod 0555 /usr/local/bin/node")).toBeLessThan(
709-
dockerfile.indexOf("USER 10001:10001"),
710-
);
711-
expect(dockerfile).not.toContain(
712-
"chown -R counterlab-codex:counterlab-codex /app",
708+
expect(dockerfile).toContain("groupadd --gid 10002 counterlab-generator");
709+
expect(dockerfile).toContain(
710+
"chown -R root:counterlab-generator /opt/codex",
713711
);
714-
expect(dockerfile).toContain("USER 10001:10001");
712+
expect(dockerfile).toContain("chown 10001:10002 /work/jobs");
713+
expect(dockerfile).toContain("chmod 0555 /tmp");
714+
expect(dockerfile).toContain("USER 0:0");
715715
});
716716

717717
it("executes the real image entrypoint as its declared user before qualification", () => {

‎apps/web/worker/runner-container-env.test.ts‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,18 @@ describe("runner Container environment", () => {
2727
COUNTERLAB_RUNNER_VERIFYING_PUBLIC_KEY: signingKeys.publicKey,
2828
NODE_ENV: "production",
2929
PORT: "8080",
30-
COUNTERLAB_RUNNER_WORK_ROOT: "/tmp/counterlab-jobs",
31-
COUNTERLAB_CODEX_HOME_ROOT: "/tmp/counterlab-codex",
32-
COUNTERLAB_CODEX_ROOT: "/opt/codex",
33-
COUNTERLAB_CODEX_EXECUTABLE: "/usr/local/bin/codex",
34-
COUNTERLAB_LANDLOCK_LAUNCHER: "/opt/counterlab/landlock_launcher.py",
30+
COUNTERLAB_RUNNER_WORK_ROOT: "/work/jobs",
31+
COUNTERLAB_CODEX_HOME_ROOT: "/run/counterlab-codex",
32+
COUNTERLAB_CODEX_EXECUTABLE: "/opt/codex/bin/codex",
33+
COUNTERLAB_RUNNER_NODE_EXECUTABLE: "/usr/local/bin/node",
34+
COUNTERLAB_PRIVSEP_CLIENT_BUNDLE: "/app/privsep-client.mjs",
35+
COUNTERLAB_PRIVSEP_SOCKET: "/run/counterlab-privsep/broker.sock",
3536
COUNTERLAB_SETPRIV_EXECUTABLE: "/usr/bin/setpriv",
3637
COUNTERLAB_PYTHON_EXECUTABLE: "/opt/counterlab-venv/bin/python",
37-
COUNTERLAB_CODEX_UID: "10001",
38-
COUNTERLAB_CODEX_GID: "10001",
38+
COUNTERLAB_RUNNER_UID: "10001",
39+
COUNTERLAB_RUNNER_GID: "10001",
40+
COUNTERLAB_GENERATOR_UID: "10002",
41+
COUNTERLAB_GENERATOR_GID: "10002",
3942
COUNTERLAB_RUNNER_ONE_SHOT: "1",
4043
COUNTERLAB_RUNNER_SOURCE_COMMIT: "b".repeat(40),
4144
COUNTERLAB_RUNNER_IMAGE_DIGEST: `sha256:${"c".repeat(64)}`,

‎apps/web/worker/runner-container-env.ts‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,18 @@ export function createRunnerContainerEnvVars(
2525
COUNTERLAB_RUNNER_VERIFYING_PUBLIC_KEY: runnerVerifyingPublicKey,
2626
NODE_ENV: "production",
2727
PORT: "8080",
28-
COUNTERLAB_RUNNER_WORK_ROOT: "/tmp/counterlab-jobs",
29-
COUNTERLAB_CODEX_HOME_ROOT: "/tmp/counterlab-codex",
30-
COUNTERLAB_CODEX_ROOT: "/opt/codex",
31-
COUNTERLAB_CODEX_EXECUTABLE: "/usr/local/bin/codex",
32-
COUNTERLAB_LANDLOCK_LAUNCHER: "/opt/counterlab/landlock_launcher.py",
28+
COUNTERLAB_RUNNER_WORK_ROOT: "/work/jobs",
29+
COUNTERLAB_CODEX_HOME_ROOT: "/run/counterlab-codex",
30+
COUNTERLAB_CODEX_EXECUTABLE: "/opt/codex/bin/codex",
31+
COUNTERLAB_RUNNER_NODE_EXECUTABLE: "/usr/local/bin/node",
32+
COUNTERLAB_PRIVSEP_CLIENT_BUNDLE: "/app/privsep-client.mjs",
33+
COUNTERLAB_PRIVSEP_SOCKET: "/run/counterlab-privsep/broker.sock",
3334
COUNTERLAB_SETPRIV_EXECUTABLE: "/usr/bin/setpriv",
3435
COUNTERLAB_PYTHON_EXECUTABLE: "/opt/counterlab-venv/bin/python",
35-
COUNTERLAB_CODEX_UID: "10001",
36-
COUNTERLAB_CODEX_GID: "10001",
36+
COUNTERLAB_RUNNER_UID: "10001",
37+
COUNTERLAB_RUNNER_GID: "10001",
38+
COUNTERLAB_GENERATOR_UID: "10002",
39+
COUNTERLAB_GENERATOR_GID: "10002",
3740
COUNTERLAB_RUNNER_ONE_SHOT: "1",
3841
COUNTERLAB_RUNNER_SOURCE_COMMIT:
3942
bindings.COUNTERLAB_RUNNER_SOURCE_COMMIT ?? "",

‎apps/web/worker/runner-control-plane.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ describe("HttpRunnerDispatcher", () => {
116116
},
117117
startOptions: {
118118
envVars: {},
119-
entrypoint: ["/usr/local/bin/node", "/app/runner.mjs"],
119+
entrypoint: ["/usr/local/bin/node", "/app/privsep.mjs"],
120120
},
121121
});
122122
expect(fetch).toHaveBeenCalledWith("http://runner.internal/ready", {
@@ -436,7 +436,7 @@ describe("HttpRunnerDispatcher", () => {
436436
},
437437
startOptions: {
438438
envVars: environment,
439-
entrypoint: ["/usr/local/bin/node", "/app/runner.mjs"],
439+
entrypoint: ["/usr/local/bin/node", "/app/privsep.mjs"],
440440
},
441441
});
442442
expect(startAndWaitForPorts.mock.invocationCallOrder[0]).toBeLessThan(

‎apps/web/worker/runner-control-plane.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@ export class CloudflareContainerRunnerDispatcher implements RunnerDispatcher {
357357
},
358358
startOptions: {
359359
envVars: this.containerEnvironment,
360-
entrypoint: ["/usr/local/bin/node", "/app/runner.mjs"],
360+
entrypoint: ["/usr/local/bin/node", "/app/privsep.mjs"],
361361
},
362362
});
363363
} catch (error) {
@@ -406,7 +406,7 @@ export class CloudflareContainerRunnerDispatcher implements RunnerDispatcher {
406406
},
407407
startOptions: {
408408
envVars: this.containerEnvironment,
409-
entrypoint: ["/usr/local/bin/node", "/app/runner.mjs"],
409+
entrypoint: ["/usr/local/bin/node", "/app/privsep.mjs"],
410410
},
411411
});
412412
} catch (error) {

‎packages/scientific-engine-registry/src/schema.ts‎

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -742,17 +742,105 @@ export const GenerationIsolationEvidenceV2Schema = z
742742
}
743743
});
744744

745+
export const GENERATION_ISOLATION_PROBE_VERSION_V3 =
746+
"counterlab-generation-isolation-v3" as const;
747+
748+
export const GENERATION_ISOLATION_POLICY_VERSION_V3 =
749+
"counterlab-posix-dac-process-policy-v1" as const;
750+
751+
const GenerationIsolationProcessIdentitySchema = z.strictObject({
752+
brokerUid: z.literal(0),
753+
brokerGid: z.literal(0),
754+
runnerUid: z.literal(10_001),
755+
runnerGid: z.literal(10_001),
756+
generatorUid: z.literal(10_002),
757+
generatorGid: z.literal(10_002),
758+
generatorSupplementaryGroupsCleared: z.literal(true),
759+
generatorCapabilitiesEmpty: z.literal(true),
760+
generatorNoNewPrivileges: z.literal(true),
761+
protectedPathsUnreadable: z.literal(true),
762+
protectedPathsUnwritable: z.literal(true),
763+
parentEnvironmentUnreadable: z.literal(true),
764+
brokerEnvironmentUnreadable: z.literal(true),
765+
workspaceVisible: z.literal(true),
766+
workspaceWritable: z.literal(true),
767+
outsideWorkspaceWritesDenied: z.literal(true),
768+
fixedKernelUnavailable: z.literal(true),
769+
credentialReadOnlyDuringInitialization: z.literal(true),
770+
credentialRevocationSupported: z.literal(true),
771+
boundedLaunchesEnforced: z.literal(true),
772+
maximumLaunches: z.literal(3),
773+
});
774+
775+
export const GenerationIsolationProbePayloadV3Schema = z.strictObject({
776+
schemaVersion: z.literal("3"),
777+
probeVersion: z.literal(GENERATION_ISOLATION_PROBE_VERSION_V3),
778+
service: z.literal("counterlab-hosted-runner"),
779+
probe: z.literal("non-root-startup"),
780+
checks: z.tuple([
781+
z.literal("entrypoint"),
782+
z.literal("non-root-user"),
783+
z.literal("immutable-paths"),
784+
z.literal("python"),
785+
z.literal("setpriv"),
786+
z.literal("privsep-broker"),
787+
z.literal("posix-dac-process-identity"),
788+
z.literal("writable-roots"),
789+
]),
790+
generationFilesystemReadIsolation: z.literal("OS_ENFORCED"),
791+
mechanism: z.literal("posix-dac-process-identity"),
792+
brokerIdentity: z.literal("0:0"),
793+
runnerIdentity: z.literal("10001:10001"),
794+
generatorIdentity: z.literal("10002:10002"),
795+
policyVersion: z.literal(GENERATION_ISOLATION_POLICY_VERSION_V3),
796+
processIdentity: GenerationIsolationProcessIdentitySchema,
797+
});
798+
799+
export const GenerationIsolationEvidenceV3Schema = z
800+
.strictObject({
801+
schemaVersion: z.literal("3"),
802+
status: z.literal("VERIFIED"),
803+
generationFilesystemReadIsolation: z.literal("OS_ENFORCED"),
804+
sourceCommit: GitCommitSchema,
805+
sourceTreeSha256: Sha256Schema,
806+
localImageTag: z.string().regex(/^counterlab-runner:git-[a-f0-9]{40}$/),
807+
localImageDigest: OciDigestSchema,
808+
imageUser: z.literal("0:0"),
809+
policyVersion: z.literal(GENERATION_ISOLATION_POLICY_VERSION_V3),
810+
probePayload: GenerationIsolationProbePayloadV3Schema,
811+
probePayloadSha256: Sha256Schema,
812+
verifiedAt: z.iso.datetime({ offset: true }),
813+
verifierVersion: z.literal("counterlab-generation-isolation-evidence-v3"),
814+
})
815+
.superRefine((evidence, context) => {
816+
if (
817+
evidence.localImageTag !==
818+
`counterlab-runner:git-${evidence.sourceCommit}`
819+
) {
820+
context.addIssue({
821+
code: "custom",
822+
path: ["localImageTag"],
823+
message: "generation-isolation image tag must bind the source commit",
824+
});
825+
}
826+
});
827+
745828
export const GenerationIsolationProbePayloadAnySchema = z.discriminatedUnion(
746829
"schemaVersion",
747830
[
748831
GenerationIsolationProbePayloadSchema,
749832
GenerationIsolationProbePayloadV2Schema,
833+
GenerationIsolationProbePayloadV3Schema,
750834
],
751835
);
752836

753837
export const GenerationIsolationEvidenceSchema = z.discriminatedUnion(
754838
"schemaVersion",
755-
[GenerationIsolationEvidenceV1Schema, GenerationIsolationEvidenceV2Schema],
839+
[
840+
GenerationIsolationEvidenceV1Schema,
841+
GenerationIsolationEvidenceV2Schema,
842+
GenerationIsolationEvidenceV3Schema,
843+
],
756844
);
757845

758846
export const QualifiedRunnerReleaseV1Schema = z

‎packages/scientific-engine-registry/src/types.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,10 @@ import {
1212
GenerationIsolationEvidenceSchema,
1313
GenerationIsolationEvidenceV1Schema,
1414
GenerationIsolationEvidenceV2Schema,
15+
GenerationIsolationEvidenceV3Schema,
1516
GenerationIsolationProbePayloadSchema,
1617
GenerationIsolationProbePayloadV2Schema,
18+
GenerationIsolationProbePayloadV3Schema,
1719
ScientificEngineDescriptorSchema,
1820
ScientificEngineEvidenceCatalogSchema,
1921
ScientificEngineEvidenceKindSchema,
@@ -62,12 +64,18 @@ export type GenerationIsolationEvidenceV1 = z.infer<
6264
export type GenerationIsolationEvidenceV2 = z.infer<
6365
typeof GenerationIsolationEvidenceV2Schema
6466
>;
67+
export type GenerationIsolationEvidenceV3 = z.infer<
68+
typeof GenerationIsolationEvidenceV3Schema
69+
>;
6570
export type GenerationIsolationProbePayload = z.infer<
6671
typeof GenerationIsolationProbePayloadSchema
6772
>;
6873
export type GenerationIsolationProbePayloadV2 = z.infer<
6974
typeof GenerationIsolationProbePayloadV2Schema
7075
>;
76+
export type GenerationIsolationProbePayloadV3 = z.infer<
77+
typeof GenerationIsolationProbePayloadV3Schema
78+
>;
7179

7280
export type ScientificEngineRole = z.infer<typeof ScientificEngineRoleSchema>;
7381
export type ScientificEngineDescriptor = z.infer<

0 commit comments

Comments
 (0)