Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/cloud/src/auth/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ export const withServiceLogging = <A, E, R>(
effect: Effect.Effect<A, unknown, R>,
): Effect.Effect<A, E, R> =>
effect.pipe(
Effect.tapCause((cause) => Effect.logError(`${name} failed`, cause)),
Effect.tapCause(() => Effect.logError(`${name} failed`)),
Effect.mapError(publicError),
Effect.withSpan(name),
) as Effect.Effect<A, E, R>;
10 changes: 5 additions & 5 deletions apps/cloud/src/auth/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -507,7 +507,7 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group(
Effect.gen(function* () {
yield* Effect.logWarning(
"createOrganization: could not provision the Autumn customer",
{ organizationId: org.id, error },
{ organizationId: org.id },
);
yield* captureCauseEffect(error);
}),
Expand Down Expand Up @@ -629,10 +629,10 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group(
{ organizationId },
),
),
Effect.tapError((error) =>
Effect.tapError(() =>
Effect.logError(
"deleteOrganization: org marked deleted but the Autumn customer could not be deleted; retry the deletion",
{ organizationId, error },
{ organizationId },
),
),
Effect.mapError(() => new OrganizationDeletionIncomplete({ step: "billing" })),
Expand Down Expand Up @@ -672,10 +672,10 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group(
s.deleteOrganizationCascade(organizationId, deletedAt),
)
.pipe(
Effect.tapError((error) =>
Effect.tapError(() =>
Effect.logError(
"deleteOrganization: org marked deleted, removed from WorkOS and Autumn, but local purge failed, tenant data and secrets orphaned; retry the deletion",
{ organizationId, error },
{ organizationId },
),
),
);
Expand Down
2 changes: 1 addition & 1 deletion apps/cloud/src/auth/workos-events-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ export const runWorkOsEventsSync = (): Promise<void> =>
Effect.scoped,
Effect.catchCause((cause) =>
Effect.gen(function* () {
yield* Effect.logError("workos_events: sync run failed", cause);
yield* Effect.logError("workos_events: sync run failed");
yield* captureCauseEffect(cause);
}),
),
Expand Down
9 changes: 9 additions & 0 deletions apps/cloud/src/edge/referrer-policy.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
/** Prevent document and redirect URLs (including OAuth codes) from becoming
* Referer headers. Preserve the response stream, cookies and status. WebSocket
* upgrades cannot navigate a browser and must retain their platform handle. */
export const withPrivateReferrerPolicy = (response: Response): Response => {
if (response.status === 101) return response;
const result = new Response(response.body, response);
result.headers.set("Referrer-Policy", "no-referrer");
return result;
};
2 changes: 1 addition & 1 deletion apps/cloud/src/engine/execution-gate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ export const makeExecutionLimitGate = (checkBalance: ExecutionBalanceCheck) => {
Effect.catch((error: unknown) =>
Effect.gen(function* () {
yield* Effect.sync(() => {
console.warn("[billing] execution balance check failed open:", error);
console.warn("[billing] execution balance check failed open");
});
yield* captureCauseEffect(error);
return { blocked: false } as const satisfies GateDecision;
Expand Down
3 changes: 1 addition & 2 deletions apps/cloud/src/engine/execution-rate-limit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ const failOpen = (
"rate_limit.check.error_tag": outcome.errorTag,
});
yield* Effect.sync(() => {
console.warn("[rate-limit] execution rate limit check failed open:", error);
console.warn("[rate-limit] execution rate limit check failed open");
});
if (!outcome.timedOut) yield* captureCauseEffect(error);
return { blocked: false } as const satisfies GateDecision;
Expand Down Expand Up @@ -303,7 +303,6 @@ export const makeExecutionRateLimiter = (
`[rate-limit] exemption lookup failed for ${organizationId}; treating as ${
cached ? "last known" : "not exempt"
}:`,
error,
);
});
yield* captureCauseEffect(error);
Expand Down
3 changes: 1 addition & 2 deletions apps/cloud/src/extensions/billing/member-seats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,9 @@ export const forkReportMemberSeats = (
waitUntil(Effect.runPromise(autumn.setMemberSeats(organizationId, seats)));
});
}).pipe(
Effect.catch((error) =>
Effect.catch(() =>
Effect.logWarning("reportMemberSeats: seat recount failed", {
organizationId,
error,
}),
),
Effect.withSpan("billing.reportMemberSeats"),
Expand Down
6 changes: 3 additions & 3 deletions apps/cloud/src/extensions/billing/route.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { env } from "cloudflare:workers";
import { Cause, Effect } from "effect";
import { Effect } from "effect";
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
import { autumnHandler } from "autumn-js/backend";

Expand Down Expand Up @@ -127,7 +127,7 @@ const handler = Effect.gen(function* () {
);

if (statusCode >= 400) {
console.error("[autumn] upstream error:", statusCode, response);
console.error("[autumn] upstream error", { status: statusCode });
return yield* new HttpResponseError({
status: statusCode,
code: "billing_request_failed",
Expand All @@ -139,7 +139,7 @@ const handler = Effect.gen(function* () {
}).pipe(
Effect.catchCause((err) => {
if (isServerError(err)) {
console.error("[autumn] request failed:", Cause.pretty(err));
console.error("[autumn] request failed", { status: 500 });
}
return toErrorServerResponseEffect(err);
}),
Expand Down
4 changes: 2 additions & 2 deletions apps/cloud/src/extensions/billing/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ const make = Effect.sync(() => {
// Silent billing data loss is worth paging on: autumn.trackExecution
// is fire-and-forget so the caller doesn't handle it themselves.
yield* Effect.sync(() => {
console.error("[billing] track failed:", error);
console.error("[billing] track failed");
});
yield* captureCauseEffect(error);
yield* Effect.annotateCurrentSpan({ "autumn.track.failed": true });
Expand Down Expand Up @@ -217,7 +217,7 @@ const make = Effect.sync(() => {
// Silent seat drift means wrong invoices, so failures page just
// like a lost execution track.
yield* Effect.sync(() => {
console.error("[billing] seat sync failed:", error);
console.error("[billing] seat sync failed");
});
yield* captureCauseEffect(error);
yield* Effect.annotateCurrentSpan({ "autumn.members.failed": true });
Expand Down
14 changes: 2 additions & 12 deletions apps/cloud/src/observability/error-logging.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,6 @@
import { Cause, Effect, Option, Predicate, Result } from "effect";
import { HttpRouter, HttpServerRequest } from "effect/unstable/http";

const MAX_LOGGED_CAUSE_CHARS = 4_000;

const truncate = (value: string): string =>
value.length <= MAX_LOGGED_CAUSE_CHARS
? value
: `${value.slice(0, MAX_LOGGED_CAUSE_CHARS)}\n...[truncated ${
value.length - MAX_LOGGED_CAUSE_CHARS
} chars]`;

const loggedCause = (cause: Cause.Cause<unknown>): string => truncate(Cause.pretty(cause));

const objectValue = (value: unknown, key: string): unknown =>
Predicate.hasProperty(value, key) ? value[key] : undefined;

Expand Down Expand Up @@ -65,7 +54,8 @@ export const logApiErrorCause = (
path: requestPath(request),
status: httpStatus(error),
errorTag: errorTag(error),
cause: loggedCause(cause),
// Error causes can contain provider responses, request bodies, and secrets.
// Keep only the classification; never serialize the exception or its stack.
});
};

Expand Down
37 changes: 18 additions & 19 deletions apps/cloud/src/observability/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,20 @@
// `withObservability` (in @executor-js/api) wraps every handler effect; when
// it sees an unmapped cause it asks `ErrorCapture.captureException` for a
// trace id and fails with `InternalError({ traceId })`. The client gets
// the opaque id, we get the full cause + stack in Sentry.
// the opaque id; Sentry receives a minimized diagnostic event.
// ---------------------------------------------------------------------------

import * as Sentry from "@sentry/cloudflare";
import type { ErrorEvent, Scope } from "@sentry/cloudflare";
import { Cause, Effect, Layer, Predicate } from "effect";
import type * as Tracer from "effect/Tracer";

import { minimizeDiagnosticTags, minimizeSentryEvent } from "./sentry-privacy";

import { ErrorCapture } from "@executor-js/api";
import { classifyDurableObjectError } from "@executor-js/cloudflare/mcp/durable-object-errors";
import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping";

// Drizzle/postgres-js include the failing SQL (params + bound values) in
// their error message. For OpenAPI source inserts that's 1MB+ of spec
// text which blows past terminal scrollback and hides the actual pg
// error. Sentry still receives the full, untruncated cause via
// `setExtra`; only the dev-console mirror is capped.
const MAX_CONSOLE_CAUSE_CHARS = 4_000;
const OTEL_TRACE_ID_PATTERN = /^[0-9a-f]{32}$/;
const OTEL_SPAN_ID_PATTERN = /^[0-9a-f]{16}$/;

Expand Down Expand Up @@ -52,11 +48,6 @@ export type OtelCorrelationContext = {
readonly spanId: string;
};

const truncate = (s: string): string =>
s.length <= MAX_CONSOLE_CAUSE_CHARS
? s
: `${s.slice(0, MAX_CONSOLE_CAUSE_CHARS)}\n鈥truncated ${s.length - MAX_CONSOLE_CAUSE_CHARS} chars]`;

const validOtelContext = (context: OtelCorrelationContext): boolean =>
OTEL_TRACE_ID_PATTERN.test(context.traceId) && OTEL_SPAN_ID_PATTERN.test(context.spanId);

Expand Down Expand Up @@ -212,7 +203,7 @@ export const beforeSendCloudEvent = (
options?: { readonly logPayload?: boolean },
): ErrorEvent | null => {
const reported = beforeSendWithOtelCorrelation(event, options);
return reported === null ? null : withStableGroupingFingerprint(reported);
return reported === null ? null : withStableGroupingFingerprint(minimizeSentryEvent(reported));
};

/**
Expand All @@ -230,8 +221,8 @@ export const beforeSendCloudEvent = (
export const cloudSentryOptions = (env: Env) => ({
dsn: env.SENTRY_DSN,
tracesSampleRate: 0,
enableLogs: true,
sendDefaultPii: true,
enableLogs: false,
sendDefaultPii: false,
skipOpenTelemetrySetup: true,
beforeSend: (event: ErrorEvent) =>
beforeSendCloudEvent(event, {
Expand Down Expand Up @@ -274,11 +265,12 @@ export const sentryPayloadForCause = (
// operation and no reason in it at all. Tags survive, group, and are
// searchable. Values are failure modes and operation names; never a query, a
// value, or anything customer-derived.
const CLASSIFICATION_TAG_FIELDS = ["operation", "reason", "status"] as const;
const CLASSIFICATION_TAG_FIELDS = ["operation", "reason", "status", "code"] as const;

/** The errors those fields are read from. An allowlist, because the fields are
* only known to be safe on the errors this app defines. */
const CLASSIFIED_ERROR_TAGS = [
"StorageError",
"UserStoreError",
"WorkOSError",
"McpSessionMetaUnavailableError",
Expand Down Expand Up @@ -323,7 +315,7 @@ const classificationTagsOf = (input: unknown): Readonly<Record<string, string>>
}
}
}
return tags;
return minimizeDiagnosticTags(tags);
};

export const captureCause = (
Expand Down Expand Up @@ -365,8 +357,15 @@ export const ErrorCaptureLive: Layer.Layer<ErrorCapture> = Layer.succeed(
ErrorCapture.of({
captureException: (cause) =>
Effect.gen(function* () {
console.error("[api] unhandled cause:", truncate(Cause.pretty(cause)));
return (yield* captureCauseEffect(cause)) ?? "";
const eventId = (yield* captureCauseEffect(cause)) ?? "";
console.error(
JSON.stringify({
event: "api_unhandled_cause",
sentry_event_id: eventId,
tags: classificationTagsOf(cause),
}),
);
return eventId;
}),
}),
);
101 changes: 101 additions & 0 deletions apps/cloud/src/observability/observability.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -409,3 +409,104 @@ describe("Durable Object platform reset noise", () => {
expect(options.beforeSend(event)).toBeNull();
});
});

describe("Sentry privacy boundary", () => {
it("rejects arbitrary values in classification tags and caller fingerprints", () => {
const secret = "SYNTHETIC_PRIVATE_MARKER";
const sent = beforeSendCloudEvent({
type: undefined,
fingerprint: [secret],
tags: {
operation: secret,
reason: secret,
status: secret,
otel_trace_id: secret,
otel_span_id: secret,
"mcp.do.cause_owner": secret,
code: secret,
"executor.ui.surface": secret,
"executor.ui.action": secret,
},
exception: { values: [{ type: secret, value: secret }] },
});
expect(sent).not.toBeNull();
expect(JSON.stringify(sent)).not.toContain(secret);
expect(sent?.exception?.values?.[0]?.type).toBe("Error");
});

it("retains known failure classifications and disables Sentry log payloads", () => {
const options = cloudSentryOptions({ SENTRY_DSN: "https://public@example.invalid/1" } as Env);
const sent = options.beforeSend({
type: undefined,
tags: { operation: "getOrganization", reason: "connect_timeout", status: 503 },
});
expect(sent?.tags).toEqual({
operation: "getOrganization",
reason: "connect_timeout",
status: "503",
});
expect(options.enableLogs).toBe(false);
expect(options.sendDefaultPii).toBe(false);
});

it("retains storage classifications without SQL or raw causes", () => {
const sent = beforeSendCloudEvent({
type: undefined,
tags: { operation: "connection.create", code: "22021" },
exception: { values: [{ type: "StorageError", value: "private SQL and bound values" }] },
extra: { cause: "private SQL and bound values" },
});
expect(sent?.tags).toEqual({ operation: "connection.create", code: "22021" });
expect(sent?.exception?.values?.[0]).toMatchObject({
type: "StorageError",
value: "connection.create failed (22021)",
});
expect(JSON.stringify(sent)).not.toContain("private SQL");
});

it("strips secrets from auto-captured errors while retaining diagnostic locations", () => {
const secret = "SYNTHETIC_PRIVATE_MARKER";
const sent = cloudSentryOptions({
SENTRY_DSN: "https://public@example.invalid/1",
} as Env).beforeSend({
type: undefined,
event_id: "safe-event-id",
message: secret,
user: { email: secret },
request: {
url: `https://example.test/?token=${secret}`,
headers: { authorization: secret },
data: secret,
},
extra: { cause: secret },
breadcrumbs: [{ message: secret }],
tags: { token: secret, otel_trace_id: traceId },
exception: {
values: [
{
type: "TypeError",
value: secret,
stacktrace: {
frames: [
{
filename: `/assets/example.js?token=${secret}`,
function: "handleRequest",
lineno: 42,
vars: { secret },
},
],
},
},
],
},
});
expect(JSON.stringify(sent)).not.toContain(secret);
expect(sent?.event_id).toBe("safe-event-id");
expect(sent?.tags?.otel_trace_id).toBe(traceId);
expect(sent?.exception?.values?.[0]?.stacktrace?.frames?.[0]).toMatchObject({
filename: "/assets/example.js",
function: "handleRequest",
lineno: 42,
});
});
});
Loading
Loading